gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/workeraccount.go

package renter

import (
	"bytes"
	"fmt"
	"io"
	"math/big"
	"net"
	"runtime"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"gitlab.com/SkynetLabs/skyd/build"
	"gitlab.com/SkynetLabs/skyd/persist"
	"gitlab.com/SkynetLabs/skyd/skymodules"
	"gitlab.com/SkynetLabs/skyd/skymodules/renter/contractor"
	"go.sia.tech/siad/crypto"
	"go.sia.tech/siad/modules"
	"go.sia.tech/siad/modules/host"
	"go.sia.tech/siad/types"

	"gitlab.com/NebulousLabs/errors"
	"gitlab.com/NebulousLabs/fastrand"
	"gitlab.com/NebulousLabs/siamux"
)

const (
	// accountBalanceDriftPercentageThreshold is the percentage threshold, in
	// relation to the balance target, at which we consider the drift to be
	// large enough to register an alert for it
	accountBalanceDriftPercentageThreshold = .1

	// withdrawalValidityPeriod defines the period (in blocks) a withdrawal
	// message remains spendable after it has been created. Together with the
	// current block height at time of creation, this period makes up the
	// WithdrawalMessage's expiry height.
	withdrawalValidityPeriod = 6

	// fundAccountGougingPercentageThreshold is the percentage threshold, in
	// relation to the allowance, at which we consider the cost of funding an
	// account to be too expensive. E.g. the cost of funding the account as many
	// times as necessary to spend the total allowance should never exceed 1% of
	// the total allowance.
	fundAccountGougingPercentageThreshold = .01
)

const (
	// the following categories are constants used to determine the
	// corresponding spending field in the account's spending details whenever
	// we pay for an rpc request using the ephemeral account as payment method.
	categoryErr spendingCategory = iota
	categoryAccountBalance
	categoryDownload
	categoryRegistryRead
	categoryRegistryWrite
	categoryRepairDownload
	categoryRepairUpload
	categorySnapshotDownload
	categorySnapshotUpload
	categorySubscription
	categoryUpdatePriceTable
	categoryUpload
)

var (
	// accountIdleCheckFrequency establishes how frequently the sync function
	// should check whether the worker is idle. A relatively high frequency is
	// okay, because this function only runs while the worker is frozen and
	// expecting to perform an expensive sync operation.
	accountIdleCheckFrequency = build.Select(build.Var{
		Dev:      time.Second * 4,
		Standard: time.Second * 5,
		Testing:  time.Second * 3,
	}).(time.Duration)

	// accountSyncRandWaitMilliseconds defines the number of random milliseconds
	// that are added to the wait time. Randomness is used to ensure that
	// workers are not all syncing at the same time - the sync operation freezes
	// workers. This number should be larger than the expected amount of time a
	// worker will be frozen multiplied by the total number of workers.
	accountSyncRandWaitMilliseconds = build.Select(build.Var{
		Dev:      int(1e3 * 60),          // 1 minute
		Standard: int(3 * 1e3 * 60 * 60), // 3 hours
		Testing:  int(1e3 * 15),          // 15 seconds - needs to be long even in testing
	}).(int)

	// accountSyncMinWaitTime defines the minimum amount of time that a worker
	// will wait between performing sync checks. This should be a large number,
	// on the order of the amount of time a worker is expected to be frozen
	// multiplied by the total number of workers.
	accountSyncMinWaitTime = build.Select(build.Var{
		Dev:      time.Minute,
		Standard: 60 * time.Minute, // 1 hour
		Testing:  10 * time.Second, // needs to be long even in testing
	}).(time.Duration)

	// accountIdleMaxWait defines the max amount of time that the worker will
	// wait to reach an idle state before firing a build.Critical and giving up
	// on becoming idle. Generally this will indicate that somewhere in the
	// worker code there is a job that is not timing out correctly.
	accountIdleMaxWait = build.Select(build.Var{
		Dev:      10 * time.Minute,
		Standard: 40 * time.Minute,
		Testing:  5 * time.Minute, // needs to be long even in testing
	}).(time.Duration)
)

type (
	// account represents a renter's ephemeral account on a host.
	account struct {
		// Information related to host communications.
		staticID        modules.AccountID
		staticHostKey   types.SiaPublicKey
		staticSecretKey crypto.SecretKey

		// Money has multiple states in an account, this is all the information
		// we need to understand the current state of the account's balance and
		// pending updates.
		//
		// The two drift fields keep track of the delta between our version of
		// the balance and the host's version of the balance. We want to keep
		// track of this drift as in the future we might add code that acts upon
		// it and penalizes the host if we find they are cheating us, or
		// behaving sub-optimally.
		balance              types.Currency
		balanceDriftPositive types.Currency
		balanceDriftNegative types.Currency
		pendingDeposits      types.Currency
		pendingWithdrawals   types.Currency
		negativeBalance      types.Currency

		// hostBalance is a field in which we store the balance the host claims
		// is in the account when we perform a sync. We can not reset our
		// balance to that as we can not blindly trust the host, but we need to
		// store it nonetheless to be able to accurately reflect the accrued
		// positive or negative balance drift
		//
		// hostBalanceNegative allows the hostBalance to go negative
		hostBalance         types.Currency
		hostBalanceNegative types.Currency

		// residue is the amount of money that is still left in the
		// ephemeral account at the time the contract renews and the
		// FundAccountCost resets, we need to store this field in order to
		// properly represent the spending details in the next period.
		residue types.Currency

		// Spending details contain a breakdown of how much money from the
		// ephemeral account got spent on what type of action. Examples of such
		// actions are downloads, registry reads, registry writes, etc.
		spending spendingDetails

		// Error tracking.
		recentErr         error
		recentErrTime     time.Time
		recentSuccessTime time.Time

		// syncAt defines what time the renter should be syncing the account to
		// the host.
		syncAt time.Time

		// forcedSyncAt is the time at which the renter scheduled a forced sync
		// of the renter's ea balance with the host
		forcedSyncAt time.Time

		// Variables to manage a race condition around account creation, where
		// the account must be available in the data structure before it has
		// been synced to disk successfully (to avoid holding a lock on the
		// account manager during a disk fsync). Anyone trying to use the
		// account will need to block on 'staticReady', and then after that is
		// closed needs to check the status of 'externActive', 'false'
		// indicating that account creation failed and the account was deleted.
		//
		// 'externActive' can be accessed freely once 'staticReady' has been
		// closed.
		staticReady  chan struct{}
		externActive bool

		// Utils. The offset refers to the offset within the file that the
		// account uses.
		mu                  sync.Mutex
		staticAlerter       *modules.GenericAlerter
		staticBalanceTarget types.Currency
		staticFile          modules.File
		staticLog           *persist.Logger
		staticOffset        int64
	}

	// spendingDetails contains a breakdown of all spending metrics, all money
	// that is being spent from an ephemeral account is accounted for in one of
	// these categories. Every field of this struct should have a corresponding
	// 'spendingCategory'.
	spendingDetails struct {
		accountBalance    types.Currency
		downloads         types.Currency
		registryReads     types.Currency
		registryWrites    types.Currency
		repairDownloads   types.Currency
		repairUploads     types.Currency
		snapshotDownloads types.Currency
		snapshotUploads   types.Currency
		subscriptions     types.Currency
		updatePriceTable  types.Currency
		uploads           types.Currency
	}

	// spendingCategory defines an enum that represent a category in the
	// spending details
	spendingCategory uint64
)

// sum returns the sum of all spending details
func (s *spendingDetails) sum() types.Currency {
	var total types.Currency
	total = total.Add(s.accountBalance)
	total = total.Add(s.downloads)
	total = total.Add(s.registryReads)
	total = total.Add(s.registryWrites)
	total = total.Add(s.repairDownloads)
	total = total.Add(s.repairUploads)
	total = total.Add(s.snapshotDownloads)
	total = total.Add(s.snapshotUploads)
	total = total.Add(s.subscriptions)
	total = total.Add(s.updatePriceTable)
	total = total.Add(s.uploads)
	return total
}

// add will add the given amount to the appropriate spending category
func (s *spendingDetails) add(category spendingCategory, amount types.Currency) {
	s.set(category, s.get(category).Add(amount))
}

// get returns the spending details for given category
func (s *spendingDetails) get(category spendingCategory) types.Currency {
	switch category {
	case categoryAccountBalance:
		return s.accountBalance
	case categoryDownload:
		return s.downloads
	case categorySnapshotDownload:
		return s.snapshotDownloads
	case categorySnapshotUpload:
		return s.snapshotUploads
	case categoryRegistryRead:
		return s.registryReads
	case categoryRegistryWrite:
		return s.registryWrites
	case categoryRepairDownload:
		return s.repairDownloads
	case categoryRepairUpload:
		return s.repairUploads
	case categorySubscription:
		return s.subscriptions
	case categoryUpdatePriceTable:
		return s.updatePriceTable
	case categoryUpload:
		return s.uploads
	case categoryErr:
		build.Critical("category is not set, developer error")
	default:
		build.Critical("category is not handled, developer error")
	}

	return types.ZeroCurrency
}

// set sets the spending details for given category and amount
func (s *spendingDetails) set(category spendingCategory, amount types.Currency) {
	switch category {
	case categoryAccountBalance:
		s.accountBalance = amount
	case categoryDownload:
		s.downloads = amount
	case categorySnapshotDownload:
		s.snapshotDownloads = amount
	case categorySnapshotUpload:
		s.snapshotUploads = amount
	case categoryRegistryRead:
		s.registryReads = amount
	case categoryRegistryWrite:
		s.registryWrites = amount
	case categoryRepairDownload:
		s.repairDownloads = amount
	case categoryRepairUpload:
		s.repairUploads = amount
	case categorySubscription:
		s.subscriptions = amount
	case categoryUpdatePriceTable:
		s.updatePriceTable = amount
	case categoryUpload:
		s.uploads = amount
	case categoryErr:
		build.Critical("category is not set, developer error")
	default:
		build.Critical("category is not handled, developer error")
	}
}

// sub will subtract the given amount from the appropriate spending category
func (s *spendingDetails) sub(category spendingCategory, amount types.Currency) {
	// safeSub is a helper function that subtracts two currencies, if the result
	// would be negative the zero currency gets returned
	safeSub := func(x, y types.Currency) types.Currency {
		if y.Cmp(x) > 0 {
			return types.ZeroCurrency
		}
		return x.Sub(y)
	}

	s.set(category, safeSub(s.get(category), amount))
}

// ProvidePayment takes a stream and various payment details and handles the
// payment by sending and processing payment request and response objects.
// Returns an error in case of failure.
//
// Note that this implementation does not 'Read' from the stream. This allows
// the caller to pass in a buffer if he so pleases in order to optimise the
// amount of writes on the actual stream.
func (a *account) ProvidePayment(stream io.ReadWriter, amount types.Currency, blockHeight types.BlockHeight) error {
	// NOTE: we purposefully do not verify if the account has sufficient funds.
	// Seeing as withdrawals are a blocking action on the host, it is perfectly
	// ok to trigger them from an account with insufficient balance.

	// create a withdrawal message
	msg := newWithdrawalMessage(a.staticID, amount, blockHeight)
	sig := crypto.SignHash(crypto.HashObject(msg), a.staticSecretKey)

	buffer := staticPoolProvidePaymentBuffers.Get()
	defer staticPoolProvidePaymentBuffers.Put(buffer)

	// send PaymentRequest
	err := modules.RPCWrite(buffer, modules.PaymentRequest{Type: modules.PayByEphemeralAccount})
	if err != nil {
		return err
	}

	// send PayByEphemeralAccountRequest
	err = modules.RPCWrite(buffer, modules.PayByEphemeralAccountRequest{
		Message:   msg,
		Signature: sig,
	})
	if err != nil {
		return err
	}
	_, err = buffer.WriteTo(stream)
	return err
}

// String returns a string representation of the account
func (a *account) String() string {
	var sb strings.Builder
	sb.WriteString("HostKey: " + a.staticHostKey.String() + "\n")
	sb.WriteString("\n")
	sb.WriteString("Balance: " + a.balance.HumanString() + "\n")
	sb.WriteString("    Balance Drift Pos  : " + a.balanceDriftPositive.HumanString() + "\n")
	sb.WriteString("    Balance Drift Neg  : " + a.balanceDriftNegative.HumanString() + "\n")
	sb.WriteString("    Pending Deposits   : " + a.pendingDeposits.HumanString() + "\n")
	sb.WriteString("    Pending Withdrawals: " + a.pendingWithdrawals.HumanString() + "\n")
	sb.WriteString("    Negative Balance   : " + a.negativeBalance.HumanString() + "\n")
	sb.WriteString("Residue: " + a.residue.HumanString() + "\n")
	sb.WriteString("\n")
	sb.WriteString("Spending Details: \n")
	sb.WriteString("    Account Balance   : " + a.spending.accountBalance.HumanString() + "\n")
	sb.WriteString("    Downloads         : " + a.spending.downloads.HumanString() + "\n")
	sb.WriteString("    Registry Reads    : " + a.spending.registryReads.HumanString() + "\n")
	sb.WriteString("    Registry Writes   : " + a.spending.registryWrites.HumanString() + "\n")
	sb.WriteString("    Repair Downloads  : " + a.spending.repairDownloads.HumanString() + "\n")
	sb.WriteString("    Repair Uploads    : " + a.spending.repairUploads.HumanString() + "\n")
	sb.WriteString("    Snapshot Downloads: " + a.spending.snapshotDownloads.HumanString() + "\n")
	sb.WriteString("    Snapshot Uploads  : " + a.spending.snapshotUploads.HumanString() + "\n")
	sb.WriteString("    Subscriptions     : " + a.spending.subscriptions.HumanString() + "\n")
	sb.WriteString("    Update PriceTable : " + a.spending.updatePriceTable.HumanString() + "\n")
	sb.WriteString("    Uploads           : " + a.spending.uploads.HumanString() + "\n")
	sb.WriteString("\n")
	sb.WriteString("Host Balance: \n")
	sb.WriteString("    Host Balance    : " + a.hostBalance.HumanString() + "\n")
	sb.WriteString("    Host Balance Neg: " + a.hostBalanceNegative.HumanString() + "\n")
	sb.WriteString("\n")
	return sb.String()
}

// availableBalance returns a new balance that takes into account pending spends
// and pending funds, yielding the balance that is available
func (a *account) availableBalance() types.Currency {
	return availableBalance(a.balance, a.negativeBalance, a.pendingDeposits, a.pendingWithdrawals)
}

// availableHostBalance returns a new balance, based on the host balance fields,
// that takes into account pending spends and pending funds
func (a *account) availableHostBalance() types.Currency {
	return availableBalance(a.hostBalance, a.hostBalanceNegative, a.pendingDeposits, a.pendingWithdrawals)
}

// callForcedSyncScheduled returns whether or not a forced account sync was
// scheduled.
func (a *account) callForcedSyncScheduled() bool {
	a.mu.Lock()
	defer a.mu.Unlock()
	return !a.forcedSyncAt.IsZero()
}

// callNeedsToSync returns whether or not the account needs to sync to the host.
func (a *account) callNeedsToSync() bool {
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.syncAt.Before(time.Now())
}

// callSpendingDetails returns the spending details for the account
func (a *account) callSpendingDetails() spendingDetails {
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.spending
}

// deposit commits a pending deposit, either after success or failure.
// Depending on the outcome the given amount will be added to the balance or
// not. If the pending delta is zero, and we altered the account balance, we
// update the account.
func (a *account) deposit(amount types.Currency) {
	// reflect the deposit in the balance field
	if amount.Cmp(a.negativeBalance) <= 0 {
		a.negativeBalance = a.negativeBalance.Sub(amount)
	} else {
		amount = amount.Sub(a.negativeBalance)
		a.negativeBalance = types.ZeroCurrency
		a.balance = a.balance.Add(amount)
	}

	// mirror the balance updates on the host balance, the host balance
	// fields are kept purely for tracking the drift
	if amount.Cmp(a.hostBalanceNegative) <= 0 {
		a.hostBalanceNegative = a.hostBalanceNegative.Sub(amount)
	} else {
		amount = amount.Sub(a.hostBalanceNegative)
		a.hostBalanceNegative = types.ZeroCurrency
		a.hostBalance = a.hostBalance.Add(amount)
	}
}

// managedAvailableBalance returns the amount of money that is available to
// spend. It is calculated by taking into account pending spends and pending
// funds.
func (a *account) managedAvailableBalance() types.Currency {
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.availableBalance()
}

// managedMaxExpectedBalance returns the max amount of money that this
// account is expected to contain after the renter has shut down.
func (a *account) managedMaxExpectedBalance() types.Currency {
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.maxExpectedBalance()
}

// managedResetSpending is called when the contract corresponding to this
// account gets renewed, when that happens the FundAccountCost is zero again,
// and thus the spending details have to be reset. In order to properly reflect
// the spending breakdown, we have to keep track of the current balance we carry
// over into the next period, which we call the residue.
func (a *account) managedResetSpending() {
	a.mu.Lock()
	defer a.mu.Unlock()

	// set the residue to the current balance and reset the spending details
	a.residue = a.balance
	a.spending = spendingDetails{}

	// persist the account upon reset
	err := a.persist()
	if err != nil {
		a.staticLog.Critical(fmt.Sprintf("failed to persist account, err: %v", err))
	}
}

// maxExpectedBalance returns the max amount of money that this account is
// expected to contain after the renter has shut down.
func (a *account) maxExpectedBalance() types.Currency {
	// NOTE: there is an edge case where an EA has some drift, positive drift
	// where the host balance is higher than the renter balance, if the account
	// is prevented from being refilled, the negative balance can be higher than
	// the expected balance so we have to safe sub here
	expectedBalance := a.balance.Add(a.pendingDeposits)
	if a.negativeBalance.Cmp(expectedBalance) > 0 {
		return types.ZeroCurrency
	}
	return expectedBalance.Sub(a.negativeBalance)
}

// managedMinExpectedBalance returns the min amount of money that this
// account is expected to contain after the renter has shut down.
func (a *account) managedMinExpectedBalance() types.Currency {
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.minExpectedBalance()
}

// minExpectedBalance returns the min amount of money that this account is
// expected to contain after the renter has shut down.
func (a *account) minExpectedBalance() types.Currency {
	return minExpectedBalance(a.balance, a.negativeBalance, a.pendingWithdrawals)
}

// managedCommitDeposit commits a pending deposit, either after success or
// failure. Depending on the outcome the given amount will be added to the
// balance or not. If the pending delta is zero, and we altered the account
// balance, we update the account.
func (a *account) managedCommitDeposit(amount types.Currency, success bool) {
	a.mu.Lock()
	defer a.mu.Unlock()

	// (no need to sanity check - the implementation of 'Sub' does this for us)
	a.pendingDeposits = a.pendingDeposits.Sub(amount)

	// deposit the money in case of success
	if success {
		a.deposit(amount)
	}
}

// managedCommitRefund commits a refund for the given spending category, it's
// treated as a simple deposit and subtracted from the corresponding field in
// the spending details
func (a *account) managedCommitRefund(category spendingCategory, refund types.Currency) {
	a.mu.Lock()
	defer a.mu.Unlock()
	a.deposit(refund)
	a.trackSpending(category, refund, true)
}

// managedCommitWithdrawal commits a pending withdrawal, either after success or
// failure. Depending on the outcome the given withdrawal amount will be
// deducted from the balance or not. If the pending delta is zero, and we
// altered the account balance, we update the account. The refund is given
// because both the refund and the withdrawal amount need to be subtracted from
// the pending withdrawals, seeing is it is no longer 'pending'. Only the
// withdrawal amount has to be subtracted from the balance because the refund
// got refunded by the host already.
func (a *account) managedCommitWithdrawal(category spendingCategory, withdrawal, refund types.Currency, withdrawalErr error) {
	a.mu.Lock()
	defer a.mu.Unlock()

	// conventience variables
	alerter := a.staticAlerter
	balanceTarget := a.staticBalanceTarget
	workerHostKey := a.staticHostKey.String()

	// (no need to sanity check - the implementation of 'Sub' does this for us)
	a.pendingWithdrawals = a.pendingWithdrawals.Sub(withdrawal.Add(refund))

	// reflect the successful withdrawal in the balance field
	if withdrawalErr == nil {
		if a.balance.Cmp(withdrawal) >= 0 {
			a.balance = a.balance.Sub(withdrawal)
		} else {
			remainder := withdrawal.Sub(a.balance)
			a.balance = types.ZeroCurrency
			a.negativeBalance = a.negativeBalance.Add(remainder)
		}

		// mirror the balance updates on the host balance, the host balance
		// fields are kept purely for tracking the drift
		if a.hostBalance.Cmp(withdrawal) >= 0 {
			a.hostBalance = a.hostBalance.Sub(withdrawal)
		} else {
			remainder := withdrawal.Sub(a.hostBalance)
			a.hostBalance = types.ZeroCurrency
			a.hostBalanceNegative = a.hostBalanceNegative.Add(remainder)
		}

		// only in case of success we track the spend and what it was spent on
		a.trackSpending(category, withdrawal, false)
	}

	// register an alert if the host indicates our EA is out of funds, but the
	// renter's account has not even reached the refill threshold, unregister it
	// if a withdrawal was successful
	if isInsufficientBalanceError(withdrawalErr) && !a.needsToRefill(balanceTarget.Div64(2)) {
		msg := fmt.Sprintf("host %v indicates the ephemeral account is out of balance while the account has not exceeded the refill threshold", workerHostKey)
		alerter.RegisterAlert(skymodules.AlertIDWorkerAccountOutOfFunds(workerHostKey), msg, workerHostKey, modules.SeverityError)
	} else if withdrawalErr == nil {
		alerter.UnregisterAlert(skymodules.AlertIDWorkerAccountOutOfFunds(workerHostKey))
	}
}

// managedNeedsToRefill returns whether or not the account needs to be refilled.
func (a *account) managedNeedsToRefill(target types.Currency) bool {
	a.mu.Lock()
	defer a.mu.Unlock()
	return a.needsToRefill(target)
}

// managedSyncBalance updates the account's balance related fields to "sync"
// with the given balance, which was returned by the host. If the given balance
// is higher or lower than the account's available balance, we update the drift
// fields in the positive or negative direction. The method returns the drift
// between the given host balance and the renter balance.
func (a *account) managedSyncBalance(balance types.Currency, forced bool) *big.Int {
	a.mu.Lock()
	defer a.mu.Unlock()

	// Reset the host balance on every sync
	defer a.resetHostBalance(balance)

	// Reset the forced sync at if this sync was forced
	if forced {
		a.forcedSyncAt = time.Time{}
	}

	// Determine how long to wait before attempting to sync again, and then
	// update the syncAt time. There is significant randomness in the
	// waiting because syncing with the host requires freezing up the
	// worker. We do not want to freeze up a large number of workers at
	// once, nor do we want to freeze them frequently.
	defer func() {
		randWait := fastrand.Intn(accountSyncRandWaitMilliseconds)
		waitTime := time.Duration(randWait) * time.Millisecond
		waitTime += accountSyncMinWaitTime
		a.syncAt = time.Now().Add(waitTime)
	}()

	// If our balance is equal to what the host communicated, we're done.
	drift := big.NewInt(0)
	currBalance := a.availableBalance()
	if currBalance.Equals(balance) {
		return drift
	}

	// However, if it is lower, or if we want to force sync, we want to reset
	// our account balance
	if forced || currBalance.Cmp(balance) < 0 {
		a.resetBalance(balance)
	}

	// Calculate the drift, which is negative if the host reports a lower balance
	if currBalance.Cmp(balance) < 0 {
		drift = balance.Sub(currBalance).Big()
	} else {
		drift = drift.Neg(currBalance.Sub(balance).Big())
	}

	// To avoid an exponential increase of the positive and negative drift
	// values, we keep track of the host's balance and reset it on every sync.
	// That way, the positive and negative deltas values measure the drift since
	// the last sync, as opposed to the drift since the account was created
	currBalance = a.availableHostBalance()
	if currBalance.Cmp(balance) < 0 {
		delta := balance.Sub(currBalance)
		a.balanceDriftPositive = a.balanceDriftPositive.Add(delta)
	}

	// If it's higher we only track the amount we drifted.
	if currBalance.Cmp(balance) > 0 {
		delta := currBalance.Sub(balance)
		a.balanceDriftNegative = a.balanceDriftNegative.Add(delta)
	}

	// Persist the account
	err := a.persist()
	if err != nil {
		a.staticLog.Printf("could not persist account, err: %v\n", err)
	}

	return drift
}

// managedStatus returns the status of the account
func (a *account) managedStatus() skymodules.WorkerAccountStatus {
	a.mu.Lock()
	defer a.mu.Unlock()

	var recentErrStr string
	if a.recentErr != nil {
		recentErrStr = a.recentErr.Error()
	}

	return skymodules.WorkerAccountStatus{
		AvailableBalance: a.availableBalance(),
		NegativeBalance:  a.negativeBalance,
		HostBalance:      a.hostBalance,

		SyncAt:       a.syncAt,
		ForcedSyncAt: a.forcedSyncAt,

		RecentErr:         recentErrStr,
		RecentErrTime:     a.recentErrTime,
		RecentSuccessTime: a.recentSuccessTime,
	}
}

// managedTrackDeposit keeps track of pending deposits by adding the given
// amount to the 'pendingDeposits' field.
func (a *account) managedTrackDeposit(amount types.Currency) {
	a.mu.Lock()
	defer a.mu.Unlock()
	a.pendingDeposits = a.pendingDeposits.Add(amount)
}

// managedTrackWithdrawal keeps track of pending withdrawals by adding the given
// amount to the 'pendingWithdrawals' field.
func (a *account) managedTrackWithdrawal(amount types.Currency) {
	a.mu.Lock()
	defer a.mu.Unlock()
	a.pendingWithdrawals = a.pendingWithdrawals.Add(amount)
}

// needsToRefill returns whether or not the account needs to be refilled.
func (a *account) needsToRefill(target types.Currency) bool {
	return a.availableBalance().Cmp(target) < 0
}

// resetBalance sets the given balance and resets the account's balance
// delta state variables. This happens when we have performanced a balance
// inquiry on the host and we decide to trust his version of the balance.
func (a *account) resetBalance(balance types.Currency) {
	a.balance = balance
	a.pendingDeposits = types.ZeroCurrency
	a.pendingWithdrawals = types.ZeroCurrency
	a.negativeBalance = types.ZeroCurrency
}

// resetHostBalance sets the given balance and resets the account's host balance
// delta state variables. This happens every time we get a balance from the host.
func (a *account) resetHostBalance(balance types.Currency) {
	a.hostBalance = balance
	a.hostBalanceNegative = types.ZeroCurrency
}

// trackSpending will keep track of the amount spent for the given spend category
func (a *account) trackSpending(category spendingCategory, amount types.Currency, refund bool) {
	// sanity check the category was set
	if category == categoryErr {
		build.Critical("tracked a spend using an uninitialized category, this is prevented as we want to track all money that is being spent without exception")
		return
	}

	// update the spending metrics
	if refund {
		a.spending.sub(category, amount)
	} else {
		a.spending.add(category, amount)
	}

	// every time we update we write the account to disk
	err := a.persist()
	if err != nil {
		a.staticLog.Critical(fmt.Sprintf("failed to persist account, err: %v", err))
	}
}

// newWithdrawalMessage is a helper function that takes a set of parameters and
// a returns a new WithdrawalMessage.
func newWithdrawalMessage(id modules.AccountID, amount types.Currency, blockHeight types.BlockHeight) modules.WithdrawalMessage {
	expiry := blockHeight + withdrawalValidityPeriod
	var nonce [modules.WithdrawalNonceSize]byte
	fastrand.Read(nonce[:])
	return modules.WithdrawalMessage{
		Account: id,
		Expiry:  expiry,
		Amount:  amount,
		Nonce:   nonce,
	}
}

// callScheduleForcedAccountSync schedules a forced sync of the worker's
// ephemeral account with the host's balance.
func (w *worker) callScheduleForcedAccountSync() {
	w.staticAccount.mu.Lock()
	defer w.staticAccount.mu.Unlock()
	w.staticAccount.forcedSyncAt = time.Now()
}

// externSyncAccountBalanceToHost is executed before the worker loop and
// corrects the account balance in case of an unclean renter shutdown. It does
// so by performing the AccountBalanceRPC and resetting the account to the
// balance communicated by the host. This only happens if our account balance is
// zero, which indicates an unclean shutdown.
//
// NOTE: it is important this function is only used when the worker has no
// in-progress jobs, neither serial nor async, to ensure the account balance
// sync does not leave the account in an undesired state. The worker should not
// be launching new jobs while this function is running. To achieve this, we
// ensure that this thread is only run from the primary work loop, which is also
// the only thread that is allowed to launch jobs. As long as this function is
// only called by that thread, and no other thread launches jobs, this function
// is threadsafe.
func (w *worker) externSyncAccountBalanceToHost(forced bool) error {
	// Return if the renter's shutting down
	if w.staticIsShuttingDown() {
		return nil
	}

	// Spin/block until the worker has no jobs in motion. This should only be
	// called from the primary loop of the worker, meaning that no new jobs will
	// be launched while we spin.
	isIdle := func() bool {
		sls := w.staticLoopState
		a := atomic.LoadUint64(&sls.atomicSerialJobRunning) == 0
		b := atomic.LoadUint64(&sls.atomicAsyncJobsRunning) == 0
		return a && b
	}
	start := time.Now()
	for !isIdle() {
		if time.Since(start) > accountIdleMaxWait {
			// The worker failed to go idle for too long. Print the loop state,
			// so we know what kind of task is keeping it busy.
			w.staticRenter.staticLog.Printf("Worker static loop state: %+v\n\n", w.staticLoopState)
			// Get the stack traces of all running goroutines.
			buf := make([]byte, skymodules.StackSize) // 64MB
			n := runtime.Stack(buf, true)
			w.staticRenter.staticLog.Println(string(buf[:n]))
			w.staticRenter.staticLog.Critical(fmt.Sprintf("worker has taken more than %v minutes to go idle", accountIdleMaxWait.Minutes()))
			return nil
		}
		awake := w.staticTG.Sleep(accountIdleCheckFrequency)
		if !awake {
			return nil
		}
	}

	// Grab the account sync lock to prevent the subscription loop from
	// starting new pending deposits or withdrawals. We do this after the
	// idle check since it might take a while for all jobs to finish and we
	// don't want to unnecessarily block subscriptions.
	w.accountSyncMu.Lock()
	defer w.accountSyncMu.Unlock()

	// Do a check to ensure that the worker is still idle after the function is
	// complete. This should help to catch any situation where the worker is
	// spinning up new jobs, even though it is not supposed to be spinning up
	// new jobs while it is performing the sync operation.
	defer func() {
		if !isIdle() {
			w.staticRenter.staticLog.Critical("worker appears to be spinning up new jobs during managedSyncAccountBalanceToHost")
		}
	}()

	// Sanity check the account's deltas are zero, indicating there are no
	// in-progress jobs
	w.staticAccount.mu.Lock()
	deltasAreZero := w.staticAccount.pendingDeposits.IsZero() && w.staticAccount.pendingWithdrawals.IsZero()
	w.staticAccount.mu.Unlock()
	if !deltasAreZero {
		build.Critical("managedSyncAccountBalanceToHost is called on a worker with an account that has non-zero deltas, indicating in-progress jobs")
	}

	// Track the outcome of the account sync - this ensures a proper working of
	// the maintenance cooldown mechanism.
	balance, err := w.managedHostAccountBalance()
	w.managedTrackAccountSyncErr(err)
	if err != nil {
		w.staticRenter.staticLog.Debugf("ERROR: failed to check account balance on host %v failed, err: %v\n", w.staticHostPubKeyStr, err)
		return err
	}

	// Sync the account with the host's version of our balance. This will update
	// our balance in case the host tells us we actually have more money, and it
	// will keep track of drift in both directions.
	drift := w.staticAccount.managedSyncBalance(balance, forced)
	if drift.Sign() == -1 {
		// if we have negative drift, we want to register an alert if it exceeds a certain threshold
		driftCurr := types.NewCurrency(drift.Abs(drift))
		threshold := w.staticRenter.staticAccountBalanceTarget.MulFloat(accountBalanceDriftPercentageThreshold)
		if driftCurr.Cmp(threshold) > 0 {
			msg := fmt.Sprintf("host %v, host balance has drifted away from the renter balance by %v", w.staticHostPubKeyStr, driftCurr.HumanString())
			w.staticRenter.staticAlerter.RegisterAlert(skymodules.AlertIDWorkerAccountBalanceDrift(w.staticHostPubKeyStr), msg, "", modules.SeverityWarning)
		}
	} else {
		// if we're in sync with the host balance, unregister the drift alert
		w.staticRenter.staticAlerter.UnregisterAlert(skymodules.AlertIDWorkerAccountBalanceDrift(w.staticHostPubKeyStr))
	}

	// TODO perform a thorough balance comparison to decide whether the drift in
	// the account balance is warranted. If not the host needs to be penalized
	// accordingly. Perform this check at startup and periodically.

	return nil
}

// managedForcedAccountSyncScheduled returns true if a forced sync was
// scheduled. A forced sync means we will adopt the host's balance regardless of
// the renter's account balance.
func (w *worker) managedForcedAccountSyncScheduled() bool {
	w.staticAccount.mu.Lock()
	defer w.staticAccount.mu.Unlock()
	return !w.staticAccount.forcedSyncAt.IsZero()
}

// managedNeedsToRefillAccount will check whether the worker's account needs to
// be refilled. This function will return false if any conditions are met which
// are likely to prevent the refill from being successful.
func (w *worker) managedNeedsToRefillAccount() (refill bool) {
	// No need to refill the account if the worker is on maintenance cooldown.
	if w.managedOnMaintenanceCooldown() {
		return false
	}

	// No need to refill if the price table is not valid, as it would only
	// result in failure anyway.
	if !w.staticPriceTable().staticValid() {
		return false
	}

	return w.staticAccount.managedNeedsToRefill(w.staticRenter.staticAccountBalanceTarget.Div64(2))
}

// managedNeedsToSyncAccountBalanceToHost returns whether the renter needs to
// sync the renter's account balance with the host's version of the account. The
// second return value indicates whether the sync was forced and should adopt
// the host balance.
func (w *worker) managedNeedsToSyncAccountBalanceToHost() (bool, bool) {
	// No need to sync if the price table is not valid, as it would only
	// result in failure anyway.
	if !w.staticPriceTable().staticValid() {
		return false, false
	}

	// Check whether we want to force sync the account
	if w.managedForcedAccountSyncScheduled() {
		return true, true
	}

	// No need to sync the account if the worker's RHP3 is on cooldown.
	if w.managedOnMaintenanceCooldown() {
		return false, false
	}

	return w.staticAccount.callNeedsToSync(), false
}

// managedRefillAccount will refill the account if it needs to be refilled
func (w *worker) managedRefillAccount() (err error) {
	// Disrupt the refill
	deps := w.staticRenter.staticDeps
	if deps.Disrupt("DisableFunding") || deps.Disrupt("DisableRefill") {
		return
	}

	// Sanity check we have a valid price table
	if !w.staticPriceTable().staticValid() {
		return errors.New("can not refill account with an invalid price table")
	}

	// The account balance dropped to below half the balance target, refill. Use
	// the max expected balance when refilling to avoid exceeding any host
	// maximums.
	balance := w.staticAccount.managedMaxExpectedBalance()
	amount := types.ZeroCurrency
	if w.staticRenter.staticAccountBalanceTarget.Cmp(balance) > 0 {
		amount = w.staticRenter.staticAccountBalanceTarget.Sub(balance)
	}
	if amount.IsZero() {
		return // nothing to do
	}
	pt := w.staticPriceTable().staticPriceTable

	// If the target amount is larger than the remaining money, adjust the
	// target. Make sure it can still cover the funding cost.
	if contract, ok := w.staticRenter.staticHostContractor.ContractByPublicKey(w.staticHostPubKey); ok {
		if amount.Add(pt.FundAccountCost).Cmp(contract.RenterFunds) > 0 && contract.RenterFunds.Cmp(pt.FundAccountCost) > 0 {
			amount = contract.RenterFunds.Sub(pt.FundAccountCost)
		}
	}

	// We track that there is a deposit in progress. Because filling an account
	// is an interactive protocol with another machine, we are never sure of the
	// exact moment that the deposit has reached our account. Instead, we track
	// the deposit as a "maybe" until we know for sure that the deposit has
	// either reached the remote machine or failed.
	//
	// At the same time that we track the deposit, we defer a function to check
	// the error on the deposit
	w.staticAccount.managedTrackDeposit(amount)
	defer func() {
		// If there was no error, the account should now be full, and will not
		// need to be refilled until the worker has spent up the funds in the
		// account.
		w.staticAccount.managedCommitDeposit(amount, err == nil)

		// Track the outcome of the account refill - this ensures a proper
		// working of the maintenance cooldown mechanism.
		cd := w.managedTrackAccountRefillErr(err)

		// If the error is nil, return.
		if err == nil {
			w.staticAccount.mu.Lock()
			w.staticAccount.recentSuccessTime = time.Now()
			w.staticAccount.mu.Unlock()
			return
		}

		// Track the error on the account for debugging purposes.
		w.staticAccount.mu.Lock()
		w.staticAccount.recentErr = err
		w.staticAccount.recentErrTime = time.Now()
		w.staticAccount.mu.Unlock()

		// Have the threadgroup wake the worker when the account comes off of
		// cooldown.
		w.staticTG.AfterFunc(time.Until(cd), func() {
			w.staticWake()
		})
	}()

	// check the current price table for gouging errors
	err = checkFundAccountGouging(w.staticPriceTable().staticPriceTable, w.staticCache().staticRenterAllowance, w.staticRenter.staticAccountBalanceTarget)
	if err != nil {
		return
	}

	// create a new stream
	var stream net.Conn
	stream, err = w.staticNewStream()
	if err != nil {
		err = errors.AddContext(err, "Unable to create a new stream")
		return
	}
	defer func() {
		closeErr := stream.Close()
		if closeErr != nil {
			w.staticRenter.staticLog.Println("ERROR: failed to close stream", closeErr)
		}
	}()

	// prepare a buffer so we can optimize our writes
	buffer := bytes.NewBuffer(nil)

	// write the specifier
	err = modules.RPCWrite(buffer, modules.RPCFundAccount)
	if err != nil {
		err = errors.AddContext(err, "could not write fund account specifier")
		return
	}

	// send price table uid
	err = modules.RPCWrite(buffer, pt.UID)
	if err != nil {
		err = errors.AddContext(err, "could not write price table uid")
		return
	}

	// send fund account request
	err = modules.RPCWrite(buffer, modules.FundAccountRequest{Account: w.staticAccount.staticID})
	if err != nil {
		err = errors.AddContext(err, "could not write the fund account request")
		return
	}

	// write contents of the buffer to the stream
	_, err = stream.Write(buffer.Bytes())
	if err != nil {
		err = errors.AddContext(err, "could not write the buffer contents")
		return
	}

	// build payment details
	details := contractor.PaymentDetails{
		Host:          w.staticHostPubKey,
		Amount:        amount.Add(pt.FundAccountCost),
		RefundAccount: modules.ZeroAccountID,
		SpendingDetails: skymodules.SpendingDetails{
			FundAccountSpending: amount,
			MaintenanceSpending: skymodules.MaintenanceSpending{
				FundAccountCost: pt.FundAccountCost,
			},
		},
	}

	// provide payment
	err = w.staticRenter.staticHostContractor.ProvidePayment(stream, &pt, details)
	if err != nil && strings.Contains(err.Error(), "balance exceeded") {
		// The host reporting that the balance has been exceeded suggests that
		// the host believes that we have more money than we believe that we
		// have.
		if !w.staticRenter.staticDeps.Disrupt("DisableCriticalOnMaxBalance") {
			// Log a critical in testing as this is very unlikely to happen due
			// to the order of events in the worker loop, seeing as we just
			// synced our account balance with the host if that was necessary
			if build.Release == "testing" {
				msg := fmt.Sprintf("amount: %v, estimatedBalance: %v, target: %v", amount, balance, w.staticRenter.staticAccountBalanceTarget)
				build.Critical("worker account refill failed with a max balance - ", msg, " - are the host max balance settings lower than the threshold balance?", err)
			}
			w.staticRenter.staticLog.Println("worker account refill failed", err)
		}
		w.staticAccount.mu.Lock()
		w.staticAccount.syncAt = time.Time{}
		w.staticAccount.mu.Unlock()
	}
	if err != nil {
		err = errors.AddContext(err, "could not provide payment for the account")
		return
	}

	// receive FundAccountResponse. The response contains a receipt and a
	// signature, which is useful for places where accountability is required,
	// but no accountability is required in this case, so we ignore the
	// response.
	var resp modules.FundAccountResponse
	err = modules.RPCRead(stream, &resp)
	if err != nil {
		err = errors.AddContext(err, "could not read the account response")
	}

	// Wake the worker so that any jobs potentially blocking on getting more
	// money in the account can be activated.
	w.staticWake()

	return nil
}

// managedHostAccountBalance performs the AccountBalanceRPC on the host
func (w *worker) managedHostAccountBalance() (_ types.Currency, err error) {
	// sanity check - only one account balance check should be running at a
	// time.
	if !atomic.CompareAndSwapUint64(&w.atomicAccountBalanceCheckRunning, 0, 1) {
		w.staticRenter.staticLog.Critical("account balance is being checked in two threads concurrently")
	}
	defer atomic.StoreUint64(&w.atomicAccountBalanceCheckRunning, 0)

	// Get a stream.
	stream, err := w.staticNewStream()
	if err != nil {
		return types.ZeroCurrency, err
	}
	defer func() {
		if err := stream.Close(); err != nil {
			w.staticRenter.staticLog.Println("ERROR: failed to close stream", err)
		}
	}()

	// try and pay by EA if possible
	if w.managedMaintenancePayByEA() {
		return w.managedHostAccountBalancePayByEA(stream)
	}
	return w.managedHostAccountBalancePayByContract(stream)
}

// managedHostAccountBalancePayByEA performs the AccountBalanceRPC on the host
// and performs a payment using the renter's ephemeral account.
func (w *worker) managedHostAccountBalancePayByEA(stream siamux.Stream) (_ types.Currency, err error) {
	// prepare a buffer so we can optimize our writes
	buffer := staticPoolCheckAccountBalanceBuffers.Get()
	defer staticPoolCheckAccountBalanceBuffers.Put(buffer)

	// write the specifier
	err = modules.RPCWrite(buffer, modules.RPCAccountBalance)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// send price table uid
	pt := w.staticPriceTable().staticPriceTable
	err = modules.RPCWrite(buffer, pt.UID)
	if err != nil {
		return types.ZeroCurrency, err
	}
	cost := pt.AccountBalanceCost

	// track the withdrawal
	w.staticAccount.managedTrackWithdrawal(cost)

	// provide payment
	bh, _ := w.managedSyncInfo()
	err = w.staticAccount.ProvidePayment(buffer, cost, bh)
	if err != nil {
		// commit the withdrawal
		w.staticAccount.managedCommitWithdrawal(categoryAccountBalance, cost, types.ZeroCurrency, err)
		err = errors.AddContext(err, "unable to provide payment by ephemeral account")
		return
	}

	// commit the withdrawal
	w.staticAccount.managedCommitWithdrawal(categoryAccountBalance, cost, types.ZeroCurrency, nil)

	// prepare the request.
	abr := modules.AccountBalanceRequest{Account: w.staticAccount.staticID}
	err = modules.RPCWrite(buffer, abr)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// write contents of the buffer to the stream
	_, err = buffer.WriteTo(stream)
	if err != nil {
		err = errors.AddContext(err, "Failed to write buffer to stream")
		return
	}

	// read the response
	var resp modules.AccountBalanceResponse
	err = modules.RPCRead(stream, &resp)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// disrupt if necessary
	if w.staticRenter.staticDeps.Disrupt("ZeroBalance") {
		return types.ZeroCurrency, nil
	}
	return resp.Balance, nil
}

// managedHostAccountBalancePayByContract performs the AccountBalanceRPC on the
// host and performs a payment using a contract.
func (w *worker) managedHostAccountBalancePayByContract(stream siamux.Stream) (_ types.Currency, err error) {
	// write the specifier
	err = modules.RPCWrite(stream, modules.RPCAccountBalance)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// send price table uid
	pt := w.staticPriceTable().staticPriceTable
	err = modules.RPCWrite(stream, pt.UID)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// build payment details
	details := contractor.PaymentDetails{
		Host:          w.staticHostPubKey,
		Amount:        pt.AccountBalanceCost,
		RefundAccount: w.staticAccount.staticID,
		SpendingDetails: skymodules.SpendingDetails{
			MaintenanceSpending: skymodules.MaintenanceSpending{
				AccountBalanceCost: pt.AccountBalanceCost,
			},
		},
	}

	// provide payment
	err = w.staticRenter.staticHostContractor.ProvidePayment(stream, &pt, details)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// prepare the request.
	abr := modules.AccountBalanceRequest{Account: w.staticAccount.staticID}
	err = modules.RPCWrite(stream, abr)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// read the response
	var resp modules.AccountBalanceResponse
	err = modules.RPCRead(stream, &resp)
	if err != nil {
		return types.ZeroCurrency, err
	}

	// disrupt if necessary
	if w.staticRenter.staticDeps.Disrupt("ZeroBalance") {
		return types.ZeroCurrency, nil
	}
	return resp.Balance, nil
}

// checkFundAccountGouging verifies the cost of funding an ephemeral account on
// the host is reasonable, if deemed unreasonable we will block the refill and
// the worker will eventually be put into cooldown.
func checkFundAccountGouging(pt modules.RPCPriceTable, allowance skymodules.Allowance, targetBalance types.Currency) error {
	// If there is no allowance, price gouging checks have to be disabled,
	// because there is no baseline for understanding what might count as price
	// gouging.
	if allowance.Funds.IsZero() {
		return nil
	}

	// In order to decide whether or not the fund account cost is too expensive,
	// we first calculate how many times we can refill the account, taking into
	// account the refill amount and the cost to effectively fund the account.
	//
	// Note: we divide the target balance by two because more often than not the
	// refill happens the moment we drop below half of the target, this means
	// that we actually refill half the target amount most of the time.
	costOfRefill := targetBalance.Div64(2).Add(pt.FundAccountCost)
	numRefills, err := allowance.Funds.Div(costOfRefill).Uint64()
	if err != nil {
		return errors.AddContext(err, "unable to check fund account gouging, could not calculate the amount of refills")
	}

	// The cost of funding is considered too expensive if the total cost is
	// above a certain % of the allowance.
	totalFundAccountCost := pt.FundAccountCost.Mul64(numRefills)
	if totalFundAccountCost.Cmp(allowance.Funds.MulFloat(fundAccountGougingPercentageThreshold)) > 0 {
		return fmt.Errorf("fund account cost %v is considered too high, the total cost of refilling the account to spend the total allowance exceeds %v%% of the allowance - price gouging protection enabled", pt.FundAccountCost, fundAccountGougingPercentageThreshold)
	}

	return nil
}

// availableBalance takes a balance, its negative counter part, pending deposits
// and withdrawals and returns a currency that reflects the resulting available
// balance
func availableBalance(balance, balanceNeg, pendingDeposits, pendingWithdrawals types.Currency) types.Currency {
	total := balance.Add(pendingDeposits)
	if total.Cmp(balanceNeg) <= 0 {
		return types.ZeroCurrency
	}
	total = total.Sub(balanceNeg)
	if pendingWithdrawals.Cmp(total) < 0 {
		return total.Sub(pendingWithdrawals)
	}
	return types.ZeroCurrency
}

// minExpectedBalance returns the min amount of money that this account is
// expected to contain after the renter has shut down.
func minExpectedBalance(balance, balanceNegative, pendingWithdrawals types.Currency) types.Currency {
	// subtract all pending withdrawals
	if balance.Cmp(pendingWithdrawals) <= 0 {
		return types.ZeroCurrency
	}
	balance = balance.Sub(pendingWithdrawals)

	// subtract the negative balance
	if balanceNegative.Cmp(balance) > 0 {
		return types.ZeroCurrency
	}
	balance = balance.Sub(balanceNegative)
	return balance
}

// isInsufficientBalanceError is a helper function that verifies whether the
// given error indicates the ephemeral account on the host is out of balance.
//
// NOTE: the host only returns this error after blocking the withdrawal for a
// certain amount of time, awaiting a potential deposit
func isInsufficientBalanceError(err error) bool {
	return err != nil && strings.Contains(err.Error(), host.ErrBalanceInsufficient.Error())
}