gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/contractor/churnlimiter.go

package contractor

import (
	"fmt"
	"sort"
	"sync"

	"gitlab.com/SkynetLabs/skyd/build"
	"gitlab.com/SkynetLabs/skyd/skymodules"
	"go.sia.tech/siad/types"

	"gitlab.com/NebulousLabs/errors"
)

// contractScoreAndUtil combines a contract with its host's score and an updated
// utility.
type contractScoreAndUtil struct {
	contract skymodules.RenterContract
	score    types.Currency
	util     skymodules.ContractUtility
}

// churnLimiter keeps track of the aggregate number of bytes stored in contracts
// marked !GFR (AKA churned contracts) in the current period.
type churnLimiter struct {
	// remainingChurnBudget is the number of bytes that the churnLimiter will
	// allow to be churned in contracts at the present moment. Note that this
	// value may be negative.
	remainingChurnBudget int

	// aggregateCurrentPeriodChurn is the aggregate size of files stored in contracts
	// churned in the current period.
	aggregateCurrentPeriodChurn uint64

	mu               sync.Mutex
	staticContractor *Contractor
}

// churnLimiterPersist is the persisted state of a churnLimiter.
type churnLimiterPersist struct {
	AggregateCurrentPeriodChurn uint64 `json:"aggregatecurrentperiodchurn"`
	RemainingChurnBudget        int    `json:"remainingchurnbudget"`
}

// managedMaxPeriodChurn returns the MaxPeriodChurn of the churnLimiter.
func (cl *churnLimiter) managedMaxPeriodChurn() uint64 {
	return cl.staticContractor.Allowance().MaxPeriodChurn
}

// callPersistData returns the churnLimiterPersist corresponding to this
// churnLimiter's state
func (cl *churnLimiter) callPersistData() churnLimiterPersist {
	cl.mu.Lock()
	defer cl.mu.Unlock()
	return churnLimiterPersist{cl.aggregateCurrentPeriodChurn, cl.remainingChurnBudget}
}

// newChurnLimiterFromPersist creates a new churnLimiter using persisted state.
func newChurnLimiterFromPersist(contractor *Contractor, persistData churnLimiterPersist) *churnLimiter {
	return &churnLimiter{
		staticContractor:            contractor,
		aggregateCurrentPeriodChurn: persistData.AggregateCurrentPeriodChurn,
		remainingChurnBudget:        persistData.RemainingChurnBudget,
	}
}

// newChurnLimiter returns a new churnLimiter.
func newChurnLimiter(contractor *Contractor) *churnLimiter {
	return &churnLimiter{staticContractor: contractor}
}

// ChurnStatus returns the current period's aggregate churn and the max churn
// per period.
func (c *Contractor) ChurnStatus() skymodules.ContractorChurnStatus {
	aggregateChurn, maxChurn := c.staticChurnLimiter.managedAggregateAndMaxChurn()
	return skymodules.ContractorChurnStatus{
		AggregateCurrentPeriodChurn: aggregateChurn,
		MaxPeriodChurn:              maxChurn,
	}
}

// callResetAggregateChurn resets the aggregate churn for this period. This
// method must be called at the beginning of every new period.
func (cl *churnLimiter) callResetAggregateChurn() {
	cl.mu.Lock()
	cl.staticContractor.staticLog.Println("Aggregate Churn for last period: ", cl.aggregateCurrentPeriodChurn)
	cl.aggregateCurrentPeriodChurn = 0
	cl.mu.Unlock()
}

// callNotifyChurnedContract adds the size of this contract's files to the aggregate
// churn in this period. Must be called when contracts are marked !GFR.
func (cl *churnLimiter) callNotifyChurnedContract(contract skymodules.RenterContract) {
	size := contract.Transaction.FileContractRevisions[0].NewFileSize
	if size == 0 {
		return
	}
	maxPeriodChurn := cl.managedMaxPeriodChurn()

	cl.mu.Lock()
	defer cl.mu.Unlock()

	cl.aggregateCurrentPeriodChurn += size
	cl.remainingChurnBudget -= int(size)
	cl.staticContractor.staticLog.Debugf("Increasing aggregate churn by %d to %d (MaxPeriodChurn: %d)", size, cl.aggregateCurrentPeriodChurn, maxPeriodChurn)
	cl.staticContractor.staticLog.Debugf("Remaining churn budget: %d", cl.remainingChurnBudget)
}

// callBumpChurnBudget increases the churn budget by a fraction of the max churn
// budget per period. Used when new blocks are processed.
func (cl *churnLimiter) callBumpChurnBudget(numBlocksAdded int, period types.BlockHeight) {
	// Don't add to churn budget when there is no period, since no allowance is
	// set yet.
	if period == types.BlockHeight(0) {
		return
	}
	maxPeriodChurn := cl.managedMaxPeriodChurn()
	maxChurnBudget := cl.managedMaxChurnBudget()
	refillBudget := cl.staticContractor.staticDeps.Disrupt("RefillChurnBudget")
	cl.mu.Lock()
	defer cl.mu.Unlock()

	// Increase churn budget as a multiple of the period budget per block. This
	// let's the remainingChurnBudget increase more quickly.
	budgetIncrease := numBlocksAdded * int(maxPeriodChurn/uint64(period))
	cl.remainingChurnBudget += budgetIncrease
	if refillBudget || cl.remainingChurnBudget > maxChurnBudget {
		cl.remainingChurnBudget = maxChurnBudget
	}
	cl.staticContractor.staticLog.Debugf("Updated churn budget: %d", cl.remainingChurnBudget)
}

// managedMaxChurnBudget returns the max allowed value for remainingChurnBudget.
func (cl *churnLimiter) managedMaxChurnBudget() int {
	// Do not let churn budget to build up to maxPeriodChurn to avoid using entire
	// period budget at once (except in special circumstances).
	return int(cl.managedMaxPeriodChurn() / 2)
}

// managedProcessSuggestedUpdates processes suggested utility updates. It
// prevents contracts from being marked as !GFR if the churn limit has been
// reached. The inputs are assumed to be contracts that have passed all critical
// utility checks. The result is the list of updates which need to be applied.
func (cl *churnLimiter) managedProcessSuggestedUpdates(queue []contractScoreAndUtil) map[types.FileContractID]skymodules.ContractUtility {
	sort.Slice(queue, func(i, j int) bool {
		return queue[i].score.Cmp(queue[j].score) < 0
	})

	var queuedContract contractScoreAndUtil
	contractUpdates := make(map[types.FileContractID]skymodules.ContractUtility)
	for len(queue) > 0 {
		queuedContract, queue = queue[0], queue[1:]

		// Churn a contract if it went from GFR in the previous util
		// (queuedContract.contract.Utility) to !GFR in the suggested util
		// (queuedContract.util) and the churnLimit has not been reached.
		turnedNotGFR := queuedContract.contract.Utility.GoodForRenew && !queuedContract.util.GoodForRenew
		churningThisContract := turnedNotGFR && cl.managedCanChurnContract(queuedContract.contract)
		if turnedNotGFR && !churningThisContract {
			cl.staticContractor.staticLog.Debugln("Avoiding churn on contract: ", queuedContract.contract.ID)
			currentBudget, periodBudget := cl.managedChurnBudget()
			cl.staticContractor.staticLog.Debugf("Remaining Churn Budget: %d. Remaining Period Budget: %d", currentBudget, periodBudget)
			queuedContract.util.GoodForRenew = true
		}

		if churningThisContract {
			cl.staticContractor.staticLog.Println("Churning contract for bad score: ", queuedContract.contract.ID, queuedContract.score)

			// If we are churning this contract, mark it as churned.
			// That way the churn is considered in the next
			// iteration.
			cl.callNotifyChurnedContract(queuedContract.contract)
		}

		// Remember contract for updating.
		contractUpdates[queuedContract.contract.ID] = queuedContract.util
	}
	return contractUpdates
}

// managedChurnBudget returns the current remaining churn budget, and the remaining
// budget for the period.
func (cl *churnLimiter) managedChurnBudget() (int, int) {
	maxPeriodChurn := cl.managedMaxPeriodChurn()
	cl.mu.Lock()
	defer cl.mu.Unlock()
	return cl.remainingChurnBudget, int(maxPeriodChurn) - int(cl.aggregateCurrentPeriodChurn)
}

// managedAggregateAndMaxChurn returns the aggregate churn for the current period,
// and the maximum churn allowed per period.
func (cl *churnLimiter) managedAggregateAndMaxChurn() (uint64, uint64) {
	maxPeriodChurn := cl.managedMaxPeriodChurn()
	cl.mu.Lock()
	defer cl.mu.Unlock()
	return cl.aggregateCurrentPeriodChurn, maxPeriodChurn
}

// managedCanChurnContract returns true if and only if the churnLimiter can
// churn the contract right now, given its current budget.
func (cl *churnLimiter) managedCanChurnContract(contract skymodules.RenterContract) bool {
	size := contract.Transaction.FileContractRevisions[0].NewFileSize
	maxPeriodChurn := cl.managedMaxPeriodChurn()
	maxChurnBudget := cl.managedMaxChurnBudget()
	cl.mu.Lock()
	defer cl.mu.Unlock()

	// Allow any size contract to be churned if the current budget is the max
	// budget. This allows large contracts to be churned if there is enough budget
	// remaining for the period, even if the contract is larger than the
	// maxChurnBudget.
	fitsInCurrentBudget := (cl.remainingChurnBudget-int(size) >= 0) || (cl.remainingChurnBudget == maxChurnBudget)
	fitsInPeriodBudget := (int(maxPeriodChurn) - int(cl.aggregateCurrentPeriodChurn) - int(size)) >= 0

	// If there has been no churn in this period, allow any size contract to be
	// churned.
	fitsInPeriodBudget = fitsInPeriodBudget || (cl.aggregateCurrentPeriodChurn == 0)

	return fitsInPeriodBudget && fitsInCurrentBudget
}

// managedMarkContractUtility checks an active contract in the contractor and
// figures out whether the contract is useful for uploading, and whether the
// contract should be renewed.
func (c *Contractor) managedMarkContractUtility(contract skymodules.RenterContract, minScoreGFR, minScoreGFU types.Currency, endHeight types.BlockHeight) (contractScoreAndUtil, utilityUpdateStatus, error) {
	// Acquire contract.
	sc, ok := c.staticContracts.Acquire(contract.ID)
	if !ok {
		return contractScoreAndUtil{}, noUpdate, errors.New("managedMarkContractUtility: Unable to acquire contract")
	}
	defer c.staticContracts.Return(sc)

	// Get latest metadata.
	u := sc.Metadata().Utility

	// If the utility is locked, do nothing.
	if u.Locked {
		return contractScoreAndUtil{}, noUpdate, nil
	}

	// Get host from hostdb and check that it's not filtered.
	host, u, needsUpdate := c.managedHostInHostDBCheck(contract)
	if needsUpdate {
		return contractScoreAndUtil{contract: contract, score: types.ZeroCurrency, util: u}, necessaryUtilityUpdate, nil
	}

	sb, err := c.staticHDB.ScoreBreakdown(host.HostDBEntry)
	if err != nil {
		c.staticLog.Println("Unable to get ScoreBreakdown for", host.PublicKey.String(), "got err:", err)
		return contractScoreAndUtil{}, noUpdate, nil // it may just be this host that has an issue.
	}

	// Do critical contract checks and update the utility if any checks fail.
	u, utilityUpdateStatus := c.managedUtilityChecks(sc.Metadata(), host, sb, minScoreGFU, minScoreGFR, endHeight)

	// Sanity check
	if utilityUpdateStatus == noUpdate && contract.Utility != u {
		build.Critical("managedMarkContractUtility: status was 'noUpdate' but utility changed")
	}

	return contractScoreAndUtil{
		contract: contract,
		score:    sb.Score,
		util:     u,
	}, utilityUpdateStatus, nil
}

// managedMarkContractsUtility checks every active contract in the contractor and
// figures out whether the contract is useful for uploading, and whether the
// contract should be renewed.
func (c *Contractor) managedMarkContractsUtility(wantedHosts uint64, endHeight types.BlockHeight) error {
	minScoreGFR, minScoreGFU, err := c.managedFindMinAllowedHostScores(wantedHosts)
	if err != nil {
		return err
	}

	// Queue for possible contracts to churn. Passed to churnLimiter for final
	// judgment.
	suggestedUpdateQueue := make([]contractScoreAndUtil, 0)

	// These updates are always applied.
	necessaryUpdateQueue := make([]contractScoreAndUtil, 0)

	// Sort the updates into suggested and necessary ones.
	for _, contract := range c.staticContracts.ViewAll() {
		update, uus, err := c.managedMarkContractUtility(contract, minScoreGFR, minScoreGFU, endHeight)
		if err != nil {
			return err
		}

		switch uus {
		case noUpdate:
		case suggestedUtilityUpdate:
			suggestedUpdateQueue = append(suggestedUpdateQueue, update)
		case necessaryUtilityUpdate:
			necessaryUpdateQueue = append(necessaryUpdateQueue, update)
		default:
			err := fmt.Errorf("undefined checkHostScore utilityUpdateStatus %v %v", uus, contract.ID)
			c.staticLog.Critical(err)
		}
	}

	// Apply the necessary updates. These always have to be applied so we
	// apply them separately.
	for _, nu := range necessaryUpdateQueue {
		if nu.contract.Utility.GoodForRenew && !nu.util.GoodForRenew {
			c.staticChurnLimiter.callNotifyChurnedContract(nu.contract)
		}
	}

	// Filter the suggested updates through the churn limiter.
	contractUpdates := c.staticChurnLimiter.managedProcessSuggestedUpdates(suggestedUpdateQueue)

	// Merge the necessary updates in.
	for _, nu := range necessaryUpdateQueue {
		_, exists := contractUpdates[nu.contract.ID]
		if exists {
			build.Critical("managedMarkContractsUtility: there shouldn't be both a suggested as well as a necessary update for a single contract")
		}
		contractUpdates[nu.contract.ID] = nu.util
	}

	// Get all contracts and apply the updates to them in-memory only.
	contractsAfterUpdates := c.Contracts()
	for i, contract := range contractsAfterUpdates {
		utility, exists := contractUpdates[contract.ID]
		if exists {
			contractsAfterUpdates[i].Utility = utility
		}
	}

	// Limit the number of active contracts.
	c.managedLimitGFUHosts(contractsAfterUpdates, contractUpdates, wantedHosts)

	// Apply the suggested updates.
	for id, utility := range contractUpdates {
		// We already called callNotifyChurnedContract for the necessary
		// updates and managedProcessSuggestedUpdates called it for
		// suggested updates IFF they were churned. So we can ignore
		// that here and pass in 'true'.
		if err := c.managedAcquireAndUpdateContractUtility(id, utility, true); err != nil {
			return errors.AddContext(err, "managedMarkContractsUtility: failed to apply update to contract")
		}
	}
	return nil
}