gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/workerjobgeneric.go

package renter

import (
	"container/list"
	"context"
	"sync"
	"time"

	"gitlab.com/NebulousLabs/errors"
)

var errDiscardingCanceledJob = errors.New("callNext: skipping and discarding already canceled job")

type (
	// jobGeneric implements the basic functionality for a job.
	jobGeneric struct {
		staticCtx context.Context

		staticQueue workerJobQueue

		// staticMetadata is a generic field on the job that can be set and
		// casted by implementations of a job
		staticMetadata interface{}

		// externExecuteTime is set when the job gets executed
		//
		// NOTE: the 'extern' prefix is used here even though the field is not
		// governed by a mutex, it can be accessed by anyone since a job is not
		// used from more than one thread - it is not static because it's not
		// set on initialization but rather by the implementation of callExecute
		externExecuteTime time.Time
	}

	// jobGenericQueue is a generic queue for a job. It has a mutex, references
	// a worker, tracks whether or not it has been killed, and has a cooldown
	// timer. It does not have an array of jobs that are in the queue, because
	// those are type specific.
	jobGenericQueue struct {
		jobs *list.List

		killed bool

		cooldownUntil       time.Time
		consecutiveFailures uint64

		// firstFailureTime is set by a failed job in case that job was executed
		// after the current value, it is cleared by a successful job, it's used
		// to conditionally increment the consecutive failures field, rather
		// than having parallelly executed failed jobs all increment it
		firstFailureTime time.Time

		recentErr     error
		recentErrTime time.Time

		staticWorkerObj *worker // name conflict with staticWorker method
		mu              sync.Mutex
	}

	// workerJob defines a job that the worker is able to perform.
	workerJob interface {
		// callDicard will discard this job, sending an error down the response
		// channel of the job. The provided error should be part of the error
		// that gets sent.
		callDiscard(error)

		// callExecute will run the actual job.
		callExecute() error

		// callExpectedBandwidth will return the amount of bandwidth that a job
		// expects to consume.
		callExpectedBandwidth() (upload uint64, download uint64)

		// staticGetMetadata returns a metadata object.
		staticGetMetadata() interface{}

		// staticCanceled returns true if the job has been canceled, false
		// otherwise.
		staticCanceled() bool
	}

	// workerJobQueue defines an interface to create a worker job queue.
	workerJobQueue interface {
		// callDiscardAll will discard all of the jobs in the queue using the
		// provided error.
		callDiscardAll(error)

		// callReportFailure should be called on the queue every time that a job
		// fails, and include the error associated with the failure and the time
		// at which the job was executed.
		callReportFailure(error, time.Time, time.Time)

		// callReportSuccess should be called on the queue every time that a job
		// succeeds.
		callReportSuccess()

		// callStatus returns the status of the queue
		callStatus() workerJobQueueStatus

		// staticWorker will return the worker of the job queue.
		staticWorker() *worker
	}

	// workerJobQueueStatus is a struct that reflects the status of the queue
	workerJobQueueStatus struct {
		size                uint64
		cooldownUntil       time.Time
		consecutiveFailures uint64
		recentErr           error
		recentErrTime       time.Time
	}
)

// newJobGeneric returns an initialized jobGeneric. The queue that is associated
// with the job should be used as the input to this function. The job will
// cancel itself if the cancelChan is closed.
func newJobGeneric(ctx context.Context, queue workerJobQueue, metadata interface{}) jobGeneric {
	return jobGeneric{
		staticCtx:      ctx,
		staticQueue:    queue,
		staticMetadata: metadata,
	}
}

// newJobGenericQueue will return an initialized generic job queue.
func newJobGenericQueue(w *worker) *jobGenericQueue {
	return &jobGenericQueue{
		jobs:            list.New(),
		staticWorkerObj: w,
	}
}

// staticCanceled returns whether or not the job has been canceled.
func (j *jobGeneric) staticCanceled() bool {
	select {
	case <-j.staticCtx.Done():
		return true
	default:
		return false
	}
}

// staticGetMetadata returns the job's metadata.
func (j *jobGeneric) staticGetMetadata() interface{} {
	return j.staticMetadata
}

// add will add a job to the queue.
func (jq *jobGenericQueue) add(j workerJob) bool {
	if jq.killed || jq.onCooldown() {
		return false
	}
	jq.jobs.PushBack(j)
	jq.staticWorkerObj.staticWake()
	return true
}

// callAdd will add a job to the queue.
func (jq *jobGenericQueue) callAdd(j workerJob) bool {
	jq.mu.Lock()
	defer jq.mu.Unlock()
	return jq.add(j)
}

// callCooldownStatus returns all necessary information to present the queues' cooldown status.
func (jq *jobGenericQueue) callCooldownStatus() (bool, bool, int, time.Duration, string) {
	jq.mu.Lock()
	defer jq.mu.Unlock()

	var coolDownErrStr string
	if jq.onCooldown() && jq.recentErr != nil {
		coolDownErrStr = jq.recentErr.Error()
	}

	var coolDownUntil time.Duration
	if jq.onCooldown() {
		coolDownUntil = time.Until(jq.cooldownUntil)
	}

	return jq.onCooldown(), jq.killed, jq.jobs.Len(), coolDownUntil, coolDownErrStr
}

// callDiscardAll will discard all jobs in the queue using the provided error.
func (jq *jobGenericQueue) callDiscardAll(err error) {
	jq.mu.Lock()
	defer jq.mu.Unlock()
	jq.discardAll(err)
}

// callKill will kill the queue, discarding all jobs and ensuring no more jobs
// can be added.
func (jq *jobGenericQueue) callKill() {
	jq.mu.Lock()
	defer jq.mu.Unlock()

	err := errors.New("worker is being killed")
	jq.discardAll(err)
	jq.killed = true
}

// callIsKilled returns whether or not the jobGenericQueue was killed or not
func (jq *jobGenericQueue) callIsKilled() bool {
	jq.mu.Lock()
	defer jq.mu.Unlock()
	return jq.killed
}

// callLen returns the number of jobs in the queue.
func (jq *jobGenericQueue) callLen() int {
	jq.mu.Lock()
	defer jq.mu.Unlock()
	return jq.jobs.Len()
}

// callNext returns the next job in the worker queue. If there is no job in the
// queue, 'nil' will be returned.
func (jq *jobGenericQueue) callNext() workerJob {
	jq.mu.Lock()
	defer jq.mu.Unlock()

	// Loop through the jobs, looking for the first job that hasn't yet been
	// canceled. Remove jobs from the queue along the way.
	for job := jq.jobs.Front(); job != nil; job = job.Next() {
		// Remove the job from the list.
		jq.jobs.Remove(job)

		// Check if the job is already canceled.
		wj := job.Value.(workerJob)
		if wj.staticCanceled() {
			wj.callDiscard(errDiscardingCanceledJob)
			continue
		}
		return wj
	}

	// Job queue is empty, return nil.
	return nil
}

// callOnCooldown returns whether the queue is on cooldown.
func (jq *jobGenericQueue) callOnCooldown() bool {
	jq.mu.Lock()
	defer jq.mu.Unlock()
	return jq.onCooldown()
}

// callReportFailure reports that a job has failed within the queue. This will
// cause all remaining jobs in the queue to be discarded, and will put the queue
// on cooldown.
func (jq *jobGenericQueue) callReportFailure(err error, executedAt, failedAt time.Time) {
	jq.mu.Lock()
	defer jq.mu.Unlock()

	// only update the cooldown if we're currently not on cooldown
	if !jq.onCooldown() {
		jq.cooldownUntil = cooldownUntil(jq.consecutiveFailures)
	}

	jq.recentErr = errors.AddContext(err, "discarding all jobs in this queue and going on cooldown")
	jq.recentErrTime = time.Now()

	// discard all jobs in the queue
	jq.discardAll(jq.recentErr)

	// only if the job was executed after the time of the first failure we want
	// to count it as a consective failure, when that is the case we also want
	// to update the time of the first failure to the current time
	//
	// NOTE: this is to ensure multiple concurrent jobs that fail at about the
	// same time don't all count towards the consecutive failures, causing the
	// cooldown to go from zero to max immediately
	if executedAt.After(jq.firstFailureTime) {
		jq.consecutiveFailures++
		jq.firstFailureTime = failedAt
	}
}

// callReportSuccess lets the job queue know that there was a successsful job.
// Note that this will reset the consecutive failure count, but will not reset
// the recentErr value - the recentErr value is left as an error so that when
// debugging later, developers and users can see what errors had been caused by
// past issues.
func (jq *jobGenericQueue) callReportSuccess() {
	jq.mu.Lock()
	jq.consecutiveFailures = 0
	jq.firstFailureTime = time.Time{}
	jq.mu.Unlock()
}

// callStatus returns the queue status
func (jq *jobGenericQueue) callStatus() workerJobQueueStatus {
	jq.mu.Lock()
	defer jq.mu.Unlock()
	return workerJobQueueStatus{
		size:                uint64(jq.jobs.Len()),
		cooldownUntil:       jq.cooldownUntil,
		consecutiveFailures: jq.consecutiveFailures,
		recentErr:           jq.recentErr,
		recentErrTime:       jq.recentErrTime,
	}
}

// discardAll will drop all jobs from the queue.
func (jq *jobGenericQueue) discardAll(err error) {
	for job := jq.jobs.Front(); job != nil; job = job.Next() {
		wj := job.Value.(workerJob)
		wj.callDiscard(err)
	}
	jq.jobs = list.New()
}

// staticWorker will return the worker that is associated with this job queue.
func (jq *jobGenericQueue) staticWorker() *worker {
	return jq.staticWorkerObj
}

// onCooldown returns whether the queue is on cooldown.
func (jq *jobGenericQueue) onCooldown() bool {
	return time.Now().Before(jq.cooldownUntil)
}