github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/fs/rc/jobs/job.go (about)

     1  // Manage background jobs that the rc is running
     2  
     3  package jobs
     4  
     5  import (
     6  	"context"
     7  	"fmt"
     8  	"runtime/debug"
     9  	"sync"
    10  	"sync/atomic"
    11  	"time"
    12  
    13  	"github.com/pkg/errors"
    14  	"github.com/rclone/rclone/fs"
    15  	"github.com/rclone/rclone/fs/accounting"
    16  	"github.com/rclone/rclone/fs/rc"
    17  )
    18  
    19  // Job describes an asynchronous task started via the rc package
    20  type Job struct {
    21  	mu        sync.Mutex
    22  	ID        int64     `json:"id"`
    23  	Group     string    `json:"group"`
    24  	StartTime time.Time `json:"startTime"`
    25  	EndTime   time.Time `json:"endTime"`
    26  	Error     string    `json:"error"`
    27  	Finished  bool      `json:"finished"`
    28  	Success   bool      `json:"success"`
    29  	Duration  float64   `json:"duration"`
    30  	Output    rc.Params `json:"output"`
    31  	Stop      func()    `json:"-"`
    32  
    33  	// realErr is the Error before printing it as a string, it's used to return
    34  	// the real error to the upper application layers while still printing the
    35  	// string error message.
    36  	realErr error
    37  }
    38  
    39  // Jobs describes a collection of running tasks
    40  type Jobs struct {
    41  	mu            sync.RWMutex
    42  	jobs          map[int64]*Job
    43  	opt           *rc.Options
    44  	expireRunning bool
    45  }
    46  
    47  var (
    48  	running = newJobs()
    49  	jobID   = int64(0)
    50  )
    51  
    52  // newJobs makes a new Jobs structure
    53  func newJobs() *Jobs {
    54  	return &Jobs{
    55  		jobs: map[int64]*Job{},
    56  		opt:  &rc.DefaultOpt,
    57  	}
    58  }
    59  
    60  // SetOpt sets the options when they are known
    61  func SetOpt(opt *rc.Options) {
    62  	running.opt = opt
    63  }
    64  
    65  // SetInitialJobID allows for setting jobID before starting any jobs.
    66  func SetInitialJobID(id int64) {
    67  	if !atomic.CompareAndSwapInt64(&jobID, 0, id) {
    68  		panic("Setting jobID is only possible before starting any jobs")
    69  	}
    70  }
    71  
    72  // kickExpire makes sure Expire is running
    73  func (jobs *Jobs) kickExpire() {
    74  	jobs.mu.Lock()
    75  	defer jobs.mu.Unlock()
    76  	if !jobs.expireRunning {
    77  		time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire)
    78  		jobs.expireRunning = true
    79  	}
    80  }
    81  
    82  // Expire expires any jobs that haven't been collected
    83  func (jobs *Jobs) Expire() {
    84  	jobs.mu.Lock()
    85  	defer jobs.mu.Unlock()
    86  	now := time.Now()
    87  	for ID, job := range jobs.jobs {
    88  		job.mu.Lock()
    89  		if job.Finished && now.Sub(job.EndTime) > jobs.opt.JobExpireDuration {
    90  			delete(jobs.jobs, ID)
    91  		}
    92  		job.mu.Unlock()
    93  	}
    94  	if len(jobs.jobs) != 0 {
    95  		time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire)
    96  		jobs.expireRunning = true
    97  	} else {
    98  		jobs.expireRunning = false
    99  	}
   100  }
   101  
   102  // IDs returns the IDs of the running jobs
   103  func (jobs *Jobs) IDs() (IDs []int64) {
   104  	jobs.mu.RLock()
   105  	defer jobs.mu.RUnlock()
   106  	IDs = []int64{}
   107  	for ID := range jobs.jobs {
   108  		IDs = append(IDs, ID)
   109  	}
   110  	return IDs
   111  }
   112  
   113  // Get a job with a given ID or nil if it doesn't exist
   114  func (jobs *Jobs) Get(ID int64) *Job {
   115  	jobs.mu.RLock()
   116  	defer jobs.mu.RUnlock()
   117  	return jobs.jobs[ID]
   118  }
   119  
   120  // mark the job as finished
   121  func (job *Job) finish(out rc.Params, err error) {
   122  	job.mu.Lock()
   123  	job.EndTime = time.Now()
   124  	if out == nil {
   125  		out = make(rc.Params)
   126  	}
   127  	job.Output = out
   128  	job.Duration = job.EndTime.Sub(job.StartTime).Seconds()
   129  	if err != nil {
   130  		job.realErr = err
   131  		job.Error = err.Error()
   132  		job.Success = false
   133  	} else {
   134  		job.realErr = nil
   135  		job.Error = ""
   136  		job.Success = true
   137  	}
   138  	job.Finished = true
   139  	job.mu.Unlock()
   140  	running.kickExpire() // make sure this job gets expired
   141  }
   142  
   143  // run the job until completion writing the return status
   144  func (job *Job) run(ctx context.Context, fn rc.Func, in rc.Params) {
   145  	defer func() {
   146  		if r := recover(); r != nil {
   147  			job.finish(nil, errors.Errorf("panic received: %v \n%s", r, string(debug.Stack())))
   148  		}
   149  	}()
   150  	job.finish(fn(ctx, in))
   151  }
   152  
   153  func getGroup(in rc.Params) string {
   154  	// Check to see if the group is set
   155  	group, err := in.GetString("_group")
   156  	if rc.NotErrParamNotFound(err) {
   157  		fs.Errorf(nil, "Can't get _group param %+v", err)
   158  	}
   159  	delete(in, "_group")
   160  	return group
   161  }
   162  
   163  // NewAsyncJob start a new asynchronous Job off
   164  func (jobs *Jobs) NewAsyncJob(fn rc.Func, in rc.Params) *Job {
   165  	id := atomic.AddInt64(&jobID, 1)
   166  
   167  	group := getGroup(in)
   168  	if group == "" {
   169  		group = fmt.Sprintf("job/%d", id)
   170  	}
   171  	ctx := accounting.WithStatsGroup(context.Background(), group)
   172  	ctx, cancel := context.WithCancel(ctx)
   173  	stop := func() {
   174  		cancel()
   175  		// Wait for cancel to propagate before returning.
   176  		<-ctx.Done()
   177  	}
   178  	job := &Job{
   179  		ID:        id,
   180  		Group:     group,
   181  		StartTime: time.Now(),
   182  		Stop:      stop,
   183  	}
   184  	jobs.mu.Lock()
   185  	jobs.jobs[job.ID] = job
   186  	jobs.mu.Unlock()
   187  	go job.run(ctx, fn, in)
   188  	return job
   189  }
   190  
   191  // NewSyncJob start a new synchronous Job off
   192  func (jobs *Jobs) NewSyncJob(ctx context.Context, in rc.Params) (*Job, context.Context) {
   193  	id := atomic.AddInt64(&jobID, 1)
   194  	group := getGroup(in)
   195  	if group == "" {
   196  		group = fmt.Sprintf("job/%d", id)
   197  	}
   198  	ctxG := accounting.WithStatsGroup(ctx, fmt.Sprintf("job/%d", id))
   199  	ctx, cancel := context.WithCancel(ctxG)
   200  	stop := func() {
   201  		cancel()
   202  		// Wait for cancel to propagate before returning.
   203  		<-ctx.Done()
   204  	}
   205  	job := &Job{
   206  		ID:        id,
   207  		Group:     group,
   208  		StartTime: time.Now(),
   209  		Stop:      stop,
   210  	}
   211  	jobs.mu.Lock()
   212  	jobs.jobs[job.ID] = job
   213  	jobs.mu.Unlock()
   214  	return job, ctx
   215  }
   216  
   217  // StartAsyncJob starts a new job asynchronously and returns a Param suitable
   218  // for output.
   219  func StartAsyncJob(fn rc.Func, in rc.Params) (rc.Params, error) {
   220  	job := running.NewAsyncJob(fn, in)
   221  	out := make(rc.Params)
   222  	out["jobid"] = job.ID
   223  	return out, nil
   224  }
   225  
   226  // ExecuteJob executes new job synchronously and returns a Param suitable for
   227  // output.
   228  func ExecuteJob(ctx context.Context, fn rc.Func, in rc.Params) (rc.Params, int64, error) {
   229  	job, ctx := running.NewSyncJob(ctx, in)
   230  	job.run(ctx, fn, in)
   231  	return job.Output, job.ID, job.realErr
   232  }
   233  
   234  func init() {
   235  	rc.Add(rc.Call{
   236  		Path:  "job/status",
   237  		Fn:    rcJobStatus,
   238  		Title: "Reads the status of the job ID",
   239  		Help: `Parameters
   240  
   241  - jobid - id of the job (integer)
   242  
   243  Results
   244  
   245  - finished - boolean
   246  - duration - time in seconds that the job ran for
   247  - endTime - time the job finished (eg "2018-10-26T18:50:20.528746884+01:00")
   248  - error - error from the job or empty string for no error
   249  - finished - boolean whether the job has finished or not
   250  - id - as passed in above
   251  - startTime - time the job started (eg "2018-10-26T18:50:20.528336039+01:00")
   252  - success - boolean - true for success false otherwise
   253  - output - output of the job as would have been returned if called synchronously
   254  - progress - output of the progress related to the underlying job
   255  `,
   256  	})
   257  }
   258  
   259  // Returns the status of a job
   260  func rcJobStatus(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   261  	jobID, err := in.GetInt64("jobid")
   262  	if err != nil {
   263  		return nil, err
   264  	}
   265  	job := running.Get(jobID)
   266  	if job == nil {
   267  		return nil, errors.New("job not found")
   268  	}
   269  	job.mu.Lock()
   270  	defer job.mu.Unlock()
   271  	out = make(rc.Params)
   272  	err = rc.Reshape(&out, job)
   273  	if err != nil {
   274  		return nil, errors.Wrap(err, "reshape failed in job status")
   275  	}
   276  	return out, nil
   277  }
   278  
   279  func init() {
   280  	rc.Add(rc.Call{
   281  		Path:  "job/list",
   282  		Fn:    rcJobList,
   283  		Title: "Lists the IDs of the running jobs",
   284  		Help: `Parameters - None
   285  
   286  Results
   287  
   288  - jobids - array of integer job ids
   289  `,
   290  	})
   291  }
   292  
   293  // Returns list of job ids.
   294  func rcJobList(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   295  	out = make(rc.Params)
   296  	out["jobids"] = running.IDs()
   297  	return out, nil
   298  }
   299  
   300  func init() {
   301  	rc.Add(rc.Call{
   302  		Path:  "job/stop",
   303  		Fn:    rcJobStop,
   304  		Title: "Stop the running job",
   305  		Help: `Parameters
   306  
   307  - jobid - id of the job (integer)
   308  `,
   309  	})
   310  }
   311  
   312  // Stops the running job.
   313  func rcJobStop(ctx context.Context, in rc.Params) (out rc.Params, err error) {
   314  	jobID, err := in.GetInt64("jobid")
   315  	if err != nil {
   316  		return nil, err
   317  	}
   318  	job := running.Get(jobID)
   319  	if job == nil {
   320  		return nil, errors.New("job not found")
   321  	}
   322  	job.mu.Lock()
   323  	defer job.mu.Unlock()
   324  	out = make(rc.Params)
   325  	job.Stop()
   326  	return out, nil
   327  }