github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/fs/rc/jobs/job.go (about) 1 // Manage background jobs that the rc is running 2 3 package jobs 4 5 import ( 6 "context" 7 "fmt" 8 "runtime/debug" 9 "sync" 10 "sync/atomic" 11 "time" 12 13 "github.com/pkg/errors" 14 "github.com/rclone/rclone/fs" 15 "github.com/rclone/rclone/fs/accounting" 16 "github.com/rclone/rclone/fs/rc" 17 ) 18 19 // Job describes an asynchronous task started via the rc package 20 type Job struct { 21 mu sync.Mutex 22 ID int64 `json:"id"` 23 Group string `json:"group"` 24 StartTime time.Time `json:"startTime"` 25 EndTime time.Time `json:"endTime"` 26 Error string `json:"error"` 27 Finished bool `json:"finished"` 28 Success bool `json:"success"` 29 Duration float64 `json:"duration"` 30 Output rc.Params `json:"output"` 31 Stop func() `json:"-"` 32 33 // realErr is the Error before printing it as a string, it's used to return 34 // the real error to the upper application layers while still printing the 35 // string error message. 36 realErr error 37 } 38 39 // Jobs describes a collection of running tasks 40 type Jobs struct { 41 mu sync.RWMutex 42 jobs map[int64]*Job 43 opt *rc.Options 44 expireRunning bool 45 } 46 47 var ( 48 running = newJobs() 49 jobID = int64(0) 50 ) 51 52 // newJobs makes a new Jobs structure 53 func newJobs() *Jobs { 54 return &Jobs{ 55 jobs: map[int64]*Job{}, 56 opt: &rc.DefaultOpt, 57 } 58 } 59 60 // SetOpt sets the options when they are known 61 func SetOpt(opt *rc.Options) { 62 running.opt = opt 63 } 64 65 // SetInitialJobID allows for setting jobID before starting any jobs. 66 func SetInitialJobID(id int64) { 67 if !atomic.CompareAndSwapInt64(&jobID, 0, id) { 68 panic("Setting jobID is only possible before starting any jobs") 69 } 70 } 71 72 // kickExpire makes sure Expire is running 73 func (jobs *Jobs) kickExpire() { 74 jobs.mu.Lock() 75 defer jobs.mu.Unlock() 76 if !jobs.expireRunning { 77 time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire) 78 jobs.expireRunning = true 79 } 80 } 81 82 // Expire expires any jobs that haven't been collected 83 func (jobs *Jobs) Expire() { 84 jobs.mu.Lock() 85 defer jobs.mu.Unlock() 86 now := time.Now() 87 for ID, job := range jobs.jobs { 88 job.mu.Lock() 89 if job.Finished && now.Sub(job.EndTime) > jobs.opt.JobExpireDuration { 90 delete(jobs.jobs, ID) 91 } 92 job.mu.Unlock() 93 } 94 if len(jobs.jobs) != 0 { 95 time.AfterFunc(jobs.opt.JobExpireInterval, jobs.Expire) 96 jobs.expireRunning = true 97 } else { 98 jobs.expireRunning = false 99 } 100 } 101 102 // IDs returns the IDs of the running jobs 103 func (jobs *Jobs) IDs() (IDs []int64) { 104 jobs.mu.RLock() 105 defer jobs.mu.RUnlock() 106 IDs = []int64{} 107 for ID := range jobs.jobs { 108 IDs = append(IDs, ID) 109 } 110 return IDs 111 } 112 113 // Get a job with a given ID or nil if it doesn't exist 114 func (jobs *Jobs) Get(ID int64) *Job { 115 jobs.mu.RLock() 116 defer jobs.mu.RUnlock() 117 return jobs.jobs[ID] 118 } 119 120 // mark the job as finished 121 func (job *Job) finish(out rc.Params, err error) { 122 job.mu.Lock() 123 job.EndTime = time.Now() 124 if out == nil { 125 out = make(rc.Params) 126 } 127 job.Output = out 128 job.Duration = job.EndTime.Sub(job.StartTime).Seconds() 129 if err != nil { 130 job.realErr = err 131 job.Error = err.Error() 132 job.Success = false 133 } else { 134 job.realErr = nil 135 job.Error = "" 136 job.Success = true 137 } 138 job.Finished = true 139 job.mu.Unlock() 140 running.kickExpire() // make sure this job gets expired 141 } 142 143 // run the job until completion writing the return status 144 func (job *Job) run(ctx context.Context, fn rc.Func, in rc.Params) { 145 defer func() { 146 if r := recover(); r != nil { 147 job.finish(nil, errors.Errorf("panic received: %v \n%s", r, string(debug.Stack()))) 148 } 149 }() 150 job.finish(fn(ctx, in)) 151 } 152 153 func getGroup(in rc.Params) string { 154 // Check to see if the group is set 155 group, err := in.GetString("_group") 156 if rc.NotErrParamNotFound(err) { 157 fs.Errorf(nil, "Can't get _group param %+v", err) 158 } 159 delete(in, "_group") 160 return group 161 } 162 163 // NewAsyncJob start a new asynchronous Job off 164 func (jobs *Jobs) NewAsyncJob(fn rc.Func, in rc.Params) *Job { 165 id := atomic.AddInt64(&jobID, 1) 166 167 group := getGroup(in) 168 if group == "" { 169 group = fmt.Sprintf("job/%d", id) 170 } 171 ctx := accounting.WithStatsGroup(context.Background(), group) 172 ctx, cancel := context.WithCancel(ctx) 173 stop := func() { 174 cancel() 175 // Wait for cancel to propagate before returning. 176 <-ctx.Done() 177 } 178 job := &Job{ 179 ID: id, 180 Group: group, 181 StartTime: time.Now(), 182 Stop: stop, 183 } 184 jobs.mu.Lock() 185 jobs.jobs[job.ID] = job 186 jobs.mu.Unlock() 187 go job.run(ctx, fn, in) 188 return job 189 } 190 191 // NewSyncJob start a new synchronous Job off 192 func (jobs *Jobs) NewSyncJob(ctx context.Context, in rc.Params) (*Job, context.Context) { 193 id := atomic.AddInt64(&jobID, 1) 194 group := getGroup(in) 195 if group == "" { 196 group = fmt.Sprintf("job/%d", id) 197 } 198 ctxG := accounting.WithStatsGroup(ctx, fmt.Sprintf("job/%d", id)) 199 ctx, cancel := context.WithCancel(ctxG) 200 stop := func() { 201 cancel() 202 // Wait for cancel to propagate before returning. 203 <-ctx.Done() 204 } 205 job := &Job{ 206 ID: id, 207 Group: group, 208 StartTime: time.Now(), 209 Stop: stop, 210 } 211 jobs.mu.Lock() 212 jobs.jobs[job.ID] = job 213 jobs.mu.Unlock() 214 return job, ctx 215 } 216 217 // StartAsyncJob starts a new job asynchronously and returns a Param suitable 218 // for output. 219 func StartAsyncJob(fn rc.Func, in rc.Params) (rc.Params, error) { 220 job := running.NewAsyncJob(fn, in) 221 out := make(rc.Params) 222 out["jobid"] = job.ID 223 return out, nil 224 } 225 226 // ExecuteJob executes new job synchronously and returns a Param suitable for 227 // output. 228 func ExecuteJob(ctx context.Context, fn rc.Func, in rc.Params) (rc.Params, int64, error) { 229 job, ctx := running.NewSyncJob(ctx, in) 230 job.run(ctx, fn, in) 231 return job.Output, job.ID, job.realErr 232 } 233 234 func init() { 235 rc.Add(rc.Call{ 236 Path: "job/status", 237 Fn: rcJobStatus, 238 Title: "Reads the status of the job ID", 239 Help: `Parameters 240 241 - jobid - id of the job (integer) 242 243 Results 244 245 - finished - boolean 246 - duration - time in seconds that the job ran for 247 - endTime - time the job finished (eg "2018-10-26T18:50:20.528746884+01:00") 248 - error - error from the job or empty string for no error 249 - finished - boolean whether the job has finished or not 250 - id - as passed in above 251 - startTime - time the job started (eg "2018-10-26T18:50:20.528336039+01:00") 252 - success - boolean - true for success false otherwise 253 - output - output of the job as would have been returned if called synchronously 254 - progress - output of the progress related to the underlying job 255 `, 256 }) 257 } 258 259 // Returns the status of a job 260 func rcJobStatus(ctx context.Context, in rc.Params) (out rc.Params, err error) { 261 jobID, err := in.GetInt64("jobid") 262 if err != nil { 263 return nil, err 264 } 265 job := running.Get(jobID) 266 if job == nil { 267 return nil, errors.New("job not found") 268 } 269 job.mu.Lock() 270 defer job.mu.Unlock() 271 out = make(rc.Params) 272 err = rc.Reshape(&out, job) 273 if err != nil { 274 return nil, errors.Wrap(err, "reshape failed in job status") 275 } 276 return out, nil 277 } 278 279 func init() { 280 rc.Add(rc.Call{ 281 Path: "job/list", 282 Fn: rcJobList, 283 Title: "Lists the IDs of the running jobs", 284 Help: `Parameters - None 285 286 Results 287 288 - jobids - array of integer job ids 289 `, 290 }) 291 } 292 293 // Returns list of job ids. 294 func rcJobList(ctx context.Context, in rc.Params) (out rc.Params, err error) { 295 out = make(rc.Params) 296 out["jobids"] = running.IDs() 297 return out, nil 298 } 299 300 func init() { 301 rc.Add(rc.Call{ 302 Path: "job/stop", 303 Fn: rcJobStop, 304 Title: "Stop the running job", 305 Help: `Parameters 306 307 - jobid - id of the job (integer) 308 `, 309 }) 310 } 311 312 // Stops the running job. 313 func rcJobStop(ctx context.Context, in rc.Params) (out rc.Params, err error) { 314 jobID, err := in.GetInt64("jobid") 315 if err != nil { 316 return nil, err 317 } 318 job := running.Get(jobID) 319 if job == nil { 320 return nil, errors.New("job not found") 321 } 322 job.mu.Lock() 323 defer job.mu.Unlock() 324 out = make(rc.Params) 325 job.Stop() 326 return out, nil 327 }