github.com/mattermost/mattermost-plugin-api@v0.1.4/cluster/job_once_scheduler.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See LICENSE.txt for license information. 3 4 package cluster 5 6 import ( 7 "strings" 8 "sync" 9 "time" 10 11 "github.com/pkg/errors" 12 ) 13 14 // syncedCallback uses the mutex to make things predictable for the client: the callback will be 15 // called once at a time (the client does not need to worry about concurrency within the callback) 16 type syncedCallback struct { 17 mu sync.Mutex 18 callback func(string, any) 19 } 20 21 type syncedJobs struct { 22 mu sync.RWMutex 23 jobs map[string]*JobOnce 24 } 25 26 type JobOnceScheduler struct { 27 pluginAPI JobPluginAPI 28 29 startedMu sync.RWMutex 30 started bool 31 32 activeJobs *syncedJobs 33 storedCallback *syncedCallback 34 } 35 36 var schedulerOnce sync.Once 37 var s *JobOnceScheduler 38 39 // GetJobOnceScheduler returns a scheduler which is ready to have its callback set. Repeated 40 // calls will return the same scheduler. 41 func GetJobOnceScheduler(pluginAPI JobPluginAPI) *JobOnceScheduler { 42 schedulerOnce.Do(func() { 43 s = &JobOnceScheduler{ 44 pluginAPI: pluginAPI, 45 activeJobs: &syncedJobs{ 46 jobs: make(map[string]*JobOnce), 47 }, 48 storedCallback: &syncedCallback{}, 49 } 50 }) 51 return s 52 } 53 54 // Start starts the Scheduler. It finds all previous ScheduleOnce jobs and starts them running, and 55 // fires any jobs that have reached or exceeded their runAt time. Thus, even if a cluster goes down 56 // and is restarted, Start will restart previously scheduled jobs. 57 func (s *JobOnceScheduler) Start() error { 58 s.startedMu.Lock() 59 defer s.startedMu.Unlock() 60 if s.started { 61 return errors.New("scheduler has already been started") 62 } 63 64 if err := s.verifyCallbackExists(); err != nil { 65 return errors.Wrap(err, "callback not found; cannot start scheduler") 66 } 67 68 if err := s.scheduleNewJobsFromDB(); err != nil { 69 return errors.Wrap(err, "could not start JobOnceScheduler due to error") 70 } 71 72 go s.pollForNewScheduledJobs() 73 74 s.started = true 75 76 return nil 77 } 78 79 // SetCallback sets the scheduler's callback. When a job fires, the callback will be called with 80 // the job's id. 81 func (s *JobOnceScheduler) SetCallback(callback func(string, any)) error { 82 if callback == nil { 83 return errors.New("callback cannot be nil") 84 } 85 86 s.storedCallback.mu.Lock() 87 defer s.storedCallback.mu.Unlock() 88 89 s.storedCallback.callback = callback 90 return nil 91 } 92 93 // ListScheduledJobs returns a list of the jobs in the db that have been scheduled. There is no 94 // guarantee that list is accurate by the time the caller reads the list. E.g., the jobs in the list 95 // may have been run, canceled, or new jobs may have scheduled. 96 func (s *JobOnceScheduler) ListScheduledJobs() ([]JobOnceMetadata, error) { 97 var ret []JobOnceMetadata 98 for i := 0; ; i++ { 99 keys, err := s.pluginAPI.KVList(i, keysPerPage) 100 if err != nil { 101 return nil, errors.Wrap(err, "error getting KVList") 102 } 103 for _, k := range keys { 104 if strings.HasPrefix(k, oncePrefix) { 105 metadata, err := readMetadata(s.pluginAPI, k[len(oncePrefix):]) 106 if err != nil { 107 s.pluginAPI.LogError(errors.Wrap(err, "could not retrieve data from plugin kvstore for key: "+k).Error()) 108 continue 109 } 110 if metadata == nil { 111 continue 112 } 113 114 ret = append(ret, *metadata) 115 } 116 } 117 118 if len(keys) < keysPerPage { 119 break 120 } 121 } 122 123 return ret, nil 124 } 125 126 // ScheduleOnce creates a scheduled job that will run once. When the clock reaches runAt, the 127 // callback will be called with key and props as the argument. 128 // 129 // If the job key already exists in the db, this will return an error. To reschedule a job, first 130 // cancel the original then schedule it again. 131 func (s *JobOnceScheduler) ScheduleOnce(key string, runAt time.Time, props any) (*JobOnce, error) { 132 s.startedMu.RLock() 133 defer s.startedMu.RUnlock() 134 if !s.started { 135 return nil, errors.New("start the scheduler before adding jobs") 136 } 137 138 job, err := newJobOnce(s.pluginAPI, key, runAt, s.storedCallback, s.activeJobs, props) 139 if err != nil { 140 return nil, errors.Wrap(err, "could not create new job") 141 } 142 143 if err = job.saveMetadata(); err != nil { 144 return nil, errors.Wrap(err, "could not save job metadata") 145 } 146 147 s.runAndTrack(job) 148 149 return job, nil 150 } 151 152 // Cancel cancels a job by its key. This is useful if the plugin lost the original *JobOnce, or 153 // is stopping a job found in ListScheduledJobs(). 154 func (s *JobOnceScheduler) Cancel(key string) { 155 // using an anonymous function because job.Close() below needs access to the activeJobs mutex 156 job := func() *JobOnce { 157 s.activeJobs.mu.RLock() 158 defer s.activeJobs.mu.RUnlock() 159 j, ok := s.activeJobs.jobs[key] 160 if ok { 161 return j 162 } 163 164 // Job wasn't active, so no need to call CancelWhileHoldingMutex (which shuts down the 165 // goroutine). There's a condition where another server in the cluster started the job, and 166 // the current server hasn't polled for it yet. To solve that case, delete it from the db. 167 mutex, err := NewMutex(s.pluginAPI, key) 168 if err != nil { 169 s.pluginAPI.LogError(errors.Wrap(err, "failed to create job mutex in Cancel for key: "+key).Error()) 170 } 171 mutex.Lock() 172 defer mutex.Unlock() 173 174 _ = s.pluginAPI.KVDelete(oncePrefix + key) 175 176 return nil 177 }() 178 179 if job != nil { 180 job.Cancel() 181 } 182 } 183 184 func (s *JobOnceScheduler) scheduleNewJobsFromDB() error { 185 scheduled, err := s.ListScheduledJobs() 186 if err != nil { 187 return errors.Wrap(err, "could not read scheduled jobs from db") 188 } 189 190 for _, m := range scheduled { 191 job, err := newJobOnce(s.pluginAPI, m.Key, m.RunAt, s.storedCallback, s.activeJobs, m.Props) 192 if err != nil { 193 s.pluginAPI.LogError(errors.Wrap(err, "could not create new job for key: "+m.Key).Error()) 194 continue 195 } 196 197 s.runAndTrack(job) 198 } 199 200 return nil 201 } 202 203 func (s *JobOnceScheduler) runAndTrack(job *JobOnce) { 204 s.activeJobs.mu.Lock() 205 defer s.activeJobs.mu.Unlock() 206 207 // has this been scheduled already on this server? 208 if _, ok := s.activeJobs.jobs[job.key]; ok { 209 return 210 } 211 212 go job.run() 213 214 s.activeJobs.jobs[job.key] = job 215 } 216 217 // pollForNewScheduledJobs will only be started once per plugin. It doesn't need to be stopped. 218 func (s *JobOnceScheduler) pollForNewScheduledJobs() { 219 for { 220 <-time.After(pollNewJobsInterval + addJitter()) 221 222 if err := s.scheduleNewJobsFromDB(); err != nil { 223 s.pluginAPI.LogError("pluginAPI scheduleOnce poller encountered an error but is still polling", "error", err) 224 } 225 } 226 } 227 228 func (s *JobOnceScheduler) verifyCallbackExists() error { 229 s.storedCallback.mu.Lock() 230 defer s.storedCallback.mu.Unlock() 231 232 if s.storedCallback.callback == nil { 233 return errors.New("set callback before starting the scheduler") 234 } 235 return nil 236 }