github.com/iron-io/functions@v0.0.0-20180820112432-d59d7d1c40b2/api/mqs/memory.go (about) 1 package mqs 2 3 import ( 4 "context" 5 "errors" 6 "math/rand" 7 "sync" 8 "time" 9 10 "github.com/Sirupsen/logrus" 11 "github.com/google/btree" 12 "github.com/iron-io/functions/api/models" 13 "github.com/iron-io/runner/common" 14 ) 15 16 type MemoryMQ struct { 17 // WorkQueue A buffered channel that we can send work requests on. 18 PriorityQueues []chan *models.Task 19 Ticker *time.Ticker 20 BTree *btree.BTree 21 Timeouts map[string]*TaskItem 22 // Protects B-tree and Timeouts 23 // If this becomes a bottleneck, consider separating the two mutexes. The 24 // goroutine to clear up timed out messages could also become a bottleneck at 25 // some point. May need to switch to bucketing of some sort. 26 Mutex sync.Mutex 27 } 28 29 var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 30 31 func randSeq(n int) string { 32 rand.Seed(time.Now().Unix()) 33 b := make([]rune, n) 34 for i := range b { 35 b[i] = letters[rand.Intn(len(letters))] 36 } 37 return string(b) 38 } 39 40 const NumPriorities = 3 41 42 func NewMemoryMQ() *MemoryMQ { 43 var queues []chan *models.Task 44 for i := 0; i < NumPriorities; i++ { 45 queues = append(queues, make(chan *models.Task, 5000)) 46 } 47 ticker := time.NewTicker(time.Second) 48 mq := &MemoryMQ{ 49 PriorityQueues: queues, 50 Ticker: ticker, 51 BTree: btree.New(2), 52 Timeouts: make(map[string]*TaskItem, 0), 53 } 54 mq.start() 55 logrus.Info("MemoryMQ initialized") 56 return mq 57 } 58 59 func (mq *MemoryMQ) start() { 60 // start goroutine to check for delayed jobs and put them onto regular queue when ready 61 go func() { 62 for range mq.Ticker.C { 63 ji := &TaskItem{ 64 StartAt: time.Now(), 65 } 66 mq.Mutex.Lock() 67 mq.BTree.AscendLessThan(ji, func(a btree.Item) bool { 68 logrus.WithFields(logrus.Fields{"queue": a}).Debug("delayed job move to queue") 69 ji2 := mq.BTree.Delete(a).(*TaskItem) 70 // put it onto the regular queue now 71 _, err := mq.pushForce(ji2.Task) 72 if err != nil { 73 logrus.WithError(err).Error("Couldn't push delayed message onto main queue") 74 } 75 return true 76 }) 77 mq.Mutex.Unlock() 78 } 79 }() 80 // start goroutine to check for messages that have timed out and put them back onto regular queue 81 // TODO: this should be like the delayed messages above. Could even be the same thing as delayed messages, but remove them if job is completed. 82 go func() { 83 for range mq.Ticker.C { 84 ji := &TaskItem{ 85 StartAt: time.Now(), 86 } 87 mq.Mutex.Lock() 88 for _, jobItem := range mq.Timeouts { 89 if jobItem.Less(ji) { 90 delete(mq.Timeouts, jobItem.Task.ID) 91 _, err := mq.pushForce(jobItem.Task) 92 if err != nil { 93 logrus.WithError(err).Error("Couldn't push timed out message onto main queue") 94 } 95 } 96 } 97 mq.Mutex.Unlock() 98 } 99 }() 100 } 101 102 // TaskItem is for the Btree, implements btree.Item 103 type TaskItem struct { 104 Task *models.Task 105 StartAt time.Time 106 } 107 108 func (ji *TaskItem) Less(than btree.Item) bool { 109 // TODO: this could lose jobs: https://godoc.org/github.com/google/btree#Item 110 ji2 := than.(*TaskItem) 111 return ji.StartAt.Before(ji2.StartAt) 112 } 113 114 func (mq *MemoryMQ) Push(ctx context.Context, job *models.Task) (*models.Task, error) { 115 _, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": job.ID}) 116 log.Println("Pushed to MQ") 117 118 // It seems to me that using the job ID in the reservation is acceptable since each job can only have one outstanding reservation. 119 // job.MsgId = randSeq(20) 120 if job.Delay > 0 { 121 // then we'll put into short term storage until ready 122 ji := &TaskItem{ 123 Task: job, 124 StartAt: time.Now().Add(time.Second * time.Duration(job.Delay)), 125 } 126 mq.Mutex.Lock() 127 replaced := mq.BTree.ReplaceOrInsert(ji) 128 mq.Mutex.Unlock() 129 if replaced != nil { 130 log.Warn("Ooops! an item was replaced and therefore lost, not good.") 131 } 132 return job, nil 133 } 134 135 // Push the work onto the queue. 136 return mq.pushForce(job) 137 } 138 139 func (mq *MemoryMQ) pushTimeout(job *models.Task) error { 140 141 ji := &TaskItem{ 142 Task: job, 143 StartAt: time.Now().Add(time.Minute), 144 } 145 mq.Mutex.Lock() 146 mq.Timeouts[job.ID] = ji 147 mq.Mutex.Unlock() 148 return nil 149 } 150 151 func (mq *MemoryMQ) pushForce(job *models.Task) (*models.Task, error) { 152 mq.PriorityQueues[*job.Priority] <- job 153 return job, nil 154 } 155 156 // This is recursive, so be careful how many channels you pass in. 157 func pickEarliestNonblocking(channels ...chan *models.Task) *models.Task { 158 if len(channels) == 0 { 159 return nil 160 } 161 162 select { 163 case job := <-channels[0]: 164 return job 165 default: 166 return pickEarliestNonblocking(channels[1:]...) 167 } 168 } 169 170 func (mq *MemoryMQ) Reserve(ctx context.Context) (*models.Task, error) { 171 job := pickEarliestNonblocking(mq.PriorityQueues[2], mq.PriorityQueues[1], mq.PriorityQueues[0]) 172 if job == nil { 173 return nil, nil 174 } 175 176 _, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": job.ID}) 177 log.Println("Reserved") 178 return job, mq.pushTimeout(job) 179 } 180 181 func (mq *MemoryMQ) Delete(ctx context.Context, job *models.Task) error { 182 _, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": job.ID}) 183 184 mq.Mutex.Lock() 185 defer mq.Mutex.Unlock() 186 _, exists := mq.Timeouts[job.ID] 187 if !exists { 188 return errors.New("Not reserved") 189 } 190 191 delete(mq.Timeouts, job.ID) 192 log.Println("Deleted") 193 return nil 194 }