github.com/iron-io/functions@v0.0.0-20180820112432-d59d7d1c40b2/api/mqs/bolt.go (about) 1 package mqs 2 3 import ( 4 "context" 5 "encoding/binary" 6 "encoding/json" 7 "errors" 8 "fmt" 9 "net/url" 10 "os" 11 "path/filepath" 12 "time" 13 14 "github.com/Sirupsen/logrus" 15 "github.com/boltdb/bolt" 16 "github.com/iron-io/functions/api/models" 17 "github.com/iron-io/runner/common" 18 ) 19 20 type BoltDbMQ struct { 21 db *bolt.DB 22 ticker *time.Ticker 23 } 24 25 type BoltDbConfig struct { 26 FileName string `mapstructure:"filename"` 27 } 28 29 func jobKey(jobID string) []byte { 30 b := make([]byte, len(jobID)+1) 31 b[0] = 'j' 32 copy(b[1:], []byte(jobID)) 33 return b 34 } 35 36 const timeoutToIDKeyPrefix = "id:" 37 38 func timeoutToIDKey(timeout []byte) []byte { 39 b := make([]byte, len(timeout)+len(timeoutToIDKeyPrefix)) 40 copy(b[:], []byte(timeoutToIDKeyPrefix)) 41 copy(b[len(timeoutToIDKeyPrefix):], []byte(timeout)) 42 return b 43 } 44 45 var delayQueueName = []byte("functions_delay") 46 47 func queueName(i int) []byte { 48 return []byte(fmt.Sprintf("functions_%d_queue", i)) 49 } 50 51 func timeoutName(i int) []byte { 52 return []byte(fmt.Sprintf("functions_%d_timeout", i)) 53 } 54 55 func NewBoltMQ(url *url.URL) (*BoltDbMQ, error) { 56 dir := filepath.Dir(url.Path) 57 log := logrus.WithFields(logrus.Fields{"mq": url.Scheme, "dir": dir}) 58 err := os.MkdirAll(dir, 0755) 59 if err != nil { 60 log.WithError(err).Errorln("Could not create data directory for mq") 61 return nil, err 62 } 63 db, err := bolt.Open(url.Path, 0655, &bolt.Options{Timeout: 1 * time.Second}) 64 if err != nil { 65 log.WithError(err).Errorln("Could not open BoltDB file for MQ") 66 return nil, err 67 } 68 69 err = db.Update(func(tx *bolt.Tx) error { 70 for i := 0; i < 3; i++ { 71 _, err := tx.CreateBucketIfNotExists(queueName(i)) 72 if err != nil { 73 log.WithError(err).Errorln("Error creating bucket") 74 return err 75 } 76 _, err = tx.CreateBucketIfNotExists(timeoutName(i)) 77 if err != nil { 78 log.WithError(err).Errorln("Error creating timeout bucket") 79 return err 80 } 81 } 82 _, err = tx.CreateBucketIfNotExists(delayQueueName) 83 if err != nil { 84 log.WithError(err).Errorln("Error creating delay bucket") 85 return err 86 } 87 return nil 88 }) 89 if err != nil { 90 log.WithError(err).Errorln("Error creating timeout bucket") 91 return nil, err 92 } 93 94 ticker := time.NewTicker(time.Second) 95 mq := &BoltDbMQ{ 96 ticker: ticker, 97 db: db, 98 } 99 mq.Start() 100 log.WithFields(logrus.Fields{"file": url.Path}).Debug("BoltDb initialized") 101 return mq, nil 102 } 103 104 func (mq *BoltDbMQ) Start() { 105 go func() { 106 // It would be nice to switch to a tick-less, next-event Timer based model. 107 for range mq.ticker.C { 108 err := mq.db.Update(func(tx *bolt.Tx) error { 109 now := uint64(time.Now().UnixNano()) 110 for i := 0; i < 3; i++ { 111 // Assume our timeouts bucket exists and has resKey encoded keys. 112 jobBucket := tx.Bucket(queueName(i)) 113 timeoutBucket := tx.Bucket(timeoutName(i)) 114 c := timeoutBucket.Cursor() 115 116 var err error 117 for k, v := c.Seek([]byte(resKeyPrefix)); k != nil; k, v = c.Next() { 118 reserved, id := resKeyToProperties(k) 119 if reserved > now { 120 break 121 } 122 err = jobBucket.Put(id, v) 123 if err != nil { 124 return err 125 } 126 timeoutBucket.Delete(k) 127 timeoutBucket.Delete(timeoutToIDKey(k)) 128 } 129 } 130 131 return nil 132 }) 133 if err != nil { 134 logrus.WithError(err).Error("boltdb reservation check error") 135 } 136 137 err = mq.db.Update(func(tx *bolt.Tx) error { 138 now := uint64(time.Now().UnixNano()) 139 // Assume our timeouts bucket exists and has resKey encoded keys. 140 delayBucket := tx.Bucket(delayQueueName) 141 c := delayBucket.Cursor() 142 143 var err error 144 for k, v := c.Seek([]byte(resKeyPrefix)); k != nil; k, v = c.Next() { 145 reserved, id := resKeyToProperties(k) 146 if reserved > now { 147 break 148 } 149 150 priority := binary.BigEndian.Uint32(v) 151 job := delayBucket.Get(id) 152 if job == nil { 153 // oops 154 logrus.Warnf("Expected delayed job, none found with id %s", id) 155 continue 156 } 157 158 jobBucket := tx.Bucket(queueName(int(priority))) 159 err = jobBucket.Put(id, job) 160 if err != nil { 161 return err 162 } 163 164 err := delayBucket.Delete(k) 165 if err != nil { 166 return err 167 } 168 169 return delayBucket.Delete(id) 170 } 171 return nil 172 }) 173 if err != nil { 174 logrus.WithError(err).Error("boltdb delay check error") 175 } 176 } 177 }() 178 } 179 180 // We insert a "reservation" at readyAt, and store the json blob at the msg 181 // key. The timer loop plucks this out and puts it in the jobs bucket when the 182 // time elapses. The value stored at the reservation key is the priority. 183 func (mq *BoltDbMQ) delayTask(job *models.Task) (*models.Task, error) { 184 readyAt := time.Now().Add(time.Duration(job.Delay) * time.Second) 185 err := mq.db.Update(func(tx *bolt.Tx) error { 186 b := tx.Bucket(delayQueueName) 187 id, _ := b.NextSequence() 188 buf, err := json.Marshal(job) 189 if err != nil { 190 return err 191 } 192 193 key := msgKey(id) 194 err = b.Put(key, buf) 195 if err != nil { 196 return err 197 } 198 199 pb := make([]byte, 4) 200 binary.BigEndian.PutUint32(pb[:], uint32(*job.Priority)) 201 reservation := resKey(key, readyAt) 202 return b.Put(reservation, pb) 203 }) 204 return job, err 205 } 206 207 func (mq *BoltDbMQ) Push(ctx context.Context, job *models.Task) (*models.Task, error) { 208 ctx, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": job.ID}) 209 log.Println("Pushed to MQ") 210 211 if job.Delay > 0 { 212 return mq.delayTask(job) 213 } 214 215 err := mq.db.Update(func(tx *bolt.Tx) error { 216 b := tx.Bucket(queueName(int(*job.Priority))) 217 218 id, _ := b.NextSequence() 219 220 buf, err := json.Marshal(job) 221 if err != nil { 222 return err 223 } 224 225 return b.Put(msgKey(id), buf) 226 }) 227 if err != nil { 228 return nil, err 229 } 230 231 return job, nil 232 233 } 234 235 const msgKeyPrefix = "j:" 236 const msgKeyLength = len(msgKeyPrefix) + 8 237 const resKeyPrefix = "r:" 238 239 // r:<timestamp>:msgKey 240 // The msgKey is used to introduce uniqueness within the timestamp. It probably isn't required. 241 const resKeyLength = len(resKeyPrefix) + msgKeyLength + 8 242 243 func msgKey(v uint64) []byte { 244 b := make([]byte, msgKeyLength) 245 copy(b[:], []byte(msgKeyPrefix)) 246 binary.BigEndian.PutUint64(b[len(msgKeyPrefix):], v) 247 return b 248 } 249 250 func resKey(jobKey []byte, reservedUntil time.Time) []byte { 251 b := make([]byte, resKeyLength) 252 copy(b[:], []byte(resKeyPrefix)) 253 binary.BigEndian.PutUint64(b[len(resKeyPrefix):], uint64(reservedUntil.UnixNano())) 254 copy(b[len(resKeyPrefix)+8:], jobKey) 255 return b 256 } 257 258 func resKeyToProperties(key []byte) (uint64, []byte) { 259 if len(key) != resKeyLength { 260 return 0, nil 261 } 262 263 reservedUntil := binary.BigEndian.Uint64(key[len(resKeyPrefix):]) 264 return reservedUntil, key[len(resKeyPrefix)+8:] 265 } 266 267 func (mq *BoltDbMQ) Reserve(ctx context.Context) (*models.Task, error) { 268 // Start a writable transaction. 269 tx, err := mq.db.Begin(true) 270 if err != nil { 271 return nil, err 272 } 273 defer tx.Rollback() 274 275 for i := 2; i >= 0; i-- { 276 // Use the transaction... 277 b := tx.Bucket(queueName(i)) 278 c := b.Cursor() 279 key, value := c.Seek([]byte(msgKeyPrefix)) 280 if key == nil { 281 // No jobs, try next bucket 282 continue 283 } 284 285 b.Delete(key) 286 287 var job models.Task 288 err = json.Unmarshal([]byte(value), &job) 289 if err != nil { 290 return nil, err 291 } 292 293 reservationKey := resKey(key, time.Now().Add(time.Minute)) 294 b = tx.Bucket(timeoutName(i)) 295 // Reserve introduces 3 keys in timeout bucket: 296 // Save reservationKey -> Task to allow release 297 // Save job.ID -> reservationKey to allow Deletes 298 // Save reservationKey -> job.ID to allow clearing job.ID -> reservationKey in recovery without unmarshaling the job. 299 // On Delete: 300 // We have job ID, we get the reservationKey 301 // Delete job.ID -> reservationKey 302 // Delete reservationKey -> job.ID 303 // Delete reservationKey -> Task 304 // On Release: 305 // We have reservationKey, we get the jobID 306 // Delete reservationKey -> job.ID 307 // Delete job.ID -> reservationKey 308 // Move reservationKey -> Task to job bucket. 309 b.Put(reservationKey, value) 310 b.Put(jobKey(job.ID), reservationKey) 311 b.Put(timeoutToIDKey(reservationKey), []byte(job.ID)) 312 313 // Commit the transaction and check for error. 314 if err := tx.Commit(); err != nil { 315 return nil, err 316 } 317 318 _, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": job.ID}) 319 log.Println("Reserved") 320 321 return &job, nil 322 } 323 324 return nil, nil 325 } 326 327 func (mq *BoltDbMQ) Delete(ctx context.Context, job *models.Task) error { 328 _, log := common.LoggerWithFields(ctx, logrus.Fields{"call_id": job.ID}) 329 defer log.Println("Deleted") 330 331 return mq.db.Update(func(tx *bolt.Tx) error { 332 b := tx.Bucket(timeoutName(int(*job.Priority))) 333 k := jobKey(job.ID) 334 335 reservationKey := b.Get(k) 336 if reservationKey == nil { 337 return errors.New("Not found") 338 } 339 340 for _, k := range [][]byte{k, timeoutToIDKey(reservationKey), reservationKey} { 341 err := b.Delete(k) 342 if err != nil { 343 return err 344 } 345 } 346 347 return nil 348 }) 349 }