github.com/iron-io/functions@v0.0.0-20180820112432-d59d7d1c40b2/api/datastore/bolt/bolt.go (about) 1 package bolt 2 3 import ( 4 "encoding/json" 5 "net/url" 6 "os" 7 "path/filepath" 8 "time" 9 10 "context" 11 12 "regexp" 13 "strings" 14 15 "github.com/Sirupsen/logrus" 16 "github.com/boltdb/bolt" 17 "github.com/iron-io/functions/api/datastore/internal/datastoreutil" 18 "github.com/iron-io/functions/api/models" 19 ) 20 21 type BoltDatastore struct { 22 routesBucket []byte 23 appsBucket []byte 24 logsBucket []byte 25 extrasBucket []byte 26 db *bolt.DB 27 log logrus.FieldLogger 28 } 29 30 func New(url *url.URL) (models.Datastore, error) { 31 dir := filepath.Dir(url.Path) 32 log := logrus.WithFields(logrus.Fields{"db": url.Scheme, "dir": dir}) 33 err := os.MkdirAll(dir, 0755) 34 if err != nil { 35 log.WithError(err).Errorln("Could not create data directory for db") 36 return nil, err 37 } 38 log.WithFields(logrus.Fields{"path": url.Path}).Debug("Creating bolt db") 39 db, err := bolt.Open(url.Path, 0655, &bolt.Options{Timeout: 1 * time.Second}) 40 if err != nil { 41 log.WithError(err).Errorln("Error on bolt.Open") 42 return nil, err 43 } 44 // I don't think we need a prefix here do we? Made it blank. If we do, we should call the query param "prefix" instead of bucket. 45 bucketPrefix := "" 46 if url.Query()["bucket"] != nil { 47 bucketPrefix = url.Query()["bucket"][0] 48 } 49 routesBucketName := []byte(bucketPrefix + "routes") 50 appsBucketName := []byte(bucketPrefix + "apps") 51 logsBucketName := []byte(bucketPrefix + "logs") 52 extrasBucketName := []byte(bucketPrefix + "extras") // todo: think of a better name 53 err = db.Update(func(tx *bolt.Tx) error { 54 for _, name := range [][]byte{routesBucketName, appsBucketName, logsBucketName, extrasBucketName} { 55 _, err := tx.CreateBucketIfNotExists(name) 56 if err != nil { 57 log.WithError(err).WithFields(logrus.Fields{"name": name}).Error("create bucket") 58 return err 59 } 60 } 61 return nil 62 }) 63 if err != nil { 64 log.WithError(err).Errorln("Error creating bolt buckets") 65 return nil, err 66 } 67 68 ds := &BoltDatastore{ 69 routesBucket: routesBucketName, 70 appsBucket: appsBucketName, 71 logsBucket: logsBucketName, 72 extrasBucket: extrasBucketName, 73 db: db, 74 log: log, 75 } 76 log.WithFields(logrus.Fields{"prefix": bucketPrefix, "file": url.Path}).Debug("BoltDB initialized") 77 78 return datastoreutil.NewValidator(ds), nil 79 } 80 81 func (ds *BoltDatastore) InsertApp(ctx context.Context, app *models.App) (*models.App, error) { 82 appname := []byte(app.Name) 83 84 err := ds.db.Update(func(tx *bolt.Tx) error { 85 bIm := tx.Bucket(ds.appsBucket) 86 87 v := bIm.Get(appname) 88 if v != nil { 89 return models.ErrAppsAlreadyExists 90 } 91 92 buf, err := json.Marshal(app) 93 if err != nil { 94 return err 95 } 96 97 err = bIm.Put(appname, buf) 98 if err != nil { 99 return err 100 } 101 bjParent := tx.Bucket(ds.routesBucket) 102 _, err = bjParent.CreateBucketIfNotExists([]byte(app.Name)) 103 if err != nil { 104 return err 105 } 106 return nil 107 }) 108 109 return app, err 110 } 111 112 func (ds *BoltDatastore) UpdateApp(ctx context.Context, newapp *models.App) (*models.App, error) { 113 var app *models.App 114 appname := []byte(newapp.Name) 115 116 err := ds.db.Update(func(tx *bolt.Tx) error { 117 bIm := tx.Bucket(ds.appsBucket) 118 119 v := bIm.Get(appname) 120 if v == nil { 121 return models.ErrAppsNotFound 122 } 123 124 err := json.Unmarshal(v, &app) 125 if err != nil { 126 return err 127 } 128 129 app.UpdateConfig(newapp.Config) 130 131 buf, err := json.Marshal(app) 132 if err != nil { 133 return err 134 } 135 136 err = bIm.Put(appname, buf) 137 if err != nil { 138 return err 139 } 140 bjParent := tx.Bucket(ds.routesBucket) 141 _, err = bjParent.CreateBucketIfNotExists([]byte(app.Name)) 142 if err != nil { 143 return err 144 } 145 return nil 146 }) 147 148 return app, err 149 } 150 151 func (ds *BoltDatastore) RemoveApp(ctx context.Context, appName string) error { 152 err := ds.db.Update(func(tx *bolt.Tx) error { 153 bIm := tx.Bucket(ds.appsBucket) 154 err := bIm.Delete([]byte(appName)) 155 if err != nil { 156 return err 157 } 158 bjParent := tx.Bucket(ds.routesBucket) 159 err = bjParent.DeleteBucket([]byte(appName)) 160 if err != nil { 161 return err 162 } 163 return nil 164 }) 165 return err 166 } 167 168 func (ds *BoltDatastore) GetApps(ctx context.Context, filter *models.AppFilter) ([]*models.App, error) { 169 res := []*models.App{} 170 err := ds.db.View(func(tx *bolt.Tx) error { 171 b := tx.Bucket(ds.appsBucket) 172 err2 := b.ForEach(func(key, v []byte) error { 173 app := &models.App{} 174 err := json.Unmarshal(v, app) 175 if err != nil { 176 return err 177 } 178 if applyAppFilter(app, filter) { 179 res = append(res, app) 180 } 181 return nil 182 }) 183 if err2 != nil { 184 logrus.WithError(err2).Errorln("Couldn't get apps!") 185 } 186 return nil 187 }) 188 if err != nil { 189 return nil, err 190 } 191 return res, nil 192 } 193 194 func (ds *BoltDatastore) GetApp(ctx context.Context, name string) (*models.App, error) { 195 var res *models.App 196 err := ds.db.View(func(tx *bolt.Tx) error { 197 b := tx.Bucket(ds.appsBucket) 198 v := b.Get([]byte(name)) 199 if v != nil { 200 app := &models.App{} 201 err := json.Unmarshal(v, app) 202 if err != nil { 203 return err 204 } 205 res = app 206 } else { 207 return models.ErrAppsNotFound 208 } 209 return nil 210 }) 211 if err != nil { 212 return nil, err 213 } 214 return res, nil 215 } 216 217 func (ds *BoltDatastore) getRouteBucketForApp(tx *bolt.Tx, appName string) (*bolt.Bucket, error) { 218 // todo: should this be reversed? Make a bucket for each app that contains sub buckets for routes, etc 219 bp := tx.Bucket(ds.routesBucket) 220 b := bp.Bucket([]byte(appName)) 221 if b == nil { 222 return nil, models.ErrAppsNotFound 223 } 224 return b, nil 225 } 226 227 func (ds *BoltDatastore) InsertRoute(ctx context.Context, route *models.Route) (*models.Route, error) { 228 routePath := []byte(route.Path) 229 230 err := ds.db.Update(func(tx *bolt.Tx) error { 231 b, err := ds.getRouteBucketForApp(tx, route.AppName) 232 if err != nil { 233 return err 234 } 235 236 v := b.Get(routePath) 237 if v != nil { 238 return models.ErrRoutesAlreadyExists 239 } 240 241 buf, err := json.Marshal(route) 242 if err != nil { 243 return err 244 } 245 246 err = b.Put(routePath, buf) 247 if err != nil { 248 return err 249 } 250 return nil 251 }) 252 if err != nil { 253 return nil, err 254 } 255 return route, nil 256 } 257 258 func (ds *BoltDatastore) UpdateRoute(ctx context.Context, newroute *models.Route) (*models.Route, error) { 259 routePath := []byte(newroute.Path) 260 261 var route *models.Route 262 263 err := ds.db.Update(func(tx *bolt.Tx) error { 264 b, err := ds.getRouteBucketForApp(tx, newroute.AppName) 265 if err != nil { 266 return err 267 } 268 269 v := b.Get(routePath) 270 if v == nil { 271 return models.ErrRoutesNotFound 272 } 273 274 err = json.Unmarshal(v, &route) 275 if err != nil { 276 return err 277 } 278 279 route.Update(newroute) 280 281 buf, err := json.Marshal(route) 282 if err != nil { 283 return err 284 } 285 286 return b.Put(routePath, buf) 287 }) 288 if err != nil { 289 return nil, err 290 } 291 return route, nil 292 } 293 294 func (ds *BoltDatastore) RemoveRoute(ctx context.Context, appName, routePath string) error { 295 err := ds.db.Update(func(tx *bolt.Tx) error { 296 b, err := ds.getRouteBucketForApp(tx, appName) 297 if err != nil { 298 return err 299 } 300 301 err = b.Delete([]byte(routePath)) 302 if err != nil { 303 return err 304 } 305 return nil 306 }) 307 if err != nil { 308 return err 309 } 310 return nil 311 } 312 313 func (ds *BoltDatastore) GetRoute(ctx context.Context, appName, routePath string) (*models.Route, error) { 314 var route *models.Route 315 err := ds.db.View(func(tx *bolt.Tx) error { 316 b, err := ds.getRouteBucketForApp(tx, appName) 317 if err != nil { 318 return err 319 } 320 321 v := b.Get([]byte(routePath)) 322 if v == nil { 323 return models.ErrRoutesNotFound 324 } 325 326 if v != nil { 327 err = json.Unmarshal(v, &route) 328 } 329 return err 330 }) 331 return route, err 332 } 333 334 func (ds *BoltDatastore) GetRoutesByApp(ctx context.Context, appName string, filter *models.RouteFilter) ([]*models.Route, error) { 335 res := []*models.Route{} 336 err := ds.db.View(func(tx *bolt.Tx) error { 337 b := tx.Bucket(ds.routesBucket).Bucket([]byte(appName)) 338 if b == nil { 339 return nil 340 } 341 342 i := 0 343 c := b.Cursor() 344 345 var k, v []byte 346 k, v = c.Last() 347 348 // Iterate backwards, newest first 349 for ; k != nil; k, v = c.Prev() { 350 var route models.Route 351 err := json.Unmarshal(v, &route) 352 if err != nil { 353 return err 354 } 355 if applyRouteFilter(&route, filter) { 356 i++ 357 res = append(res, &route) 358 } 359 } 360 return nil 361 }) 362 if err != nil { 363 return nil, err 364 } 365 return res, nil 366 } 367 368 func (ds *BoltDatastore) GetRoutes(ctx context.Context, filter *models.RouteFilter) ([]*models.Route, error) { 369 res := []*models.Route{} 370 err := ds.db.View(func(tx *bolt.Tx) error { 371 i := 0 372 rbucket := tx.Bucket(ds.routesBucket) 373 374 b := rbucket.Cursor() 375 var k, v []byte 376 k, v = b.First() 377 378 // Iterates all buckets 379 for ; k != nil && v == nil; k, v = b.Next() { 380 bucket := rbucket.Bucket(k) 381 r := bucket.Cursor() 382 var k2, v2 []byte 383 k2, v2 = r.Last() 384 // Iterate all routes 385 for ; k2 != nil; k2, v2 = r.Prev() { 386 var route models.Route 387 err := json.Unmarshal(v2, &route) 388 if err != nil { 389 return err 390 } 391 if applyRouteFilter(&route, filter) { 392 i++ 393 res = append(res, &route) 394 } 395 } 396 } 397 return nil 398 }) 399 if err != nil { 400 return nil, err 401 } 402 return res, nil 403 } 404 405 func (ds *BoltDatastore) Put(ctx context.Context, key, value []byte) error { 406 ds.db.Update(func(tx *bolt.Tx) error { 407 b := tx.Bucket(ds.extrasBucket) // todo: maybe namespace by app? 408 err := b.Put(key, value) 409 return err 410 }) 411 return nil 412 } 413 414 func (ds *BoltDatastore) Get(ctx context.Context, key []byte) ([]byte, error) { 415 var ret []byte 416 ds.db.View(func(tx *bolt.Tx) error { 417 b := tx.Bucket(ds.extrasBucket) 418 ret = b.Get(key) 419 return nil 420 }) 421 return ret, nil 422 } 423 424 func applyAppFilter(app *models.App, filter *models.AppFilter) bool { 425 if filter != nil && filter.Name != "" { 426 nameLike, err := regexp.MatchString(strings.Replace(filter.Name, "%", ".*", -1), app.Name) 427 return err == nil && nameLike 428 } 429 430 return true 431 } 432 433 func applyRouteFilter(route *models.Route, filter *models.RouteFilter) bool { 434 return filter == nil || (filter.Path == "" || route.Path == filter.Path) && 435 (filter.AppName == "" || route.AppName == filter.AppName) && 436 (filter.Image == "" || route.Image == filter.Image) 437 }