github.com/golang/dep@v0.5.4/gps/source_cache_bolt.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package gps 6 7 import ( 8 "fmt" 9 "log" 10 "os" 11 "path" 12 "path/filepath" 13 "strings" 14 "time" 15 16 "github.com/boltdb/bolt" 17 "github.com/golang/dep/gps/internal/pb" 18 "github.com/golang/dep/gps/pkgtree" 19 "github.com/golang/protobuf/proto" 20 "github.com/jmank88/nuts" 21 "github.com/pkg/errors" 22 ) 23 24 // boltCacheFilename is a versioned filename for the bolt cache. The version 25 // must be incremented whenever incompatible changes are made. 26 const boltCacheFilename = "bolt-v1.db" 27 28 // boltCache manages a bolt.DB cache and provides singleSourceCaches. 29 type boltCache struct { 30 db *bolt.DB 31 epoch int64 // getters will not return values older than this unix timestamp 32 logger *log.Logger // info logging 33 } 34 35 // newBoltCache returns a new boltCache backed by a BoltDB file under the cache directory. 36 func newBoltCache(cd string, epoch int64, logger *log.Logger) (*boltCache, error) { 37 path := filepath.Join(cd, boltCacheFilename) 38 dir := filepath.Dir(path) 39 if fi, err := os.Stat(dir); os.IsNotExist(err) { 40 if err := os.MkdirAll(dir, os.ModeDir|os.ModePerm); err != nil { 41 return nil, errors.Wrapf(err, "failed to create source cache directory: %s", dir) 42 } 43 } else if err != nil { 44 return nil, errors.Wrapf(err, "failed to check source cache directory: %s", dir) 45 } else if !fi.IsDir() { 46 return nil, errors.Wrapf(err, "source cache path is not directory: %s", dir) 47 } 48 db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second}) 49 if err != nil { 50 return nil, errors.Wrapf(err, "failed to open BoltDB cache file %q", path) 51 } 52 return &boltCache{ 53 db: db, 54 epoch: epoch, 55 logger: logger, 56 }, nil 57 } 58 59 // newSingleSourceCache returns a new singleSourceCache for pi. 60 func (c *boltCache) newSingleSourceCache(pi ProjectIdentifier) singleSourceCache { 61 return &singleSourceCacheBolt{ 62 boltCache: c, 63 sourceName: []byte(pi.normalizedSource()), 64 } 65 } 66 67 // close releases all cache resources. 68 func (c *boltCache) close() error { 69 return errors.Wrapf(c.db.Close(), "error closing Bolt database %q", c.db.String()) 70 } 71 72 // singleSourceCacheBolt implements a singleSourceCache backed by a persistent BoltDB file. 73 // Version mappings are timestamped, and the `epoch` field limits the age of returned values. 74 // Database access methods are safe for concurrent use. 75 // 76 // Implementation: 77 // 78 // Each source has a top-level bucket containing sub-buckets for (1) versions and (2) revisions. 79 // 80 // 1) Versions buckets hold version keys with revision values: 81 // 82 // Bucket: "v<timestamp>" 83 // Keys: Unpaired Versions serialized via ConstraintMsg 84 // Values: "<revision>" 85 // 86 // 2) Revision buckets hold (a) manifest and lock data for various ProjectAnalyzers, 87 // (b) package trees, and (c) version lists. 88 // 89 // Bucket: "r<revision>" 90 // 91 // a) Manifest and Lock info are stored in buckets derived from ProjectAnalyzer.Info: 92 // 93 // Sub-Bucket: "<name>.<version>m", "<name>.<version>l" 94 // Keys/Values: Manifest or Lock fields 95 // 96 // b) Package tree buckets contain package import path keys and package-or-error buckets: 97 // 98 // Sub-Bucket: "p" 99 // Sub-Bucket: "<import_path>" 100 // Key/Values: PackageOrErr fields 101 // 102 // c) Revision-versions buckets contain lists of version values: 103 // 104 // Sub-Bucket: "v<timestamp>" 105 // Keys: "<sequence_number>" 106 // Values: Unpaired Versions serialized via ConstraintMsg 107 type singleSourceCacheBolt struct { 108 *boltCache 109 sourceName []byte 110 } 111 112 func (s *singleSourceCacheBolt) setManifestAndLock(rev Revision, ai ProjectAnalyzerInfo, m Manifest, l Lock) { 113 err := s.updateRevBucket(rev, func(b *bolt.Bucket) error { 114 info := ai.String() 115 name := make([]byte, len(info)+1) 116 copy(name, info) 117 name[len(info)] = 'm' 118 119 if b.Bucket(name) != nil { 120 if err := b.DeleteBucket(name); err != nil { 121 return err 122 } 123 } 124 125 // Manifest 126 mb, err := b.CreateBucket(name) 127 if err != nil { 128 return err 129 } 130 if err := cachePutManifest(mb, m); err != nil { 131 return errors.Wrap(err, "failed to put manifest") 132 } 133 if l == nil { 134 return nil 135 } 136 137 // Lock 138 name[len(info)] = 'l' 139 if b.Bucket(name) != nil { 140 if err := b.DeleteBucket(name); err != nil { 141 return err 142 } 143 } 144 lb, err := b.CreateBucket(name) 145 if err != nil { 146 return err 147 } 148 return errors.Wrap(cachePutLock(lb, l), "failed to put lock") 149 }) 150 if err != nil { 151 s.logger.Println(errors.Wrapf(err, "failed to cache manifest/lock for revision %q, analyzer: %v", rev, ai)) 152 } 153 } 154 155 func (s *singleSourceCacheBolt) getManifestAndLock(rev Revision, ai ProjectAnalyzerInfo) (m Manifest, l Lock, ok bool) { 156 err := s.viewRevBucket(rev, func(b *bolt.Bucket) error { 157 info := ai.String() 158 name := make([]byte, len(info)+1) 159 copy(name, info) 160 name[len(info)] = 'm' 161 162 // Manifest 163 mb := b.Bucket(name) 164 if mb == nil { 165 return nil 166 } 167 var err error 168 m, err = cacheGetManifest(mb) 169 if err != nil { 170 return errors.Wrap(err, "failed to get manifest") 171 } 172 173 // Lock 174 name[len(info)] = 'l' 175 lb := b.Bucket(name) 176 if lb == nil { 177 ok = true 178 return nil 179 } 180 l, err = cacheGetLock(lb) 181 if err != nil { 182 return errors.Wrap(err, "failed to get lock") 183 } 184 185 ok = true 186 return nil 187 }) 188 if err != nil { 189 s.logger.Println(errors.Wrapf(err, "failed to get cached manifest/lock for revision %q, analyzer: %v", rev, ai)) 190 } 191 return 192 } 193 194 func (s *singleSourceCacheBolt) setPackageTree(rev Revision, ptree pkgtree.PackageTree) { 195 err := s.updateRevBucket(rev, func(b *bolt.Bucket) error { 196 if b.Bucket(cacheKeyPTree) != nil { 197 if err := b.DeleteBucket(cacheKeyPTree); err != nil { 198 return err 199 } 200 } 201 ptrees, err := b.CreateBucket(cacheKeyPTree) 202 if err != nil { 203 return err 204 } 205 206 root := string(ptree.ImportRoot) 207 for ip, poe := range ptree.Packages { 208 // Stored by relative import path. 209 rip := strings.TrimPrefix(ip, root) 210 if rip == "" { 211 rip = "/" 212 } 213 pb, err := ptrees.CreateBucket([]byte(rip)) 214 if err != nil { 215 return err 216 } 217 218 if err := cachePutPackageOrErr(pb, poe); err != nil { 219 return err 220 } 221 } 222 return nil 223 }) 224 if err != nil { 225 s.logger.Println(errors.Wrapf(err, "failed to cache package tree for revision %q", rev)) 226 } 227 } 228 229 func (s *singleSourceCacheBolt) getPackageTree(rev Revision, pr ProjectRoot) (ptree pkgtree.PackageTree, ok bool) { 230 err := s.viewRevBucket(rev, func(b *bolt.Bucket) error { 231 ptrees := b.Bucket(cacheKeyPTree) 232 if ptrees == nil { 233 return nil 234 } 235 236 pkgs := make(map[string]pkgtree.PackageOrErr) 237 err := ptrees.ForEach(func(rip, _ []byte) error { 238 poe, err := cacheGetPackageOrErr(ptrees.Bucket(rip)) 239 if err != nil { 240 return err 241 } 242 srip := string(rip) 243 if srip == "/" { 244 srip = "" 245 } 246 // Return full import paths. 247 ip := path.Join(string(pr), srip) 248 if poe.Err == nil { 249 poe.P.ImportPath = ip 250 } 251 pkgs[ip] = poe 252 return nil 253 }) 254 if err != nil { 255 return err 256 } 257 ptree.ImportRoot = string(pr) 258 ptree.Packages = pkgs 259 ok = true 260 return nil 261 }) 262 if err != nil { 263 s.logger.Println(errors.Wrapf(err, "failed to get cached package tree for revision %q", rev)) 264 } 265 return 266 } 267 268 func (s *singleSourceCacheBolt) markRevisionExists(rev Revision) { 269 err := s.updateRevBucket(rev, func(versions *bolt.Bucket) error { 270 return nil 271 }) 272 if err != nil { 273 s.logger.Println(errors.Wrapf(err, "failed to mark revision %q in cache", rev)) 274 } 275 } 276 277 func (s *singleSourceCacheBolt) setVersionMap(pvs []PairedVersion) { 278 err := s.updateSourceBucket(func(src *bolt.Bucket) error { 279 if err := cachePrefixDelete(src, cacheVersion); err != nil { 280 return err 281 } 282 vk := cacheTimestampedKey(cacheVersion, time.Now()) 283 versions, err := src.CreateBucket(vk) 284 if err != nil { 285 return err 286 } 287 288 c := src.Cursor() 289 for k, _ := c.Seek(cacheKeyRevision); len(k) > 0 && k[0] == cacheRevision; k, _ = c.Next() { 290 rb := src.Bucket(k) 291 if err := cachePrefixDelete(rb, cacheVersion); err != nil { 292 return err 293 } 294 } 295 296 revVersions := make(map[Revision]*bolt.Bucket) 297 key := make(nuts.Key, nuts.KeyLen(uint64(len(pvs)-1))) 298 var msg pb.Constraint 299 for i, pv := range pvs { 300 uv, rev := pv.Unpair(), pv.Revision() 301 uv.copyTo(&msg) 302 uvB, err := proto.Marshal(&msg) 303 if err != nil { 304 return errors.Wrapf(err, "failed to serialize UnpairedVersion: %#v", uv) 305 } 306 307 if err := versions.Put(uvB, []byte(rev)); err != nil { 308 return errors.Wrap(err, "failed to put version->revision") 309 } 310 311 b, err := src.CreateBucketIfNotExists(cacheRevisionName(rev)) 312 if err != nil { 313 return errors.Wrapf(err, "failed to create bucket for revision: %s", rev) 314 } 315 316 var versions *bolt.Bucket 317 if versions = revVersions[rev]; versions == nil { 318 err := cachePrefixDelete(b, cacheVersion) 319 if err != nil { 320 return err 321 } 322 versions, err = b.CreateBucket(vk) 323 if err != nil { 324 return errors.Wrapf(err, "failed to create bucket for revision versions: %s", rev) 325 } 326 revVersions[rev] = versions 327 } 328 329 key.Put(uint64(i)) 330 if err := versions.Put(key, uvB); err != nil { 331 return errors.Wrap(err, "failed to put revision->version") 332 } 333 } 334 return nil 335 }) 336 if err != nil { 337 s.logger.Println(errors.Wrap(err, "failed to cache version map")) 338 } 339 } 340 341 func (s *singleSourceCacheBolt) getVersionsFor(rev Revision) (uvs []UnpairedVersion, ok bool) { 342 err := s.viewRevBucket(rev, func(b *bolt.Bucket) error { 343 versions := cacheFindLatestValid(b, cacheVersion, s.epoch) 344 if versions == nil { 345 return nil 346 } 347 348 ok = true 349 350 var msg pb.Constraint 351 return versions.ForEach(func(_, v []byte) error { 352 if err := proto.Unmarshal(v, &msg); err != nil { 353 return err 354 } 355 uv, err := unpairedVersionFromCache(&msg) 356 if err != nil { 357 return err 358 } 359 uvs = append(uvs, uv) 360 return nil 361 }) 362 }) 363 if err != nil { 364 s.logger.Println(errors.Wrapf(err, "failed to get cached versions for revision %q", rev)) 365 return nil, false 366 } 367 return 368 } 369 370 func (s *singleSourceCacheBolt) getAllVersions() (pvs []PairedVersion, ok bool) { 371 err := s.viewSourceBucket(func(src *bolt.Bucket) error { 372 versions := cacheFindLatestValid(src, cacheVersion, s.epoch) 373 if versions == nil { 374 return nil 375 } 376 377 var msg pb.Constraint 378 return versions.ForEach(func(k, v []byte) error { 379 if err := proto.Unmarshal(k, &msg); err != nil { 380 return err 381 } 382 uv, err := unpairedVersionFromCache(&msg) 383 if err != nil { 384 return err 385 } 386 pvs = append(pvs, uv.Pair(Revision(v))) 387 ok = true 388 return nil 389 }) 390 }) 391 if err != nil { 392 s.logger.Println(errors.Wrap(err, "failed to get all cached versions")) 393 return nil, false 394 } 395 return 396 } 397 398 func (s *singleSourceCacheBolt) getRevisionFor(uv UnpairedVersion) (rev Revision, ok bool) { 399 err := s.viewSourceBucket(func(src *bolt.Bucket) error { 400 versions := cacheFindLatestValid(src, cacheVersion, s.epoch) 401 if versions == nil { 402 return nil 403 } 404 405 var msg pb.Constraint 406 uv.copyTo(&msg) 407 b, err := proto.Marshal(&msg) 408 if err != nil { 409 return errors.Wrapf(err, "failed to serialize UnpairedVersion: %#v", uv) 410 } 411 412 v := versions.Get(b) 413 if len(v) > 0 { 414 rev = Revision(v) 415 ok = true 416 } 417 return nil 418 }) 419 if err != nil { 420 s.logger.Println(errors.Wrapf(err, "failed to get cached revision for unpaired version: %v", uv)) 421 } 422 return 423 } 424 425 func (s *singleSourceCacheBolt) toRevision(v Version) (rev Revision, ok bool) { 426 switch t := v.(type) { 427 case Revision: 428 return t, true 429 case PairedVersion: 430 return t.Revision(), true 431 case UnpairedVersion: 432 return s.getRevisionFor(t) 433 default: 434 s.logger.Println(fmt.Sprintf("failed to get cached revision for version %v: unknown type %T", v, v)) 435 return "", false 436 } 437 } 438 439 func (s *singleSourceCacheBolt) toUnpaired(v Version) (uv UnpairedVersion, ok bool) { 440 const errMsg = "failed to get cached unpaired version for version: %v" 441 switch t := v.(type) { 442 case UnpairedVersion: 443 return t, true 444 case PairedVersion: 445 return t.Unpair(), true 446 case Revision: 447 err := s.viewRevBucket(t, func(b *bolt.Bucket) error { 448 versions := cacheFindLatestValid(b, cacheVersion, s.epoch) 449 if versions == nil { 450 return nil 451 } 452 453 _, v := versions.Cursor().First() 454 if len(v) == 0 { 455 return nil 456 } 457 var msg pb.Constraint 458 if err := proto.Unmarshal(v, &msg); err != nil { 459 return err 460 } 461 var err error 462 uv, err = unpairedVersionFromCache(&msg) 463 if err != nil { 464 return err 465 } 466 467 ok = true 468 return nil 469 }) 470 if err != nil { 471 s.logger.Println(errors.Wrapf(err, errMsg, v)) 472 } 473 return 474 default: 475 s.logger.Println(fmt.Sprintf(errMsg, v)) 476 return 477 } 478 } 479 480 // cacheRevisionName returns the bucket name for rev. 481 func cacheRevisionName(rev Revision) []byte { 482 name := make([]byte, 1+len(rev)) 483 name[0] = 'r' 484 copy(name[1:], string(rev)) 485 return name 486 } 487 488 // viewSourceBucket executes view with the source bucket, if it exists. 489 func (s *singleSourceCacheBolt) viewSourceBucket(view func(b *bolt.Bucket) error) error { 490 return s.db.View(func(tx *bolt.Tx) error { 491 b := tx.Bucket(s.sourceName) 492 if b == nil { 493 return nil 494 } 495 return view(b) 496 }) 497 } 498 499 // updateSourceBucket executes update (in batch) with the source bucket, creating it first if necessary. 500 func (s *singleSourceCacheBolt) updateSourceBucket(update func(b *bolt.Bucket) error) error { 501 return s.db.Batch(func(tx *bolt.Tx) error { 502 b, err := tx.CreateBucketIfNotExists(s.sourceName) 503 if err != nil { 504 return errors.Wrapf(err, "failed to create bucket: %s", s.sourceName) 505 } 506 return update(b) 507 }) 508 } 509 510 // viewRevBucket executes view with rev's bucket for this source, if it exists. 511 func (s *singleSourceCacheBolt) viewRevBucket(rev Revision, view func(b *bolt.Bucket) error) error { 512 return s.viewSourceBucket(func(src *bolt.Bucket) error { 513 b := src.Bucket(cacheRevisionName(rev)) 514 if b == nil { 515 return nil 516 } 517 return view(b) 518 }) 519 } 520 521 // updateRevBucket executes update with rev's bucket for this source, creating it first if necessary. 522 func (s *singleSourceCacheBolt) updateRevBucket(rev Revision, update func(b *bolt.Bucket) error) error { 523 return s.updateSourceBucket(func(src *bolt.Bucket) error { 524 name := cacheRevisionName(rev) 525 b, err := src.CreateBucketIfNotExists(name) 526 if err != nil { 527 return errors.Wrapf(err, "failed to create bucket: %s", name) 528 } 529 return update(b) 530 }) 531 }