github.com/golang/dep@v0.5.4/gps/source_cache_bolt_encode.go (about) 1 // Copyright 2017 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package gps 6 7 import ( 8 "encoding/binary" 9 "strings" 10 "time" 11 12 "github.com/boltdb/bolt" 13 "github.com/golang/dep/gps/internal/pb" 14 "github.com/golang/dep/gps/pkgtree" 15 "github.com/golang/protobuf/proto" 16 "github.com/jmank88/nuts" 17 "github.com/pkg/errors" 18 ) 19 20 var ( 21 cacheKeyComment = []byte("c") 22 cacheKeyConstraint = cacheKeyComment 23 cacheKeyError = []byte("e") 24 cacheKeyInputImports = []byte("m") 25 cacheKeyIgnored = []byte("i") 26 cacheKeyImport = cacheKeyIgnored 27 cacheKeyLock = []byte("l") 28 cacheKeyName = []byte("n") 29 cacheKeyOverride = []byte("o") 30 cacheKeyPTree = []byte("p") 31 cacheKeyRequired = []byte("r") 32 cacheKeyRevision = cacheKeyRequired 33 cacheKeyTestImport = []byte("t") 34 35 cacheRevision = byte('r') 36 cacheVersion = byte('v') 37 ) 38 39 // propertiesFromCache returns a new ProjectRoot and ProjectProperties with the fields from m. 40 func propertiesFromCache(m *pb.ProjectProperties) (ProjectRoot, ProjectProperties, error) { 41 ip := ProjectRoot(m.Root) 42 var pp ProjectProperties 43 pp.Source = m.Source 44 45 if m.Constraint == nil { 46 pp.Constraint = Any() 47 } else { 48 c, err := constraintFromCache(m.Constraint) 49 if err != nil { 50 return "", ProjectProperties{}, err 51 } 52 pp.Constraint = c 53 } 54 55 return ip, pp, nil 56 } 57 58 // projectPropertiesMsgs is a convenience tuple. 59 type projectPropertiesMsgs struct { 60 pp pb.ProjectProperties 61 c pb.Constraint 62 } 63 64 // copyFrom sets the ProjectPropertiesMsg fields from ip and pp. 65 func (ms *projectPropertiesMsgs) copyFrom(ip ProjectRoot, pp ProjectProperties) { 66 ms.pp.Root = string(ip) 67 ms.pp.Source = pp.Source 68 69 if pp.Constraint != nil && !IsAny(pp.Constraint) { 70 pp.Constraint.copyTo(&ms.c) 71 ms.pp.Constraint = &ms.c 72 } else { 73 ms.pp.Constraint = nil 74 } 75 } 76 77 // cachePutManifest stores a Manifest in the bolt.Bucket. 78 func cachePutManifest(b *bolt.Bucket, m Manifest) error { 79 var ppMsg projectPropertiesMsgs 80 81 constraints := m.DependencyConstraints() 82 if len(constraints) > 0 { 83 cs, err := b.CreateBucket(cacheKeyConstraint) 84 if err != nil { 85 return err 86 } 87 key := make(nuts.Key, nuts.KeyLen(uint64(len(constraints)-1))) 88 var i uint64 89 for ip, pp := range constraints { 90 ppMsg.copyFrom(ip, pp) 91 v, err := proto.Marshal(&ppMsg.pp) 92 if err != nil { 93 return err 94 } 95 key.Put(i) 96 i++ 97 if err := cs.Put(key, v); err != nil { 98 return err 99 } 100 } 101 } 102 103 rm, ok := m.(RootManifest) 104 if !ok { 105 return nil 106 } 107 108 ignored := rm.IgnoredPackages().ToSlice() 109 if len(ignored) > 0 { 110 ig, err := b.CreateBucket(cacheKeyIgnored) 111 if err != nil { 112 return err 113 } 114 key := make(nuts.Key, nuts.KeyLen(uint64(len(ignored)-1))) 115 var i uint64 116 for _, ip := range ignored { 117 key.Put(i) 118 i++ 119 if err := ig.Put(key, []byte(ip)); err != nil { 120 return err 121 } 122 } 123 } 124 125 overrides := rm.Overrides() 126 if len(overrides) > 0 { 127 ovr, err := b.CreateBucket(cacheKeyOverride) 128 if err != nil { 129 return err 130 } 131 key := make(nuts.Key, nuts.KeyLen(uint64(len(overrides)-1))) 132 var i uint64 133 for ip, pp := range overrides { 134 ppMsg.copyFrom(ip, pp) 135 v, err := proto.Marshal(&ppMsg.pp) 136 if err != nil { 137 return err 138 } 139 key.Put(i) 140 i++ 141 if err := ovr.Put(key, v); err != nil { 142 return err 143 } 144 } 145 } 146 147 required := rm.RequiredPackages() 148 if len(required) > 0 { 149 req, err := b.CreateBucket(cacheKeyRequired) 150 if err != nil { 151 return err 152 } 153 key := make(nuts.Key, nuts.KeyLen(uint64(len(required)-1))) 154 var i uint64 155 for ip, ok := range required { 156 if ok { 157 key.Put(i) 158 i++ 159 if err := req.Put(key, []byte(ip)); err != nil { 160 return err 161 } 162 } 163 } 164 } 165 166 return nil 167 } 168 169 // cacheGetManifest returns a new RootManifest with the data retrieved from the bolt.Bucket. 170 func cacheGetManifest(b *bolt.Bucket) (RootManifest, error) { 171 //TODO consider storing slice/map lens to enable calling make() with capacity 172 m := &simpleRootManifest{ 173 c: make(ProjectConstraints), 174 ovr: make(ProjectConstraints), 175 req: make(map[string]bool), 176 } 177 178 // Constraints 179 if cs := b.Bucket(cacheKeyConstraint); cs != nil { 180 var msg pb.ProjectProperties 181 err := cs.ForEach(func(_, v []byte) error { 182 if err := proto.Unmarshal(v, &msg); err != nil { 183 return err 184 } 185 ip, pp, err := propertiesFromCache(&msg) 186 if err != nil { 187 return err 188 } 189 m.c[ip] = pp 190 return nil 191 }) 192 if err != nil { 193 return nil, errors.Wrap(err, "failed to get constraints") 194 } 195 } 196 197 // Ignored 198 if ig := b.Bucket(cacheKeyIgnored); ig != nil { 199 var igslice []string 200 err := ig.ForEach(func(_, v []byte) error { 201 igslice = append(igslice, string(v)) 202 return nil 203 }) 204 m.ig = pkgtree.NewIgnoredRuleset(igslice) 205 if err != nil { 206 return nil, errors.Wrap(err, "failed to get ignored") 207 } 208 } 209 210 // Overrides 211 if os := b.Bucket(cacheKeyOverride); os != nil { 212 var msg pb.ProjectProperties 213 err := os.ForEach(func(_, v []byte) error { 214 if err := proto.Unmarshal(v, &msg); err != nil { 215 return err 216 } 217 ip, pp, err := propertiesFromCache(&msg) 218 if err != nil { 219 return err 220 } 221 m.ovr[ip] = pp 222 return nil 223 }) 224 if err != nil { 225 return nil, errors.Wrap(err, "failed to get overrides") 226 } 227 } 228 229 // Required 230 if req := b.Bucket(cacheKeyRequired); req != nil { 231 err := req.ForEach(func(_, v []byte) error { 232 m.req[string(v)] = true 233 return nil 234 }) 235 if err != nil { 236 return nil, errors.Wrap(err, "failed to get required") 237 } 238 } 239 240 return m, nil 241 } 242 243 // copyTo returns a serializable representation of lp. 244 func (lp lockedProject) copyTo(msg *pb.LockedProject, c *pb.Constraint) { 245 if lp.v == nil { 246 msg.UnpairedVersion = nil 247 } else { 248 lp.v.copyTo(c) 249 msg.UnpairedVersion = c 250 } 251 252 msg.Root = string(lp.pi.ProjectRoot) 253 msg.Source = lp.pi.Source 254 msg.Revision = string(lp.r) 255 msg.Packages = lp.pkgs 256 } 257 258 // copyLockedProjectTo hydrates pointers to serializable representations of a 259 // LockedProject with the appropriate data. 260 func copyLockedProjectTo(lp LockedProject, msg *pb.LockedProject, c *pb.Constraint) { 261 if nlp, ok := lp.(lockedProject); ok { 262 nlp.copyTo(msg, c) 263 return 264 } 265 266 v := lp.Version() 267 if v == nil { 268 msg.UnpairedVersion = nil 269 } else { 270 v.copyTo(c) 271 msg.UnpairedVersion = c 272 273 switch tv := v.(type) { 274 case Revision: 275 msg.Revision = string(tv) 276 case versionPair: 277 msg.Revision = string(tv.r) 278 } 279 } 280 281 pi := lp.Ident() 282 msg.Root = string(pi.ProjectRoot) 283 msg.Source = pi.Source 284 msg.Packages = lp.Packages() 285 } 286 287 // lockedProjectFromCache returns a new LockedProject with fields from m. 288 func lockedProjectFromCache(m *pb.LockedProject) (LockedProject, error) { 289 var uv UnpairedVersion 290 var err error 291 if m.UnpairedVersion != nil { 292 uv, err = unpairedVersionFromCache(m.UnpairedVersion) 293 if err != nil { 294 return lockedProject{}, err 295 } 296 } 297 return lockedProject{ 298 pi: ProjectIdentifier{ 299 ProjectRoot: ProjectRoot(m.Root), 300 Source: m.Source, 301 }, 302 v: uv, 303 r: Revision(m.Revision), 304 pkgs: m.Packages, 305 }, nil 306 } 307 308 // cachePutLock stores the Lock as fields in the bolt.Bucket. 309 func cachePutLock(b *bolt.Bucket, l Lock) error { 310 // Input imports, if present. 311 byt := []byte(strings.Join(l.InputImports(), "#")) 312 if err := b.Put(cacheKeyInputImports, byt); err != nil { 313 return errors.Wrap(err, "failed to put input imports") 314 } 315 316 // Projects 317 if projects := l.Projects(); len(projects) > 0 { 318 lb, err := b.CreateBucket(cacheKeyLock) 319 if err != nil { 320 return err 321 } 322 key := make(nuts.Key, nuts.KeyLen(uint64(len(projects)-1))) 323 var msg pb.LockedProject 324 var cMsg pb.Constraint 325 for i, lp := range projects { 326 copyLockedProjectTo(lp, &msg, &cMsg) 327 v, err := proto.Marshal(&msg) 328 if err != nil { 329 return err 330 } 331 key.Put(uint64(i)) 332 if err := lb.Put(key, v); err != nil { 333 return err 334 } 335 } 336 } 337 338 return nil 339 } 340 341 // cacheGetLock returns a new *safeLock with the fields retrieved from the bolt.Bucket. 342 func cacheGetLock(b *bolt.Bucket) (*safeLock, error) { 343 l := &safeLock{} 344 if ii := b.Get(cacheKeyInputImports); len(ii) > 0 { 345 l.i = strings.Split(string(ii), "#") 346 } 347 348 if locked := b.Bucket(cacheKeyLock); locked != nil { 349 var msg pb.LockedProject 350 err := locked.ForEach(func(_, v []byte) error { 351 if err := proto.Unmarshal(v, &msg); err != nil { 352 return err 353 } 354 lp, err := lockedProjectFromCache(&msg) 355 if err != nil { 356 return err 357 } 358 l.p = append(l.p, lp) 359 return nil 360 }) 361 if err != nil { 362 return nil, errors.Wrap(err, "failed to get locked projects") 363 } 364 } 365 return l, nil 366 } 367 368 // cachePutPackageOrError stores the pkgtree.PackageOrErr as fields in the bolt.Bucket. 369 // Package.ImportPath is ignored. 370 func cachePutPackageOrErr(b *bolt.Bucket, poe pkgtree.PackageOrErr) error { 371 if poe.Err != nil { 372 err := b.Put(cacheKeyError, []byte(poe.Err.Error())) 373 return errors.Wrapf(err, "failed to put error: %v", poe.Err) 374 } 375 if len(poe.P.CommentPath) > 0 { 376 err := b.Put(cacheKeyComment, []byte(poe.P.CommentPath)) 377 if err != nil { 378 return errors.Wrapf(err, "failed to put package: %v", poe.P) 379 } 380 } 381 if len(poe.P.Imports) > 0 { 382 ip, err := b.CreateBucket(cacheKeyImport) 383 if err != nil { 384 return err 385 } 386 key := make(nuts.Key, nuts.KeyLen(uint64(len(poe.P.Imports)-1))) 387 for i := range poe.P.Imports { 388 v := []byte(poe.P.Imports[i]) 389 key.Put(uint64(i)) 390 if err := ip.Put(key, v); err != nil { 391 return err 392 } 393 } 394 } 395 396 if len(poe.P.Name) > 0 { 397 err := b.Put(cacheKeyName, []byte(poe.P.Name)) 398 if err != nil { 399 return errors.Wrapf(err, "failed to put package: %v", poe.P) 400 } 401 } 402 403 if len(poe.P.TestImports) > 0 { 404 ip, err := b.CreateBucket(cacheKeyTestImport) 405 if err != nil { 406 return err 407 } 408 key := make(nuts.Key, nuts.KeyLen(uint64(len(poe.P.TestImports)-1))) 409 for i := range poe.P.TestImports { 410 v := []byte(poe.P.TestImports[i]) 411 key.Put(uint64(i)) 412 if err := ip.Put(key, v); err != nil { 413 return err 414 } 415 } 416 } 417 return nil 418 } 419 420 // cacheGetPackageOrErr returns a new pkgtree.PackageOrErr with fields retrieved 421 // from the bolt.Bucket. 422 func cacheGetPackageOrErr(b *bolt.Bucket) (pkgtree.PackageOrErr, error) { 423 if v := b.Get(cacheKeyError); len(v) > 0 { 424 return pkgtree.PackageOrErr{ 425 Err: errors.New(string(v)), 426 }, nil 427 } 428 429 var p pkgtree.Package 430 p.CommentPath = string(b.Get(cacheKeyComment)) 431 if ip := b.Bucket(cacheKeyImport); ip != nil { 432 err := ip.ForEach(func(_, v []byte) error { 433 p.Imports = append(p.Imports, string(v)) 434 return nil 435 }) 436 if err != nil { 437 return pkgtree.PackageOrErr{}, err 438 } 439 } 440 p.Name = string(b.Get(cacheKeyName)) 441 if tip := b.Bucket(cacheKeyTestImport); tip != nil { 442 err := tip.ForEach(func(_, v []byte) error { 443 p.TestImports = append(p.TestImports, string(v)) 444 return nil 445 }) 446 if err != nil { 447 return pkgtree.PackageOrErr{}, err 448 } 449 } 450 return pkgtree.PackageOrErr{P: p}, nil 451 } 452 453 // cacheTimestampedKey returns a prefixed key with a trailing timestamp. 454 func cacheTimestampedKey(pre byte, t time.Time) []byte { 455 b := make([]byte, 9) 456 b[0] = pre 457 binary.BigEndian.PutUint64(b[1:], uint64(t.Unix())) 458 return b 459 } 460 461 // boltTxOrBucket is a minimal interface satisfied by bolt.Tx and bolt.Bucket. 462 type boltTxOrBucket interface { 463 Cursor() *bolt.Cursor 464 DeleteBucket([]byte) error 465 Bucket([]byte) *bolt.Bucket 466 } 467 468 // cachePrefixDelete prefix scans and deletes each bucket. 469 func cachePrefixDelete(tob boltTxOrBucket, pre byte) error { 470 c := tob.Cursor() 471 for k, _ := c.Seek([]byte{pre}); len(k) > 0 && k[0] == pre; k, _ = c.Next() { 472 if err := tob.DeleteBucket(k); err != nil { 473 return errors.Wrapf(err, "failed to delete bucket: %s", k) 474 } 475 } 476 return nil 477 } 478 479 // cacheFindLatestValid prefix scans for the latest bucket which is timestamped >= epoch, 480 // or returns nil if none exists. 481 func cacheFindLatestValid(tob boltTxOrBucket, pre byte, epoch int64) *bolt.Bucket { 482 c := tob.Cursor() 483 var latest []byte 484 for k, _ := c.Seek([]byte{pre}); len(k) > 0 && k[0] == pre; k, _ = c.Next() { 485 latest = k 486 } 487 if latest == nil { 488 return nil 489 } 490 ts := latest[1:] 491 if len(ts) != 8 { 492 return nil 493 } 494 if int64(binary.BigEndian.Uint64(ts)) < epoch { 495 return nil 496 } 497 return tob.Bucket(latest) 498 }