github.com/golang/dep@v0.5.4/gps/source_cache_bolt.go (about)

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package gps
     6  
     7  import (
     8  	"fmt"
     9  	"log"
    10  	"os"
    11  	"path"
    12  	"path/filepath"
    13  	"strings"
    14  	"time"
    15  
    16  	"github.com/boltdb/bolt"
    17  	"github.com/golang/dep/gps/internal/pb"
    18  	"github.com/golang/dep/gps/pkgtree"
    19  	"github.com/golang/protobuf/proto"
    20  	"github.com/jmank88/nuts"
    21  	"github.com/pkg/errors"
    22  )
    23  
    24  // boltCacheFilename is a versioned filename for the bolt cache. The version
    25  // must be incremented whenever incompatible changes are made.
    26  const boltCacheFilename = "bolt-v1.db"
    27  
    28  // boltCache manages a bolt.DB cache and provides singleSourceCaches.
    29  type boltCache struct {
    30  	db     *bolt.DB
    31  	epoch  int64       // getters will not return values older than this unix timestamp
    32  	logger *log.Logger // info logging
    33  }
    34  
    35  // newBoltCache returns a new boltCache backed by a BoltDB file under the cache directory.
    36  func newBoltCache(cd string, epoch int64, logger *log.Logger) (*boltCache, error) {
    37  	path := filepath.Join(cd, boltCacheFilename)
    38  	dir := filepath.Dir(path)
    39  	if fi, err := os.Stat(dir); os.IsNotExist(err) {
    40  		if err := os.MkdirAll(dir, os.ModeDir|os.ModePerm); err != nil {
    41  			return nil, errors.Wrapf(err, "failed to create source cache directory: %s", dir)
    42  		}
    43  	} else if err != nil {
    44  		return nil, errors.Wrapf(err, "failed to check source cache directory: %s", dir)
    45  	} else if !fi.IsDir() {
    46  		return nil, errors.Wrapf(err, "source cache path is not directory: %s", dir)
    47  	}
    48  	db, err := bolt.Open(path, 0600, &bolt.Options{Timeout: 1 * time.Second})
    49  	if err != nil {
    50  		return nil, errors.Wrapf(err, "failed to open BoltDB cache file %q", path)
    51  	}
    52  	return &boltCache{
    53  		db:     db,
    54  		epoch:  epoch,
    55  		logger: logger,
    56  	}, nil
    57  }
    58  
    59  // newSingleSourceCache returns a new singleSourceCache for pi.
    60  func (c *boltCache) newSingleSourceCache(pi ProjectIdentifier) singleSourceCache {
    61  	return &singleSourceCacheBolt{
    62  		boltCache:  c,
    63  		sourceName: []byte(pi.normalizedSource()),
    64  	}
    65  }
    66  
    67  // close releases all cache resources.
    68  func (c *boltCache) close() error {
    69  	return errors.Wrapf(c.db.Close(), "error closing Bolt database %q", c.db.String())
    70  }
    71  
    72  // singleSourceCacheBolt implements a singleSourceCache backed by a persistent BoltDB file.
    73  // Version mappings are timestamped, and the `epoch` field limits the age of returned values.
    74  // Database access methods are safe for concurrent use.
    75  //
    76  // Implementation:
    77  //
    78  // Each source has a top-level bucket containing sub-buckets for (1) versions and (2) revisions.
    79  //
    80  // 1) Versions buckets hold version keys with revision values:
    81  //
    82  //	Bucket: "v<timestamp>"
    83  //	Keys: Unpaired Versions serialized via ConstraintMsg
    84  //	Values: "<revision>"
    85  //
    86  // 2) Revision buckets hold (a) manifest and lock data for various ProjectAnalyzers,
    87  // (b) package trees, and (c) version lists.
    88  //
    89  //	Bucket: "r<revision>"
    90  //
    91  // a) Manifest and Lock info are stored in buckets derived from ProjectAnalyzer.Info:
    92  //
    93  //	Sub-Bucket: "<name>.<version>m", "<name>.<version>l"
    94  //	Keys/Values: Manifest or Lock fields
    95  //
    96  // b) Package tree buckets contain package import path keys and package-or-error buckets:
    97  //
    98  //	Sub-Bucket: "p"
    99  //	Sub-Bucket: "<import_path>"
   100  //	Key/Values: PackageOrErr fields
   101  //
   102  // c) Revision-versions buckets contain lists of version values:
   103  //
   104  //	Sub-Bucket: "v<timestamp>"
   105  //	Keys: "<sequence_number>"
   106  //	Values: Unpaired Versions serialized via ConstraintMsg
   107  type singleSourceCacheBolt struct {
   108  	*boltCache
   109  	sourceName []byte
   110  }
   111  
   112  func (s *singleSourceCacheBolt) setManifestAndLock(rev Revision, ai ProjectAnalyzerInfo, m Manifest, l Lock) {
   113  	err := s.updateRevBucket(rev, func(b *bolt.Bucket) error {
   114  		info := ai.String()
   115  		name := make([]byte, len(info)+1)
   116  		copy(name, info)
   117  		name[len(info)] = 'm'
   118  
   119  		if b.Bucket(name) != nil {
   120  			if err := b.DeleteBucket(name); err != nil {
   121  				return err
   122  			}
   123  		}
   124  
   125  		// Manifest
   126  		mb, err := b.CreateBucket(name)
   127  		if err != nil {
   128  			return err
   129  		}
   130  		if err := cachePutManifest(mb, m); err != nil {
   131  			return errors.Wrap(err, "failed to put manifest")
   132  		}
   133  		if l == nil {
   134  			return nil
   135  		}
   136  
   137  		// Lock
   138  		name[len(info)] = 'l'
   139  		if b.Bucket(name) != nil {
   140  			if err := b.DeleteBucket(name); err != nil {
   141  				return err
   142  			}
   143  		}
   144  		lb, err := b.CreateBucket(name)
   145  		if err != nil {
   146  			return err
   147  		}
   148  		return errors.Wrap(cachePutLock(lb, l), "failed to put lock")
   149  	})
   150  	if err != nil {
   151  		s.logger.Println(errors.Wrapf(err, "failed to cache manifest/lock for revision %q, analyzer: %v", rev, ai))
   152  	}
   153  }
   154  
   155  func (s *singleSourceCacheBolt) getManifestAndLock(rev Revision, ai ProjectAnalyzerInfo) (m Manifest, l Lock, ok bool) {
   156  	err := s.viewRevBucket(rev, func(b *bolt.Bucket) error {
   157  		info := ai.String()
   158  		name := make([]byte, len(info)+1)
   159  		copy(name, info)
   160  		name[len(info)] = 'm'
   161  
   162  		// Manifest
   163  		mb := b.Bucket(name)
   164  		if mb == nil {
   165  			return nil
   166  		}
   167  		var err error
   168  		m, err = cacheGetManifest(mb)
   169  		if err != nil {
   170  			return errors.Wrap(err, "failed to get manifest")
   171  		}
   172  
   173  		// Lock
   174  		name[len(info)] = 'l'
   175  		lb := b.Bucket(name)
   176  		if lb == nil {
   177  			ok = true
   178  			return nil
   179  		}
   180  		l, err = cacheGetLock(lb)
   181  		if err != nil {
   182  			return errors.Wrap(err, "failed to get lock")
   183  		}
   184  
   185  		ok = true
   186  		return nil
   187  	})
   188  	if err != nil {
   189  		s.logger.Println(errors.Wrapf(err, "failed to get cached manifest/lock for revision %q, analyzer: %v", rev, ai))
   190  	}
   191  	return
   192  }
   193  
   194  func (s *singleSourceCacheBolt) setPackageTree(rev Revision, ptree pkgtree.PackageTree) {
   195  	err := s.updateRevBucket(rev, func(b *bolt.Bucket) error {
   196  		if b.Bucket(cacheKeyPTree) != nil {
   197  			if err := b.DeleteBucket(cacheKeyPTree); err != nil {
   198  				return err
   199  			}
   200  		}
   201  		ptrees, err := b.CreateBucket(cacheKeyPTree)
   202  		if err != nil {
   203  			return err
   204  		}
   205  
   206  		root := string(ptree.ImportRoot)
   207  		for ip, poe := range ptree.Packages {
   208  			// Stored by relative import path.
   209  			rip := strings.TrimPrefix(ip, root)
   210  			if rip == "" {
   211  				rip = "/"
   212  			}
   213  			pb, err := ptrees.CreateBucket([]byte(rip))
   214  			if err != nil {
   215  				return err
   216  			}
   217  
   218  			if err := cachePutPackageOrErr(pb, poe); err != nil {
   219  				return err
   220  			}
   221  		}
   222  		return nil
   223  	})
   224  	if err != nil {
   225  		s.logger.Println(errors.Wrapf(err, "failed to cache package tree for revision %q", rev))
   226  	}
   227  }
   228  
   229  func (s *singleSourceCacheBolt) getPackageTree(rev Revision, pr ProjectRoot) (ptree pkgtree.PackageTree, ok bool) {
   230  	err := s.viewRevBucket(rev, func(b *bolt.Bucket) error {
   231  		ptrees := b.Bucket(cacheKeyPTree)
   232  		if ptrees == nil {
   233  			return nil
   234  		}
   235  
   236  		pkgs := make(map[string]pkgtree.PackageOrErr)
   237  		err := ptrees.ForEach(func(rip, _ []byte) error {
   238  			poe, err := cacheGetPackageOrErr(ptrees.Bucket(rip))
   239  			if err != nil {
   240  				return err
   241  			}
   242  			srip := string(rip)
   243  			if srip == "/" {
   244  				srip = ""
   245  			}
   246  			// Return full import paths.
   247  			ip := path.Join(string(pr), srip)
   248  			if poe.Err == nil {
   249  				poe.P.ImportPath = ip
   250  			}
   251  			pkgs[ip] = poe
   252  			return nil
   253  		})
   254  		if err != nil {
   255  			return err
   256  		}
   257  		ptree.ImportRoot = string(pr)
   258  		ptree.Packages = pkgs
   259  		ok = true
   260  		return nil
   261  	})
   262  	if err != nil {
   263  		s.logger.Println(errors.Wrapf(err, "failed to get cached package tree for revision %q", rev))
   264  	}
   265  	return
   266  }
   267  
   268  func (s *singleSourceCacheBolt) markRevisionExists(rev Revision) {
   269  	err := s.updateRevBucket(rev, func(versions *bolt.Bucket) error {
   270  		return nil
   271  	})
   272  	if err != nil {
   273  		s.logger.Println(errors.Wrapf(err, "failed to mark revision %q in cache", rev))
   274  	}
   275  }
   276  
   277  func (s *singleSourceCacheBolt) setVersionMap(pvs []PairedVersion) {
   278  	err := s.updateSourceBucket(func(src *bolt.Bucket) error {
   279  		if err := cachePrefixDelete(src, cacheVersion); err != nil {
   280  			return err
   281  		}
   282  		vk := cacheTimestampedKey(cacheVersion, time.Now())
   283  		versions, err := src.CreateBucket(vk)
   284  		if err != nil {
   285  			return err
   286  		}
   287  
   288  		c := src.Cursor()
   289  		for k, _ := c.Seek(cacheKeyRevision); len(k) > 0 && k[0] == cacheRevision; k, _ = c.Next() {
   290  			rb := src.Bucket(k)
   291  			if err := cachePrefixDelete(rb, cacheVersion); err != nil {
   292  				return err
   293  			}
   294  		}
   295  
   296  		revVersions := make(map[Revision]*bolt.Bucket)
   297  		key := make(nuts.Key, nuts.KeyLen(uint64(len(pvs)-1)))
   298  		var msg pb.Constraint
   299  		for i, pv := range pvs {
   300  			uv, rev := pv.Unpair(), pv.Revision()
   301  			uv.copyTo(&msg)
   302  			uvB, err := proto.Marshal(&msg)
   303  			if err != nil {
   304  				return errors.Wrapf(err, "failed to serialize UnpairedVersion: %#v", uv)
   305  			}
   306  
   307  			if err := versions.Put(uvB, []byte(rev)); err != nil {
   308  				return errors.Wrap(err, "failed to put version->revision")
   309  			}
   310  
   311  			b, err := src.CreateBucketIfNotExists(cacheRevisionName(rev))
   312  			if err != nil {
   313  				return errors.Wrapf(err, "failed to create bucket for revision: %s", rev)
   314  			}
   315  
   316  			var versions *bolt.Bucket
   317  			if versions = revVersions[rev]; versions == nil {
   318  				err := cachePrefixDelete(b, cacheVersion)
   319  				if err != nil {
   320  					return err
   321  				}
   322  				versions, err = b.CreateBucket(vk)
   323  				if err != nil {
   324  					return errors.Wrapf(err, "failed to create bucket for revision versions: %s", rev)
   325  				}
   326  				revVersions[rev] = versions
   327  			}
   328  
   329  			key.Put(uint64(i))
   330  			if err := versions.Put(key, uvB); err != nil {
   331  				return errors.Wrap(err, "failed to put revision->version")
   332  			}
   333  		}
   334  		return nil
   335  	})
   336  	if err != nil {
   337  		s.logger.Println(errors.Wrap(err, "failed to cache version map"))
   338  	}
   339  }
   340  
   341  func (s *singleSourceCacheBolt) getVersionsFor(rev Revision) (uvs []UnpairedVersion, ok bool) {
   342  	err := s.viewRevBucket(rev, func(b *bolt.Bucket) error {
   343  		versions := cacheFindLatestValid(b, cacheVersion, s.epoch)
   344  		if versions == nil {
   345  			return nil
   346  		}
   347  
   348  		ok = true
   349  
   350  		var msg pb.Constraint
   351  		return versions.ForEach(func(_, v []byte) error {
   352  			if err := proto.Unmarshal(v, &msg); err != nil {
   353  				return err
   354  			}
   355  			uv, err := unpairedVersionFromCache(&msg)
   356  			if err != nil {
   357  				return err
   358  			}
   359  			uvs = append(uvs, uv)
   360  			return nil
   361  		})
   362  	})
   363  	if err != nil {
   364  		s.logger.Println(errors.Wrapf(err, "failed to get cached versions for revision %q", rev))
   365  		return nil, false
   366  	}
   367  	return
   368  }
   369  
   370  func (s *singleSourceCacheBolt) getAllVersions() (pvs []PairedVersion, ok bool) {
   371  	err := s.viewSourceBucket(func(src *bolt.Bucket) error {
   372  		versions := cacheFindLatestValid(src, cacheVersion, s.epoch)
   373  		if versions == nil {
   374  			return nil
   375  		}
   376  
   377  		var msg pb.Constraint
   378  		return versions.ForEach(func(k, v []byte) error {
   379  			if err := proto.Unmarshal(k, &msg); err != nil {
   380  				return err
   381  			}
   382  			uv, err := unpairedVersionFromCache(&msg)
   383  			if err != nil {
   384  				return err
   385  			}
   386  			pvs = append(pvs, uv.Pair(Revision(v)))
   387  			ok = true
   388  			return nil
   389  		})
   390  	})
   391  	if err != nil {
   392  		s.logger.Println(errors.Wrap(err, "failed to get all cached versions"))
   393  		return nil, false
   394  	}
   395  	return
   396  }
   397  
   398  func (s *singleSourceCacheBolt) getRevisionFor(uv UnpairedVersion) (rev Revision, ok bool) {
   399  	err := s.viewSourceBucket(func(src *bolt.Bucket) error {
   400  		versions := cacheFindLatestValid(src, cacheVersion, s.epoch)
   401  		if versions == nil {
   402  			return nil
   403  		}
   404  
   405  		var msg pb.Constraint
   406  		uv.copyTo(&msg)
   407  		b, err := proto.Marshal(&msg)
   408  		if err != nil {
   409  			return errors.Wrapf(err, "failed to serialize UnpairedVersion: %#v", uv)
   410  		}
   411  
   412  		v := versions.Get(b)
   413  		if len(v) > 0 {
   414  			rev = Revision(v)
   415  			ok = true
   416  		}
   417  		return nil
   418  	})
   419  	if err != nil {
   420  		s.logger.Println(errors.Wrapf(err, "failed to get cached revision for unpaired version: %v", uv))
   421  	}
   422  	return
   423  }
   424  
   425  func (s *singleSourceCacheBolt) toRevision(v Version) (rev Revision, ok bool) {
   426  	switch t := v.(type) {
   427  	case Revision:
   428  		return t, true
   429  	case PairedVersion:
   430  		return t.Revision(), true
   431  	case UnpairedVersion:
   432  		return s.getRevisionFor(t)
   433  	default:
   434  		s.logger.Println(fmt.Sprintf("failed to get cached revision for version %v: unknown type %T", v, v))
   435  		return "", false
   436  	}
   437  }
   438  
   439  func (s *singleSourceCacheBolt) toUnpaired(v Version) (uv UnpairedVersion, ok bool) {
   440  	const errMsg = "failed to get cached unpaired version for version: %v"
   441  	switch t := v.(type) {
   442  	case UnpairedVersion:
   443  		return t, true
   444  	case PairedVersion:
   445  		return t.Unpair(), true
   446  	case Revision:
   447  		err := s.viewRevBucket(t, func(b *bolt.Bucket) error {
   448  			versions := cacheFindLatestValid(b, cacheVersion, s.epoch)
   449  			if versions == nil {
   450  				return nil
   451  			}
   452  
   453  			_, v := versions.Cursor().First()
   454  			if len(v) == 0 {
   455  				return nil
   456  			}
   457  			var msg pb.Constraint
   458  			if err := proto.Unmarshal(v, &msg); err != nil {
   459  				return err
   460  			}
   461  			var err error
   462  			uv, err = unpairedVersionFromCache(&msg)
   463  			if err != nil {
   464  				return err
   465  			}
   466  
   467  			ok = true
   468  			return nil
   469  		})
   470  		if err != nil {
   471  			s.logger.Println(errors.Wrapf(err, errMsg, v))
   472  		}
   473  		return
   474  	default:
   475  		s.logger.Println(fmt.Sprintf(errMsg, v))
   476  		return
   477  	}
   478  }
   479  
   480  // cacheRevisionName returns the bucket name for rev.
   481  func cacheRevisionName(rev Revision) []byte {
   482  	name := make([]byte, 1+len(rev))
   483  	name[0] = 'r'
   484  	copy(name[1:], string(rev))
   485  	return name
   486  }
   487  
   488  // viewSourceBucket executes view with the source bucket, if it exists.
   489  func (s *singleSourceCacheBolt) viewSourceBucket(view func(b *bolt.Bucket) error) error {
   490  	return s.db.View(func(tx *bolt.Tx) error {
   491  		b := tx.Bucket(s.sourceName)
   492  		if b == nil {
   493  			return nil
   494  		}
   495  		return view(b)
   496  	})
   497  }
   498  
   499  // updateSourceBucket executes update (in batch) with the source bucket, creating it first if necessary.
   500  func (s *singleSourceCacheBolt) updateSourceBucket(update func(b *bolt.Bucket) error) error {
   501  	return s.db.Batch(func(tx *bolt.Tx) error {
   502  		b, err := tx.CreateBucketIfNotExists(s.sourceName)
   503  		if err != nil {
   504  			return errors.Wrapf(err, "failed to create bucket: %s", s.sourceName)
   505  		}
   506  		return update(b)
   507  	})
   508  }
   509  
   510  // viewRevBucket executes view with rev's bucket for this source, if it exists.
   511  func (s *singleSourceCacheBolt) viewRevBucket(rev Revision, view func(b *bolt.Bucket) error) error {
   512  	return s.viewSourceBucket(func(src *bolt.Bucket) error {
   513  		b := src.Bucket(cacheRevisionName(rev))
   514  		if b == nil {
   515  			return nil
   516  		}
   517  		return view(b)
   518  	})
   519  }
   520  
   521  // updateRevBucket executes update with rev's bucket for this source, creating it first if necessary.
   522  func (s *singleSourceCacheBolt) updateRevBucket(rev Revision, update func(b *bolt.Bucket) error) error {
   523  	return s.updateSourceBucket(func(src *bolt.Bucket) error {
   524  		name := cacheRevisionName(rev)
   525  		b, err := src.CreateBucketIfNotExists(name)
   526  		if err != nil {
   527  			return errors.Wrapf(err, "failed to create bucket: %s", name)
   528  		}
   529  		return update(b)
   530  	})
   531  }