github.com/golang/dep@v0.5.4/gps/source_cache_bolt_encode.go (about)

     1  // Copyright 2017 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package gps
     6  
     7  import (
     8  	"encoding/binary"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/boltdb/bolt"
    13  	"github.com/golang/dep/gps/internal/pb"
    14  	"github.com/golang/dep/gps/pkgtree"
    15  	"github.com/golang/protobuf/proto"
    16  	"github.com/jmank88/nuts"
    17  	"github.com/pkg/errors"
    18  )
    19  
    20  var (
    21  	cacheKeyComment      = []byte("c")
    22  	cacheKeyConstraint   = cacheKeyComment
    23  	cacheKeyError        = []byte("e")
    24  	cacheKeyInputImports = []byte("m")
    25  	cacheKeyIgnored      = []byte("i")
    26  	cacheKeyImport       = cacheKeyIgnored
    27  	cacheKeyLock         = []byte("l")
    28  	cacheKeyName         = []byte("n")
    29  	cacheKeyOverride     = []byte("o")
    30  	cacheKeyPTree        = []byte("p")
    31  	cacheKeyRequired     = []byte("r")
    32  	cacheKeyRevision     = cacheKeyRequired
    33  	cacheKeyTestImport   = []byte("t")
    34  
    35  	cacheRevision = byte('r')
    36  	cacheVersion  = byte('v')
    37  )
    38  
    39  // propertiesFromCache returns a new ProjectRoot and ProjectProperties with the fields from m.
    40  func propertiesFromCache(m *pb.ProjectProperties) (ProjectRoot, ProjectProperties, error) {
    41  	ip := ProjectRoot(m.Root)
    42  	var pp ProjectProperties
    43  	pp.Source = m.Source
    44  
    45  	if m.Constraint == nil {
    46  		pp.Constraint = Any()
    47  	} else {
    48  		c, err := constraintFromCache(m.Constraint)
    49  		if err != nil {
    50  			return "", ProjectProperties{}, err
    51  		}
    52  		pp.Constraint = c
    53  	}
    54  
    55  	return ip, pp, nil
    56  }
    57  
    58  // projectPropertiesMsgs is a convenience tuple.
    59  type projectPropertiesMsgs struct {
    60  	pp pb.ProjectProperties
    61  	c  pb.Constraint
    62  }
    63  
    64  // copyFrom sets the ProjectPropertiesMsg fields from ip and pp.
    65  func (ms *projectPropertiesMsgs) copyFrom(ip ProjectRoot, pp ProjectProperties) {
    66  	ms.pp.Root = string(ip)
    67  	ms.pp.Source = pp.Source
    68  
    69  	if pp.Constraint != nil && !IsAny(pp.Constraint) {
    70  		pp.Constraint.copyTo(&ms.c)
    71  		ms.pp.Constraint = &ms.c
    72  	} else {
    73  		ms.pp.Constraint = nil
    74  	}
    75  }
    76  
    77  // cachePutManifest stores a Manifest in the bolt.Bucket.
    78  func cachePutManifest(b *bolt.Bucket, m Manifest) error {
    79  	var ppMsg projectPropertiesMsgs
    80  
    81  	constraints := m.DependencyConstraints()
    82  	if len(constraints) > 0 {
    83  		cs, err := b.CreateBucket(cacheKeyConstraint)
    84  		if err != nil {
    85  			return err
    86  		}
    87  		key := make(nuts.Key, nuts.KeyLen(uint64(len(constraints)-1)))
    88  		var i uint64
    89  		for ip, pp := range constraints {
    90  			ppMsg.copyFrom(ip, pp)
    91  			v, err := proto.Marshal(&ppMsg.pp)
    92  			if err != nil {
    93  				return err
    94  			}
    95  			key.Put(i)
    96  			i++
    97  			if err := cs.Put(key, v); err != nil {
    98  				return err
    99  			}
   100  		}
   101  	}
   102  
   103  	rm, ok := m.(RootManifest)
   104  	if !ok {
   105  		return nil
   106  	}
   107  
   108  	ignored := rm.IgnoredPackages().ToSlice()
   109  	if len(ignored) > 0 {
   110  		ig, err := b.CreateBucket(cacheKeyIgnored)
   111  		if err != nil {
   112  			return err
   113  		}
   114  		key := make(nuts.Key, nuts.KeyLen(uint64(len(ignored)-1)))
   115  		var i uint64
   116  		for _, ip := range ignored {
   117  			key.Put(i)
   118  			i++
   119  			if err := ig.Put(key, []byte(ip)); err != nil {
   120  				return err
   121  			}
   122  		}
   123  	}
   124  
   125  	overrides := rm.Overrides()
   126  	if len(overrides) > 0 {
   127  		ovr, err := b.CreateBucket(cacheKeyOverride)
   128  		if err != nil {
   129  			return err
   130  		}
   131  		key := make(nuts.Key, nuts.KeyLen(uint64(len(overrides)-1)))
   132  		var i uint64
   133  		for ip, pp := range overrides {
   134  			ppMsg.copyFrom(ip, pp)
   135  			v, err := proto.Marshal(&ppMsg.pp)
   136  			if err != nil {
   137  				return err
   138  			}
   139  			key.Put(i)
   140  			i++
   141  			if err := ovr.Put(key, v); err != nil {
   142  				return err
   143  			}
   144  		}
   145  	}
   146  
   147  	required := rm.RequiredPackages()
   148  	if len(required) > 0 {
   149  		req, err := b.CreateBucket(cacheKeyRequired)
   150  		if err != nil {
   151  			return err
   152  		}
   153  		key := make(nuts.Key, nuts.KeyLen(uint64(len(required)-1)))
   154  		var i uint64
   155  		for ip, ok := range required {
   156  			if ok {
   157  				key.Put(i)
   158  				i++
   159  				if err := req.Put(key, []byte(ip)); err != nil {
   160  					return err
   161  				}
   162  			}
   163  		}
   164  	}
   165  
   166  	return nil
   167  }
   168  
   169  // cacheGetManifest returns a new RootManifest with the data retrieved from the bolt.Bucket.
   170  func cacheGetManifest(b *bolt.Bucket) (RootManifest, error) {
   171  	//TODO consider storing slice/map lens to enable calling make() with capacity
   172  	m := &simpleRootManifest{
   173  		c:   make(ProjectConstraints),
   174  		ovr: make(ProjectConstraints),
   175  		req: make(map[string]bool),
   176  	}
   177  
   178  	// Constraints
   179  	if cs := b.Bucket(cacheKeyConstraint); cs != nil {
   180  		var msg pb.ProjectProperties
   181  		err := cs.ForEach(func(_, v []byte) error {
   182  			if err := proto.Unmarshal(v, &msg); err != nil {
   183  				return err
   184  			}
   185  			ip, pp, err := propertiesFromCache(&msg)
   186  			if err != nil {
   187  				return err
   188  			}
   189  			m.c[ip] = pp
   190  			return nil
   191  		})
   192  		if err != nil {
   193  			return nil, errors.Wrap(err, "failed to get constraints")
   194  		}
   195  	}
   196  
   197  	// Ignored
   198  	if ig := b.Bucket(cacheKeyIgnored); ig != nil {
   199  		var igslice []string
   200  		err := ig.ForEach(func(_, v []byte) error {
   201  			igslice = append(igslice, string(v))
   202  			return nil
   203  		})
   204  		m.ig = pkgtree.NewIgnoredRuleset(igslice)
   205  		if err != nil {
   206  			return nil, errors.Wrap(err, "failed to get ignored")
   207  		}
   208  	}
   209  
   210  	// Overrides
   211  	if os := b.Bucket(cacheKeyOverride); os != nil {
   212  		var msg pb.ProjectProperties
   213  		err := os.ForEach(func(_, v []byte) error {
   214  			if err := proto.Unmarshal(v, &msg); err != nil {
   215  				return err
   216  			}
   217  			ip, pp, err := propertiesFromCache(&msg)
   218  			if err != nil {
   219  				return err
   220  			}
   221  			m.ovr[ip] = pp
   222  			return nil
   223  		})
   224  		if err != nil {
   225  			return nil, errors.Wrap(err, "failed to get overrides")
   226  		}
   227  	}
   228  
   229  	// Required
   230  	if req := b.Bucket(cacheKeyRequired); req != nil {
   231  		err := req.ForEach(func(_, v []byte) error {
   232  			m.req[string(v)] = true
   233  			return nil
   234  		})
   235  		if err != nil {
   236  			return nil, errors.Wrap(err, "failed to get required")
   237  		}
   238  	}
   239  
   240  	return m, nil
   241  }
   242  
   243  // copyTo returns a serializable representation of lp.
   244  func (lp lockedProject) copyTo(msg *pb.LockedProject, c *pb.Constraint) {
   245  	if lp.v == nil {
   246  		msg.UnpairedVersion = nil
   247  	} else {
   248  		lp.v.copyTo(c)
   249  		msg.UnpairedVersion = c
   250  	}
   251  
   252  	msg.Root = string(lp.pi.ProjectRoot)
   253  	msg.Source = lp.pi.Source
   254  	msg.Revision = string(lp.r)
   255  	msg.Packages = lp.pkgs
   256  }
   257  
   258  // copyLockedProjectTo hydrates pointers to serializable representations of a
   259  // LockedProject with the appropriate data.
   260  func copyLockedProjectTo(lp LockedProject, msg *pb.LockedProject, c *pb.Constraint) {
   261  	if nlp, ok := lp.(lockedProject); ok {
   262  		nlp.copyTo(msg, c)
   263  		return
   264  	}
   265  
   266  	v := lp.Version()
   267  	if v == nil {
   268  		msg.UnpairedVersion = nil
   269  	} else {
   270  		v.copyTo(c)
   271  		msg.UnpairedVersion = c
   272  
   273  		switch tv := v.(type) {
   274  		case Revision:
   275  			msg.Revision = string(tv)
   276  		case versionPair:
   277  			msg.Revision = string(tv.r)
   278  		}
   279  	}
   280  
   281  	pi := lp.Ident()
   282  	msg.Root = string(pi.ProjectRoot)
   283  	msg.Source = pi.Source
   284  	msg.Packages = lp.Packages()
   285  }
   286  
   287  // lockedProjectFromCache returns a new LockedProject with fields from m.
   288  func lockedProjectFromCache(m *pb.LockedProject) (LockedProject, error) {
   289  	var uv UnpairedVersion
   290  	var err error
   291  	if m.UnpairedVersion != nil {
   292  		uv, err = unpairedVersionFromCache(m.UnpairedVersion)
   293  		if err != nil {
   294  			return lockedProject{}, err
   295  		}
   296  	}
   297  	return lockedProject{
   298  		pi: ProjectIdentifier{
   299  			ProjectRoot: ProjectRoot(m.Root),
   300  			Source:      m.Source,
   301  		},
   302  		v:    uv,
   303  		r:    Revision(m.Revision),
   304  		pkgs: m.Packages,
   305  	}, nil
   306  }
   307  
   308  // cachePutLock stores the Lock as fields in the bolt.Bucket.
   309  func cachePutLock(b *bolt.Bucket, l Lock) error {
   310  	// Input imports, if present.
   311  	byt := []byte(strings.Join(l.InputImports(), "#"))
   312  	if err := b.Put(cacheKeyInputImports, byt); err != nil {
   313  		return errors.Wrap(err, "failed to put input imports")
   314  	}
   315  
   316  	// Projects
   317  	if projects := l.Projects(); len(projects) > 0 {
   318  		lb, err := b.CreateBucket(cacheKeyLock)
   319  		if err != nil {
   320  			return err
   321  		}
   322  		key := make(nuts.Key, nuts.KeyLen(uint64(len(projects)-1)))
   323  		var msg pb.LockedProject
   324  		var cMsg pb.Constraint
   325  		for i, lp := range projects {
   326  			copyLockedProjectTo(lp, &msg, &cMsg)
   327  			v, err := proto.Marshal(&msg)
   328  			if err != nil {
   329  				return err
   330  			}
   331  			key.Put(uint64(i))
   332  			if err := lb.Put(key, v); err != nil {
   333  				return err
   334  			}
   335  		}
   336  	}
   337  
   338  	return nil
   339  }
   340  
   341  // cacheGetLock returns a new *safeLock with the fields retrieved from the bolt.Bucket.
   342  func cacheGetLock(b *bolt.Bucket) (*safeLock, error) {
   343  	l := &safeLock{}
   344  	if ii := b.Get(cacheKeyInputImports); len(ii) > 0 {
   345  		l.i = strings.Split(string(ii), "#")
   346  	}
   347  
   348  	if locked := b.Bucket(cacheKeyLock); locked != nil {
   349  		var msg pb.LockedProject
   350  		err := locked.ForEach(func(_, v []byte) error {
   351  			if err := proto.Unmarshal(v, &msg); err != nil {
   352  				return err
   353  			}
   354  			lp, err := lockedProjectFromCache(&msg)
   355  			if err != nil {
   356  				return err
   357  			}
   358  			l.p = append(l.p, lp)
   359  			return nil
   360  		})
   361  		if err != nil {
   362  			return nil, errors.Wrap(err, "failed to get locked projects")
   363  		}
   364  	}
   365  	return l, nil
   366  }
   367  
   368  // cachePutPackageOrError stores the pkgtree.PackageOrErr as fields in the bolt.Bucket.
   369  // Package.ImportPath is ignored.
   370  func cachePutPackageOrErr(b *bolt.Bucket, poe pkgtree.PackageOrErr) error {
   371  	if poe.Err != nil {
   372  		err := b.Put(cacheKeyError, []byte(poe.Err.Error()))
   373  		return errors.Wrapf(err, "failed to put error: %v", poe.Err)
   374  	}
   375  	if len(poe.P.CommentPath) > 0 {
   376  		err := b.Put(cacheKeyComment, []byte(poe.P.CommentPath))
   377  		if err != nil {
   378  			return errors.Wrapf(err, "failed to put package: %v", poe.P)
   379  		}
   380  	}
   381  	if len(poe.P.Imports) > 0 {
   382  		ip, err := b.CreateBucket(cacheKeyImport)
   383  		if err != nil {
   384  			return err
   385  		}
   386  		key := make(nuts.Key, nuts.KeyLen(uint64(len(poe.P.Imports)-1)))
   387  		for i := range poe.P.Imports {
   388  			v := []byte(poe.P.Imports[i])
   389  			key.Put(uint64(i))
   390  			if err := ip.Put(key, v); err != nil {
   391  				return err
   392  			}
   393  		}
   394  	}
   395  
   396  	if len(poe.P.Name) > 0 {
   397  		err := b.Put(cacheKeyName, []byte(poe.P.Name))
   398  		if err != nil {
   399  			return errors.Wrapf(err, "failed to put package: %v", poe.P)
   400  		}
   401  	}
   402  
   403  	if len(poe.P.TestImports) > 0 {
   404  		ip, err := b.CreateBucket(cacheKeyTestImport)
   405  		if err != nil {
   406  			return err
   407  		}
   408  		key := make(nuts.Key, nuts.KeyLen(uint64(len(poe.P.TestImports)-1)))
   409  		for i := range poe.P.TestImports {
   410  			v := []byte(poe.P.TestImports[i])
   411  			key.Put(uint64(i))
   412  			if err := ip.Put(key, v); err != nil {
   413  				return err
   414  			}
   415  		}
   416  	}
   417  	return nil
   418  }
   419  
   420  // cacheGetPackageOrErr returns a new pkgtree.PackageOrErr with fields retrieved
   421  // from the bolt.Bucket.
   422  func cacheGetPackageOrErr(b *bolt.Bucket) (pkgtree.PackageOrErr, error) {
   423  	if v := b.Get(cacheKeyError); len(v) > 0 {
   424  		return pkgtree.PackageOrErr{
   425  			Err: errors.New(string(v)),
   426  		}, nil
   427  	}
   428  
   429  	var p pkgtree.Package
   430  	p.CommentPath = string(b.Get(cacheKeyComment))
   431  	if ip := b.Bucket(cacheKeyImport); ip != nil {
   432  		err := ip.ForEach(func(_, v []byte) error {
   433  			p.Imports = append(p.Imports, string(v))
   434  			return nil
   435  		})
   436  		if err != nil {
   437  			return pkgtree.PackageOrErr{}, err
   438  		}
   439  	}
   440  	p.Name = string(b.Get(cacheKeyName))
   441  	if tip := b.Bucket(cacheKeyTestImport); tip != nil {
   442  		err := tip.ForEach(func(_, v []byte) error {
   443  			p.TestImports = append(p.TestImports, string(v))
   444  			return nil
   445  		})
   446  		if err != nil {
   447  			return pkgtree.PackageOrErr{}, err
   448  		}
   449  	}
   450  	return pkgtree.PackageOrErr{P: p}, nil
   451  }
   452  
   453  // cacheTimestampedKey returns a prefixed key with a trailing timestamp.
   454  func cacheTimestampedKey(pre byte, t time.Time) []byte {
   455  	b := make([]byte, 9)
   456  	b[0] = pre
   457  	binary.BigEndian.PutUint64(b[1:], uint64(t.Unix()))
   458  	return b
   459  }
   460  
   461  // boltTxOrBucket is a minimal interface satisfied by bolt.Tx and bolt.Bucket.
   462  type boltTxOrBucket interface {
   463  	Cursor() *bolt.Cursor
   464  	DeleteBucket([]byte) error
   465  	Bucket([]byte) *bolt.Bucket
   466  }
   467  
   468  // cachePrefixDelete prefix scans and deletes each bucket.
   469  func cachePrefixDelete(tob boltTxOrBucket, pre byte) error {
   470  	c := tob.Cursor()
   471  	for k, _ := c.Seek([]byte{pre}); len(k) > 0 && k[0] == pre; k, _ = c.Next() {
   472  		if err := tob.DeleteBucket(k); err != nil {
   473  			return errors.Wrapf(err, "failed to delete bucket: %s", k)
   474  		}
   475  	}
   476  	return nil
   477  }
   478  
   479  // cacheFindLatestValid prefix scans for the latest bucket which is timestamped >= epoch,
   480  // or returns nil if none exists.
   481  func cacheFindLatestValid(tob boltTxOrBucket, pre byte, epoch int64) *bolt.Bucket {
   482  	c := tob.Cursor()
   483  	var latest []byte
   484  	for k, _ := c.Seek([]byte{pre}); len(k) > 0 && k[0] == pre; k, _ = c.Next() {
   485  		latest = k
   486  	}
   487  	if latest == nil {
   488  		return nil
   489  	}
   490  	ts := latest[1:]
   491  	if len(ts) != 8 {
   492  		return nil
   493  	}
   494  	if int64(binary.BigEndian.Uint64(ts)) < epoch {
   495  		return nil
   496  	}
   497  	return tob.Bucket(latest)
   498  }