github.com/mheon/docker@v0.11.2-0.20150922122814-44f47903a831/graph/pull_v2.go (about)

     1  package graph
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"io/ioutil"
     7  	"os"
     8  
     9  	"github.com/Sirupsen/logrus"
    10  	"github.com/docker/distribution"
    11  	"github.com/docker/distribution/digest"
    12  	"github.com/docker/distribution/manifest"
    13  	"github.com/docker/docker/image"
    14  	"github.com/docker/docker/pkg/progressreader"
    15  	"github.com/docker/docker/pkg/streamformatter"
    16  	"github.com/docker/docker/pkg/stringid"
    17  	"github.com/docker/docker/registry"
    18  	"github.com/docker/docker/trust"
    19  	"github.com/docker/docker/utils"
    20  	"github.com/docker/libtrust"
    21  	"golang.org/x/net/context"
    22  )
    23  
    24  type v2Puller struct {
    25  	*TagStore
    26  	endpoint  registry.APIEndpoint
    27  	config    *ImagePullConfig
    28  	sf        *streamformatter.StreamFormatter
    29  	repoInfo  *registry.RepositoryInfo
    30  	repo      distribution.Repository
    31  	sessionID string
    32  }
    33  
    34  func (p *v2Puller) Pull(tag string) (fallback bool, err error) {
    35  	// TODO(tiborvass): was ReceiveTimeout
    36  	p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull")
    37  	if err != nil {
    38  		logrus.Debugf("Error getting v2 registry: %v", err)
    39  		return true, err
    40  	}
    41  
    42  	p.sessionID = stringid.GenerateRandomID()
    43  
    44  	if err := p.pullV2Repository(tag); err != nil {
    45  		if registry.ContinueOnError(err) {
    46  			logrus.Debugf("Error trying v2 registry: %v", err)
    47  			return true, err
    48  		}
    49  		return false, err
    50  	}
    51  	return false, nil
    52  }
    53  
    54  func (p *v2Puller) pullV2Repository(tag string) (err error) {
    55  	var tags []string
    56  	taggedName := p.repoInfo.LocalName
    57  	if len(tag) > 0 {
    58  		tags = []string{tag}
    59  		taggedName = utils.ImageReference(p.repoInfo.LocalName, tag)
    60  	} else {
    61  		var err error
    62  
    63  		manSvc, err := p.repo.Manifests(context.Background())
    64  		if err != nil {
    65  			return err
    66  		}
    67  
    68  		tags, err = manSvc.Tags()
    69  		if err != nil {
    70  			return err
    71  		}
    72  
    73  	}
    74  
    75  	broadcaster, found := p.poolAdd("pull", taggedName)
    76  	broadcaster.Add(p.config.OutStream)
    77  	if found {
    78  		// Another pull of the same repository is already taking place; just wait for it to finish
    79  		return broadcaster.Wait()
    80  	}
    81  
    82  	// This must use a closure so it captures the value of err when the
    83  	// function returns, not when the 'defer' is evaluated.
    84  	defer func() {
    85  		p.poolRemoveWithError("pull", taggedName, err)
    86  	}()
    87  
    88  	var layersDownloaded bool
    89  	for _, tag := range tags {
    90  		// pulledNew is true if either new layers were downloaded OR if existing images were newly tagged
    91  		// TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus?
    92  		pulledNew, err := p.pullV2Tag(broadcaster, tag, taggedName)
    93  		if err != nil {
    94  			return err
    95  		}
    96  		layersDownloaded = layersDownloaded || pulledNew
    97  	}
    98  
    99  	writeStatus(taggedName, broadcaster, p.sf, layersDownloaded)
   100  
   101  	return nil
   102  }
   103  
   104  // downloadInfo is used to pass information from download to extractor
   105  type downloadInfo struct {
   106  	img         *image.Image
   107  	tmpFile     *os.File
   108  	digest      digest.Digest
   109  	layer       distribution.ReadSeekCloser
   110  	size        int64
   111  	err         chan error
   112  	poolKey     string
   113  	broadcaster *progressreader.Broadcaster
   114  }
   115  
   116  type errVerification struct{}
   117  
   118  func (errVerification) Error() string { return "verification failed" }
   119  
   120  func (p *v2Puller) download(di *downloadInfo) {
   121  	logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID)
   122  
   123  	blobs := p.repo.Blobs(context.Background())
   124  
   125  	desc, err := blobs.Stat(context.Background(), di.digest)
   126  	if err != nil {
   127  		logrus.Debugf("Error statting layer: %v", err)
   128  		di.err <- err
   129  		return
   130  	}
   131  	di.size = desc.Size
   132  
   133  	layerDownload, err := blobs.Open(context.Background(), di.digest)
   134  	if err != nil {
   135  		logrus.Debugf("Error fetching layer: %v", err)
   136  		di.err <- err
   137  		return
   138  	}
   139  	defer layerDownload.Close()
   140  
   141  	verifier, err := digest.NewDigestVerifier(di.digest)
   142  	if err != nil {
   143  		di.err <- err
   144  		return
   145  	}
   146  
   147  	reader := progressreader.New(progressreader.Config{
   148  		In:        ioutil.NopCloser(io.TeeReader(layerDownload, verifier)),
   149  		Out:       di.broadcaster,
   150  		Formatter: p.sf,
   151  		Size:      di.size,
   152  		NewLines:  false,
   153  		ID:        stringid.TruncateID(di.img.ID),
   154  		Action:    "Downloading",
   155  	})
   156  	io.Copy(di.tmpFile, reader)
   157  
   158  	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil))
   159  
   160  	if !verifier.Verified() {
   161  		err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest)
   162  		logrus.Error(err)
   163  		di.err <- err
   164  		return
   165  	}
   166  
   167  	di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil))
   168  
   169  	logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, di.tmpFile.Name())
   170  	di.layer = layerDownload
   171  
   172  	di.err <- nil
   173  }
   174  
   175  func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bool, err error) {
   176  	logrus.Debugf("Pulling tag from V2 registry: %q", tag)
   177  
   178  	manSvc, err := p.repo.Manifests(context.Background())
   179  	if err != nil {
   180  		return false, err
   181  	}
   182  
   183  	manifest, err := manSvc.GetByTag(tag)
   184  	if err != nil {
   185  		return false, err
   186  	}
   187  	verified, err = p.validateManifest(manifest, tag)
   188  	if err != nil {
   189  		return false, err
   190  	}
   191  	if verified {
   192  		logrus.Printf("Image manifest for %s has been verified", taggedName)
   193  	}
   194  
   195  	out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name()))
   196  
   197  	var downloads []*downloadInfo
   198  
   199  	var layerIDs []string
   200  	defer func() {
   201  		p.graph.Release(p.sessionID, layerIDs...)
   202  
   203  		for _, d := range downloads {
   204  			p.poolRemoveWithError("pull", d.poolKey, err)
   205  			if d.tmpFile != nil {
   206  				d.tmpFile.Close()
   207  				if err := os.RemoveAll(d.tmpFile.Name()); err != nil {
   208  					logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name())
   209  				}
   210  			}
   211  		}
   212  	}()
   213  
   214  	for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
   215  		img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility))
   216  		if err != nil {
   217  			logrus.Debugf("error getting image v1 json: %v", err)
   218  			return false, err
   219  		}
   220  		p.graph.Retain(p.sessionID, img.ID)
   221  		layerIDs = append(layerIDs, img.ID)
   222  
   223  		// Check if exists
   224  		if p.graph.Exists(img.ID) {
   225  			logrus.Debugf("Image already exists: %s", img.ID)
   226  			out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Already exists", nil))
   227  			continue
   228  		}
   229  		out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
   230  
   231  		d := &downloadInfo{
   232  			img:     img,
   233  			poolKey: "layer:" + img.ID,
   234  			digest:  manifest.FSLayers[i].BlobSum,
   235  			// TODO: seems like this chan buffer solved hanging problem in go1.5,
   236  			// this can indicate some deeper problem that somehow we never take
   237  			// error from channel in loop below
   238  			err: make(chan error, 1),
   239  		}
   240  
   241  		tmpFile, err := ioutil.TempFile("", "GetImageBlob")
   242  		if err != nil {
   243  			return false, err
   244  		}
   245  		d.tmpFile = tmpFile
   246  
   247  		downloads = append(downloads, d)
   248  
   249  		broadcaster, found := p.poolAdd("pull", d.poolKey)
   250  		broadcaster.Add(out)
   251  		d.broadcaster = broadcaster
   252  		if found {
   253  			d.err <- nil
   254  		} else {
   255  			go p.download(d)
   256  		}
   257  	}
   258  
   259  	var tagUpdated bool
   260  	for _, d := range downloads {
   261  		if err := <-d.err; err != nil {
   262  			return false, err
   263  		}
   264  
   265  		if d.layer == nil {
   266  			// Wait for a different pull to download and extract
   267  			// this layer.
   268  			err = d.broadcaster.Wait()
   269  			if err != nil {
   270  				return false, err
   271  			}
   272  			continue
   273  		}
   274  
   275  		d.tmpFile.Seek(0, 0)
   276  		reader := progressreader.New(progressreader.Config{
   277  			In:        d.tmpFile,
   278  			Out:       d.broadcaster,
   279  			Formatter: p.sf,
   280  			Size:      d.size,
   281  			NewLines:  false,
   282  			ID:        stringid.TruncateID(d.img.ID),
   283  			Action:    "Extracting",
   284  		})
   285  
   286  		err = p.graph.Register(d.img, reader)
   287  		if err != nil {
   288  			return false, err
   289  		}
   290  
   291  		if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil {
   292  			return false, err
   293  		}
   294  
   295  		d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
   296  		d.broadcaster.Close()
   297  		tagUpdated = true
   298  	}
   299  
   300  	manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName)
   301  	if err != nil {
   302  		return false, err
   303  	}
   304  
   305  	// Check for new tag if no layers downloaded
   306  	if !tagUpdated {
   307  		repo, err := p.Get(p.repoInfo.LocalName)
   308  		if err != nil {
   309  			return false, err
   310  		}
   311  		if repo != nil {
   312  			if _, exists := repo[tag]; !exists {
   313  				tagUpdated = true
   314  			}
   315  		} else {
   316  			tagUpdated = true
   317  		}
   318  	}
   319  
   320  	if verified && tagUpdated {
   321  		out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
   322  	}
   323  
   324  	firstID := layerIDs[len(layerIDs)-1]
   325  	if utils.DigestReference(tag) {
   326  		// TODO(stevvooe): Ideally, we should always set the digest so we can
   327  		// use the digest whether we pull by it or not. Unfortunately, the tag
   328  		// store treats the digest as a separate tag, meaning there may be an
   329  		// untagged digest image that would seem to be dangling by a user.
   330  		if err = p.SetDigest(p.repoInfo.LocalName, tag, firstID); err != nil {
   331  			return false, err
   332  		}
   333  	} else {
   334  		// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
   335  		if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil {
   336  			return false, err
   337  		}
   338  	}
   339  
   340  	if manifestDigest != "" {
   341  		out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest))
   342  	}
   343  
   344  	return tagUpdated, nil
   345  }
   346  
   347  // verifyTrustedKeys checks the keys provided against the trust store,
   348  // ensuring that the provided keys are trusted for the namespace. The keys
   349  // provided from this method must come from the signatures provided as part of
   350  // the manifest JWS package, obtained from unpackSignedManifest or libtrust.
   351  func (p *v2Puller) verifyTrustedKeys(namespace string, keys []libtrust.PublicKey) (verified bool, err error) {
   352  	if namespace[0] != '/' {
   353  		namespace = "/" + namespace
   354  	}
   355  
   356  	for _, key := range keys {
   357  		b, err := key.MarshalJSON()
   358  		if err != nil {
   359  			return false, fmt.Errorf("error marshalling public key: %s", err)
   360  		}
   361  		// Check key has read/write permission (0x03)
   362  		v, err := p.trustService.CheckKey(namespace, b, 0x03)
   363  		if err != nil {
   364  			vErr, ok := err.(trust.NotVerifiedError)
   365  			if !ok {
   366  				return false, fmt.Errorf("error running key check: %s", err)
   367  			}
   368  			logrus.Debugf("Key check result: %v", vErr)
   369  		}
   370  		verified = v
   371  	}
   372  
   373  	if verified {
   374  		logrus.Debug("Key check result: verified")
   375  	}
   376  
   377  	return
   378  }
   379  
   380  func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (verified bool, err error) {
   381  	// If pull by digest, then verify the manifest digest. NOTE: It is
   382  	// important to do this first, before any other content validation. If the
   383  	// digest cannot be verified, don't even bother with those other things.
   384  	if manifestDigest, err := digest.ParseDigest(tag); err == nil {
   385  		verifier, err := digest.NewDigestVerifier(manifestDigest)
   386  		if err != nil {
   387  			return false, err
   388  		}
   389  		payload, err := m.Payload()
   390  		if err != nil {
   391  			return false, err
   392  		}
   393  		if _, err := verifier.Write(payload); err != nil {
   394  			return false, err
   395  		}
   396  		if !verifier.Verified() {
   397  			err := fmt.Errorf("image verification failed for digest %s", manifestDigest)
   398  			logrus.Error(err)
   399  			return false, err
   400  		}
   401  	}
   402  
   403  	// TODO(tiborvass): what's the usecase for having manifest == nil and err == nil ? Shouldn't be the error be "DoesNotExist" ?
   404  	if m == nil {
   405  		return false, fmt.Errorf("image manifest does not exist for tag %q", tag)
   406  	}
   407  	if m.SchemaVersion != 1 {
   408  		return false, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag)
   409  	}
   410  	if len(m.FSLayers) != len(m.History) {
   411  		return false, fmt.Errorf("length of history not equal to number of layers for tag %q", tag)
   412  	}
   413  	if len(m.FSLayers) == 0 {
   414  		return false, fmt.Errorf("no FSLayers in manifest for tag %q", tag)
   415  	}
   416  	keys, err := manifest.Verify(m)
   417  	if err != nil {
   418  		return false, fmt.Errorf("error verifying manifest for tag %q: %v", tag, err)
   419  	}
   420  	verified, err = p.verifyTrustedKeys(m.Name, keys)
   421  	if err != nil {
   422  		return false, fmt.Errorf("error verifying manifest keys: %v", err)
   423  	}
   424  	return verified, nil
   425  }