github.com/chenchun/docker@v1.3.2-0.20150629222414-20467faf132b/graph/pull.go (about)

     1  package graph
     2  
     3  import (
     4  	"fmt"
     5  	"io"
     6  	"io/ioutil"
     7  	"net"
     8  	"net/url"
     9  	"os"
    10  	"strings"
    11  	"time"
    12  
    13  	"github.com/Sirupsen/logrus"
    14  	"github.com/docker/distribution/digest"
    15  	"github.com/docker/docker/cliconfig"
    16  	"github.com/docker/docker/image"
    17  	"github.com/docker/docker/pkg/progressreader"
    18  	"github.com/docker/docker/pkg/streamformatter"
    19  	"github.com/docker/docker/pkg/stringid"
    20  	"github.com/docker/docker/pkg/transport"
    21  	"github.com/docker/docker/registry"
    22  	"github.com/docker/docker/utils"
    23  )
    24  
    25  type ImagePullConfig struct {
    26  	MetaHeaders map[string][]string
    27  	AuthConfig  *cliconfig.AuthConfig
    28  	OutStream   io.Writer
    29  }
    30  
    31  func (s *TagStore) Pull(image string, tag string, imagePullConfig *ImagePullConfig) error {
    32  	var (
    33  		sf = streamformatter.NewJSONStreamFormatter()
    34  	)
    35  
    36  	// Resolve the Repository name from fqn to RepositoryInfo
    37  	repoInfo, err := s.registryService.ResolveRepository(image)
    38  	if err != nil {
    39  		return err
    40  	}
    41  
    42  	if err := validateRepoName(repoInfo.LocalName); err != nil {
    43  		return err
    44  	}
    45  
    46  	c, err := s.poolAdd("pull", utils.ImageReference(repoInfo.LocalName, tag))
    47  	if err != nil {
    48  		if c != nil {
    49  			// Another pull of the same repository is already taking place; just wait for it to finish
    50  			imagePullConfig.OutStream.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", repoInfo.LocalName))
    51  			<-c
    52  			return nil
    53  		}
    54  		return err
    55  	}
    56  	defer s.poolRemove("pull", utils.ImageReference(repoInfo.LocalName, tag))
    57  
    58  	logName := repoInfo.LocalName
    59  	if tag != "" {
    60  		logName = utils.ImageReference(logName, tag)
    61  	}
    62  
    63  	// Attempt pulling official content from a provided v2 mirror
    64  	if repoInfo.Index.Official {
    65  		v2mirrorEndpoint, v2mirrorRepoInfo, err := configureV2Mirror(repoInfo, s.registryService)
    66  		if err != nil {
    67  			logrus.Errorf("Error configuring mirrors: %s", err)
    68  			return err
    69  		}
    70  
    71  		if v2mirrorEndpoint != nil {
    72  			logrus.Debugf("Attempting to pull from v2 mirror: %s", v2mirrorEndpoint.URL)
    73  			return s.pullFromV2Mirror(v2mirrorEndpoint, v2mirrorRepoInfo, imagePullConfig, tag, sf, logName)
    74  		}
    75  	}
    76  
    77  	logrus.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName)
    78  
    79  	endpoint, err := repoInfo.GetEndpoint(imagePullConfig.MetaHeaders)
    80  	if err != nil {
    81  		return err
    82  	}
    83  	// TODO(tiborvass): reuse client from endpoint?
    84  	// Adds Docker-specific headers as well as user-specified headers (metaHeaders)
    85  	tr := transport.NewTransport(
    86  		registry.NewTransport(registry.ReceiveTimeout, endpoint.IsSecure),
    87  		registry.DockerHeaders(imagePullConfig.MetaHeaders)...,
    88  	)
    89  	client := registry.HTTPClient(tr)
    90  	r, err := registry.NewSession(client, imagePullConfig.AuthConfig, endpoint)
    91  	if err != nil {
    92  		return err
    93  	}
    94  
    95  	if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) {
    96  		if repoInfo.Official {
    97  			s.trustService.UpdateBase()
    98  		}
    99  
   100  		logrus.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName)
   101  		if err := s.pullV2Repository(r, imagePullConfig.OutStream, repoInfo, tag, sf); err == nil {
   102  			s.eventsService.Log("pull", logName, "")
   103  			return nil
   104  		} else if err != registry.ErrDoesNotExist && err != ErrV2RegistryUnavailable {
   105  			logrus.Errorf("Error from V2 registry: %s", err)
   106  		}
   107  
   108  		logrus.Debug("image does not exist on v2 registry, falling back to v1")
   109  	}
   110  
   111  	if utils.DigestReference(tag) {
   112  		return fmt.Errorf("pulling with digest reference failed from v2 registry")
   113  	}
   114  
   115  	logrus.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName)
   116  	if err = s.pullRepository(r, imagePullConfig.OutStream, repoInfo, tag, sf); err != nil {
   117  		return err
   118  	}
   119  
   120  	s.eventsService.Log("pull", logName, "")
   121  
   122  	return nil
   123  
   124  }
   125  
   126  func makeMirrorRepoInfo(repoInfo *registry.RepositoryInfo, mirror string) *registry.RepositoryInfo {
   127  	mirrorRepo := &registry.RepositoryInfo{
   128  		RemoteName:    repoInfo.RemoteName,
   129  		LocalName:     repoInfo.LocalName,
   130  		CanonicalName: repoInfo.CanonicalName,
   131  		Official:      false,
   132  
   133  		Index: &registry.IndexInfo{
   134  			Official: false,
   135  			Secure:   repoInfo.Index.Secure,
   136  			Name:     mirror,
   137  			Mirrors:  []string{},
   138  		},
   139  	}
   140  	return mirrorRepo
   141  }
   142  
   143  func configureV2Mirror(repoInfo *registry.RepositoryInfo, s *registry.Service) (*registry.Endpoint, *registry.RepositoryInfo, error) {
   144  	mirrors := repoInfo.Index.Mirrors
   145  	if len(mirrors) == 0 {
   146  		// no mirrors configured
   147  		return nil, nil, nil
   148  	}
   149  
   150  	v1MirrorCount := 0
   151  	var v2MirrorEndpoint *registry.Endpoint
   152  	var v2MirrorRepoInfo *registry.RepositoryInfo
   153  	for _, mirror := range mirrors {
   154  		mirrorRepoInfo := makeMirrorRepoInfo(repoInfo, mirror)
   155  		endpoint, err := registry.NewEndpoint(mirrorRepoInfo.Index, nil)
   156  		if err != nil {
   157  			logrus.Errorf("Unable to create endpoint for %s: %s", mirror, err)
   158  			continue
   159  		}
   160  		if endpoint.Version == 2 {
   161  			if v2MirrorEndpoint == nil {
   162  				v2MirrorEndpoint = endpoint
   163  				v2MirrorRepoInfo = mirrorRepoInfo
   164  			} else {
   165  				// > 1 v2 mirrors given
   166  				return nil, nil, fmt.Errorf("multiple v2 mirrors configured")
   167  			}
   168  		} else {
   169  			v1MirrorCount++
   170  		}
   171  	}
   172  
   173  	if v1MirrorCount == len(mirrors) {
   174  		// OK, but mirrors are v1
   175  		return nil, nil, nil
   176  	}
   177  	if v2MirrorEndpoint != nil && v1MirrorCount == 0 {
   178  		// OK, 1 v2 mirror specified
   179  		return v2MirrorEndpoint, v2MirrorRepoInfo, nil
   180  	}
   181  	if v2MirrorEndpoint != nil && v1MirrorCount > 0 {
   182  		return nil, nil, fmt.Errorf("v1 and v2 mirrors configured")
   183  	}
   184  	// No endpoint could be established with the given mirror configurations
   185  	// Fallback to pulling from the hub as per v1 behavior.
   186  	return nil, nil, nil
   187  }
   188  
   189  func (s *TagStore) pullFromV2Mirror(mirrorEndpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo,
   190  	imagePullConfig *ImagePullConfig, tag string, sf *streamformatter.StreamFormatter, logName string) error {
   191  
   192  	tr := transport.NewTransport(
   193  		registry.NewTransport(registry.ReceiveTimeout, mirrorEndpoint.IsSecure),
   194  		registry.DockerHeaders(imagePullConfig.MetaHeaders)...,
   195  	)
   196  	client := registry.HTTPClient(tr)
   197  	mirrorSession, err := registry.NewSession(client, &cliconfig.AuthConfig{}, mirrorEndpoint)
   198  	if err != nil {
   199  		return err
   200  	}
   201  	logrus.Debugf("Pulling v2 repository with local name %q from %s", repoInfo.LocalName, mirrorEndpoint.URL)
   202  	if err := s.pullV2Repository(mirrorSession, imagePullConfig.OutStream, repoInfo, tag, sf); err != nil {
   203  		return err
   204  	}
   205  	s.eventsService.Log("pull", logName, "")
   206  	return nil
   207  }
   208  
   209  func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, askedTag string, sf *streamformatter.StreamFormatter) error {
   210  	out.Write(sf.FormatStatus("", "Pulling repository %s", repoInfo.CanonicalName))
   211  
   212  	repoData, err := r.GetRepositoryData(repoInfo.RemoteName)
   213  	if err != nil {
   214  		if strings.Contains(err.Error(), "HTTP code: 404") {
   215  			return fmt.Errorf("Error: image %s not found", utils.ImageReference(repoInfo.RemoteName, askedTag))
   216  		}
   217  		// Unexpected HTTP error
   218  		return err
   219  	}
   220  
   221  	logrus.Debugf("Retrieving the tag list")
   222  	tagsList := make(map[string]string)
   223  	if askedTag == "" {
   224  		tagsList, err = r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName)
   225  	} else {
   226  		var tagId string
   227  		tagId, err = r.GetRemoteTag(repoData.Endpoints, repoInfo.RemoteName, askedTag)
   228  		tagsList[askedTag] = tagId
   229  	}
   230  	if err != nil {
   231  		if err == registry.ErrRepoNotFound && askedTag != "" {
   232  			return fmt.Errorf("Tag %s not found in repository %s", askedTag, repoInfo.CanonicalName)
   233  		}
   234  		logrus.Errorf("unable to get remote tags: %s", err)
   235  		return err
   236  	}
   237  
   238  	for tag, id := range tagsList {
   239  		repoData.ImgList[id] = &registry.ImgData{
   240  			ID:       id,
   241  			Tag:      tag,
   242  			Checksum: "",
   243  		}
   244  	}
   245  
   246  	logrus.Debugf("Registering tags")
   247  	// If no tag has been specified, pull them all
   248  	if askedTag == "" {
   249  		for tag, id := range tagsList {
   250  			repoData.ImgList[id].Tag = tag
   251  		}
   252  	} else {
   253  		// Otherwise, check that the tag exists and use only that one
   254  		id, exists := tagsList[askedTag]
   255  		if !exists {
   256  			return fmt.Errorf("Tag %s not found in repository %s", askedTag, repoInfo.CanonicalName)
   257  		}
   258  		repoData.ImgList[id].Tag = askedTag
   259  	}
   260  
   261  	errors := make(chan error)
   262  
   263  	layersDownloaded := false
   264  	for _, image := range repoData.ImgList {
   265  		downloadImage := func(img *registry.ImgData) {
   266  			if askedTag != "" && img.Tag != askedTag {
   267  				errors <- nil
   268  				return
   269  			}
   270  
   271  			if img.Tag == "" {
   272  				logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID)
   273  				errors <- nil
   274  				return
   275  			}
   276  
   277  			// ensure no two downloads of the same image happen at the same time
   278  			if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
   279  				if c != nil {
   280  					out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil))
   281  					<-c
   282  					out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
   283  				} else {
   284  					logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
   285  				}
   286  				errors <- nil
   287  				return
   288  			}
   289  			defer s.poolRemove("pull", "img:"+img.ID)
   290  
   291  			out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, repoInfo.CanonicalName), nil))
   292  			success := false
   293  			var lastErr, err error
   294  			var isDownloaded bool
   295  			for _, ep := range repoInfo.Index.Mirrors {
   296  				// Ensure endpoint is v1
   297  				ep = ep + "v1/"
   298  				out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, repoInfo.CanonicalName, ep), nil))
   299  				if isDownloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
   300  					// Don't report errors when pulling from mirrors.
   301  					logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err)
   302  					continue
   303  				}
   304  				layersDownloaded = layersDownloaded || isDownloaded
   305  				success = true
   306  				break
   307  			}
   308  			if !success {
   309  				for _, ep := range repoData.Endpoints {
   310  					out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, repoInfo.CanonicalName, ep), nil))
   311  					if isDownloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil {
   312  						// It's not ideal that only the last error is returned, it would be better to concatenate the errors.
   313  						// As the error is also given to the output stream the user will see the error.
   314  						lastErr = err
   315  						out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err), nil))
   316  						continue
   317  					}
   318  					layersDownloaded = layersDownloaded || isDownloaded
   319  					success = true
   320  					break
   321  				}
   322  			}
   323  			if !success {
   324  				err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, repoInfo.CanonicalName, lastErr)
   325  				out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), err.Error(), nil))
   326  				errors <- err
   327  				return
   328  			}
   329  			out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
   330  
   331  			errors <- nil
   332  		}
   333  
   334  		go downloadImage(image)
   335  	}
   336  
   337  	var lastError error
   338  	for i := 0; i < len(repoData.ImgList); i++ {
   339  		if err := <-errors; err != nil {
   340  			lastError = err
   341  		}
   342  	}
   343  	if lastError != nil {
   344  		return lastError
   345  	}
   346  
   347  	for tag, id := range tagsList {
   348  		if askedTag != "" && tag != askedTag {
   349  			continue
   350  		}
   351  		if err := s.Tag(repoInfo.LocalName, tag, id, true); err != nil {
   352  			return err
   353  		}
   354  	}
   355  
   356  	requestedTag := repoInfo.CanonicalName
   357  	if len(askedTag) > 0 {
   358  		requestedTag = utils.ImageReference(repoInfo.CanonicalName, askedTag)
   359  	}
   360  	WriteStatus(requestedTag, out, sf, layersDownloaded)
   361  	return nil
   362  }
   363  
   364  func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint string, token []string, sf *streamformatter.StreamFormatter) (bool, error) {
   365  	history, err := r.GetRemoteHistory(imgID, endpoint)
   366  	if err != nil {
   367  		return false, err
   368  	}
   369  	out.Write(sf.FormatProgress(stringid.TruncateID(imgID), "Pulling dependent layers", nil))
   370  	// FIXME: Try to stream the images?
   371  	// FIXME: Launch the getRemoteImage() in goroutines
   372  
   373  	layersDownloaded := false
   374  	for i := len(history) - 1; i >= 0; i-- {
   375  		id := history[i]
   376  
   377  		// ensure no two downloads of the same layer happen at the same time
   378  		if c, err := s.poolAdd("pull", "layer:"+id); err != nil {
   379  			logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err)
   380  			<-c
   381  		}
   382  		defer s.poolRemove("pull", "layer:"+id)
   383  
   384  		if !s.graph.Exists(id) {
   385  			out.Write(sf.FormatProgress(stringid.TruncateID(id), "Pulling metadata", nil))
   386  			var (
   387  				imgJSON []byte
   388  				imgSize int
   389  				err     error
   390  				img     *image.Image
   391  			)
   392  			retries := 5
   393  			for j := 1; j <= retries; j++ {
   394  				imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint)
   395  				if err != nil && j == retries {
   396  					out.Write(sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
   397  					return layersDownloaded, err
   398  				} else if err != nil {
   399  					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
   400  					continue
   401  				}
   402  				img, err = image.NewImgJSON(imgJSON)
   403  				layersDownloaded = true
   404  				if err != nil && j == retries {
   405  					out.Write(sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
   406  					return layersDownloaded, fmt.Errorf("Failed to parse json: %s", err)
   407  				} else if err != nil {
   408  					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
   409  					continue
   410  				} else {
   411  					break
   412  				}
   413  			}
   414  
   415  			for j := 1; j <= retries; j++ {
   416  				// Get the layer
   417  				status := "Pulling fs layer"
   418  				if j > 1 {
   419  					status = fmt.Sprintf("Pulling fs layer [retries: %d]", j)
   420  				}
   421  				out.Write(sf.FormatProgress(stringid.TruncateID(id), status, nil))
   422  				layer, err := r.GetRemoteImageLayer(img.ID, endpoint, int64(imgSize))
   423  				if uerr, ok := err.(*url.Error); ok {
   424  					err = uerr.Err
   425  				}
   426  				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
   427  					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
   428  					continue
   429  				} else if err != nil {
   430  					out.Write(sf.FormatProgress(stringid.TruncateID(id), "Error pulling dependent layers", nil))
   431  					return layersDownloaded, err
   432  				}
   433  				layersDownloaded = true
   434  				defer layer.Close()
   435  
   436  				err = s.graph.Register(img,
   437  					progressreader.New(progressreader.Config{
   438  						In:        layer,
   439  						Out:       out,
   440  						Formatter: sf,
   441  						Size:      imgSize,
   442  						NewLines:  false,
   443  						ID:        stringid.TruncateID(id),
   444  						Action:    "Downloading",
   445  					}))
   446  				if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries {
   447  					time.Sleep(time.Duration(j) * 500 * time.Millisecond)
   448  					continue
   449  				} else if err != nil {
   450  					out.Write(sf.FormatProgress(stringid.TruncateID(id), "Error downloading dependent layers", nil))
   451  					return layersDownloaded, err
   452  				} else {
   453  					break
   454  				}
   455  			}
   456  		}
   457  		out.Write(sf.FormatProgress(stringid.TruncateID(id), "Download complete", nil))
   458  	}
   459  	return layersDownloaded, nil
   460  }
   461  
   462  func WriteStatus(requestedTag string, out io.Writer, sf *streamformatter.StreamFormatter, layersDownloaded bool) {
   463  	if layersDownloaded {
   464  		out.Write(sf.FormatStatus("", "Status: Downloaded newer image for %s", requestedTag))
   465  	} else {
   466  		out.Write(sf.FormatStatus("", "Status: Image is up to date for %s", requestedTag))
   467  	}
   468  }
   469  
   470  func (s *TagStore) pullV2Repository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *streamformatter.StreamFormatter) error {
   471  	endpoint, err := r.V2RegistryEndpoint(repoInfo.Index)
   472  	if err != nil {
   473  		if repoInfo.Index.Official {
   474  			logrus.Debugf("Unable to pull from V2 registry, falling back to v1: %s", err)
   475  			return ErrV2RegistryUnavailable
   476  		}
   477  		return fmt.Errorf("error getting registry endpoint: %s", err)
   478  	}
   479  	auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, true)
   480  	if err != nil {
   481  		return fmt.Errorf("error getting authorization: %s", err)
   482  	}
   483  	if !auth.CanAuthorizeV2() {
   484  		return ErrV2RegistryUnavailable
   485  	}
   486  
   487  	var layersDownloaded bool
   488  	if tag == "" {
   489  		logrus.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName)
   490  		tags, err := r.GetV2RemoteTags(endpoint, repoInfo.RemoteName, auth)
   491  		if err != nil {
   492  			return err
   493  		}
   494  		if len(tags) == 0 {
   495  			return registry.ErrDoesNotExist
   496  		}
   497  		for _, t := range tags {
   498  			if downloaded, err := s.pullV2Tag(r, out, endpoint, repoInfo, t, sf, auth); err != nil {
   499  				return err
   500  			} else if downloaded {
   501  				layersDownloaded = true
   502  			}
   503  		}
   504  	} else {
   505  		if downloaded, err := s.pullV2Tag(r, out, endpoint, repoInfo, tag, sf, auth); err != nil {
   506  			return err
   507  		} else if downloaded {
   508  			layersDownloaded = true
   509  		}
   510  	}
   511  
   512  	requestedTag := repoInfo.CanonicalName
   513  	if len(tag) > 0 {
   514  		requestedTag = utils.ImageReference(repoInfo.CanonicalName, tag)
   515  	}
   516  	WriteStatus(requestedTag, out, sf, layersDownloaded)
   517  	return nil
   518  }
   519  
   520  func (s *TagStore) pullV2Tag(r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *streamformatter.StreamFormatter, auth *registry.RequestAuthorization) (bool, error) {
   521  	logrus.Debugf("Pulling tag from V2 registry: %q", tag)
   522  
   523  	remoteDigest, manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth)
   524  	if err != nil {
   525  		return false, err
   526  	}
   527  
   528  	// loadManifest ensures that the manifest payload has the expected digest
   529  	// if the tag is a digest reference.
   530  	localDigest, manifest, verified, err := s.loadManifest(manifestBytes, tag, remoteDigest)
   531  	if err != nil {
   532  		return false, fmt.Errorf("error verifying manifest: %s", err)
   533  	}
   534  
   535  	if verified {
   536  		logrus.Printf("Image manifest for %s has been verified", utils.ImageReference(repoInfo.CanonicalName, tag))
   537  	}
   538  	out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName))
   539  
   540  	// downloadInfo is used to pass information from download to extractor
   541  	type downloadInfo struct {
   542  		imgJSON    []byte
   543  		img        *image.Image
   544  		digest     digest.Digest
   545  		tmpFile    *os.File
   546  		length     int64
   547  		downloaded bool
   548  		err        chan error
   549  	}
   550  
   551  	downloads := make([]downloadInfo, len(manifest.FSLayers))
   552  
   553  	for i := len(manifest.FSLayers) - 1; i >= 0; i-- {
   554  		var (
   555  			sumStr  = manifest.FSLayers[i].BlobSum
   556  			imgJSON = []byte(manifest.History[i].V1Compatibility)
   557  		)
   558  
   559  		img, err := image.NewImgJSON(imgJSON)
   560  		if err != nil {
   561  			return false, fmt.Errorf("failed to parse json: %s", err)
   562  		}
   563  		downloads[i].img = img
   564  
   565  		// Check if exists
   566  		if s.graph.Exists(img.ID) {
   567  			logrus.Debugf("Image already exists: %s", img.ID)
   568  			continue
   569  		}
   570  
   571  		dgst, err := digest.ParseDigest(sumStr)
   572  		if err != nil {
   573  			return false, err
   574  		}
   575  		downloads[i].digest = dgst
   576  
   577  		out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil))
   578  
   579  		downloadFunc := func(di *downloadInfo) error {
   580  			logrus.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID)
   581  
   582  			if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil {
   583  				if c != nil {
   584  					out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil))
   585  					<-c
   586  					out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
   587  				} else {
   588  					logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err)
   589  				}
   590  			} else {
   591  				defer s.poolRemove("pull", "img:"+img.ID)
   592  				tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob")
   593  				if err != nil {
   594  					return err
   595  				}
   596  
   597  				r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, di.digest, auth)
   598  				if err != nil {
   599  					return err
   600  				}
   601  				defer r.Close()
   602  
   603  				verifier, err := digest.NewDigestVerifier(di.digest)
   604  				if err != nil {
   605  					return err
   606  				}
   607  
   608  				if _, err := io.Copy(tmpFile, progressreader.New(progressreader.Config{
   609  					In:        ioutil.NopCloser(io.TeeReader(r, verifier)),
   610  					Out:       out,
   611  					Formatter: sf,
   612  					Size:      int(l),
   613  					NewLines:  false,
   614  					ID:        stringid.TruncateID(img.ID),
   615  					Action:    "Downloading",
   616  				})); err != nil {
   617  					return fmt.Errorf("unable to copy v2 image blob data: %s", err)
   618  				}
   619  
   620  				out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Verifying Checksum", nil))
   621  
   622  				if !verifier.Verified() {
   623  					return fmt.Errorf("image layer digest verification failed for %q", di.digest)
   624  				}
   625  
   626  				out.Write(sf.FormatProgress(stringid.TruncateID(img.ID), "Download complete", nil))
   627  
   628  				logrus.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name())
   629  				di.tmpFile = tmpFile
   630  				di.length = l
   631  				di.downloaded = true
   632  			}
   633  			di.imgJSON = imgJSON
   634  
   635  			return nil
   636  		}
   637  
   638  		downloads[i].err = make(chan error)
   639  		go func(di *downloadInfo) {
   640  			di.err <- downloadFunc(di)
   641  		}(&downloads[i])
   642  	}
   643  
   644  	var tagUpdated bool
   645  	for i := len(downloads) - 1; i >= 0; i-- {
   646  		d := &downloads[i]
   647  		if d.err != nil {
   648  			if err := <-d.err; err != nil {
   649  				return false, err
   650  			}
   651  		}
   652  		if d.downloaded {
   653  			// if tmpFile is empty assume download and extracted elsewhere
   654  			defer os.Remove(d.tmpFile.Name())
   655  			defer d.tmpFile.Close()
   656  			d.tmpFile.Seek(0, 0)
   657  			if d.tmpFile != nil {
   658  				err = s.graph.Register(d.img,
   659  					progressreader.New(progressreader.Config{
   660  						In:        d.tmpFile,
   661  						Out:       out,
   662  						Formatter: sf,
   663  						Size:      int(d.length),
   664  						ID:        stringid.TruncateID(d.img.ID),
   665  						Action:    "Extracting",
   666  					}))
   667  				if err != nil {
   668  					return false, err
   669  				}
   670  
   671  				if err := s.graph.SetDigest(d.img.ID, d.digest); err != nil {
   672  					return false, err
   673  				}
   674  
   675  				// FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted)
   676  			}
   677  			out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil))
   678  			tagUpdated = true
   679  		} else {
   680  			out.Write(sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil))
   681  		}
   682  
   683  	}
   684  
   685  	// Check for new tag if no layers downloaded
   686  	if !tagUpdated {
   687  		repo, err := s.Get(repoInfo.LocalName)
   688  		if err != nil {
   689  			return false, err
   690  		}
   691  		if repo != nil {
   692  			if _, exists := repo[tag]; !exists {
   693  				tagUpdated = true
   694  			}
   695  		} else {
   696  			tagUpdated = true
   697  		}
   698  	}
   699  
   700  	if verified && tagUpdated {
   701  		out.Write(sf.FormatStatus(utils.ImageReference(repoInfo.CanonicalName, tag), "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security."))
   702  	}
   703  
   704  	if localDigest != remoteDigest { // this is not a verification check.
   705  		// NOTE(stevvooe): This is a very defensive branch and should never
   706  		// happen, since all manifest digest implementations use the same
   707  		// algorithm.
   708  		logrus.WithFields(
   709  			logrus.Fields{
   710  				"local":  localDigest,
   711  				"remote": remoteDigest,
   712  			}).Debugf("local digest does not match remote")
   713  
   714  		out.Write(sf.FormatStatus("", "Remote Digest: %s", remoteDigest))
   715  	}
   716  
   717  	out.Write(sf.FormatStatus("", "Digest: %s", localDigest))
   718  
   719  	if tag == localDigest.String() {
   720  		// TODO(stevvooe): Ideally, we should always set the digest so we can
   721  		// use the digest whether we pull by it or not. Unfortunately, the tag
   722  		// store treats the digest as a separate tag, meaning there may be an
   723  		// untagged digest image that would seem to be dangling by a user.
   724  
   725  		if err = s.SetDigest(repoInfo.LocalName, localDigest.String(), downloads[0].img.ID); err != nil {
   726  			return false, err
   727  		}
   728  	}
   729  
   730  	if !utils.DigestReference(tag) {
   731  		// only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest)
   732  		if err = s.Tag(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil {
   733  			return false, err
   734  		}
   735  	}
   736  
   737  	return tagUpdated, nil
   738  }