github.com/dottorblaster/docker@v1.5.0/graph/pull.go (about) 1 package graph 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "net" 8 "net/url" 9 "os" 10 "strings" 11 "time" 12 13 log "github.com/Sirupsen/logrus" 14 "github.com/docker/docker/engine" 15 "github.com/docker/docker/image" 16 "github.com/docker/docker/pkg/tarsum" 17 "github.com/docker/docker/registry" 18 "github.com/docker/docker/utils" 19 ) 20 21 func (s *TagStore) CmdPull(job *engine.Job) engine.Status { 22 if n := len(job.Args); n != 1 && n != 2 { 23 return job.Errorf("Usage: %s IMAGE [TAG]", job.Name) 24 } 25 26 var ( 27 localName = job.Args[0] 28 tag string 29 sf = utils.NewStreamFormatter(job.GetenvBool("json")) 30 authConfig = ®istry.AuthConfig{} 31 metaHeaders map[string][]string 32 ) 33 34 // Resolve the Repository name from fqn to RepositoryInfo 35 repoInfo, err := registry.ResolveRepositoryInfo(job, localName) 36 if err != nil { 37 return job.Error(err) 38 } 39 40 if len(job.Args) > 1 { 41 tag = job.Args[1] 42 } 43 44 job.GetenvJson("authConfig", authConfig) 45 job.GetenvJson("metaHeaders", &metaHeaders) 46 47 c, err := s.poolAdd("pull", repoInfo.LocalName+":"+tag) 48 if err != nil { 49 if c != nil { 50 // Another pull of the same repository is already taking place; just wait for it to finish 51 job.Stdout.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", repoInfo.LocalName)) 52 <-c 53 return engine.StatusOK 54 } 55 return job.Error(err) 56 } 57 defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) 58 59 log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) 60 endpoint, err := repoInfo.GetEndpoint() 61 if err != nil { 62 return job.Error(err) 63 } 64 65 r, err := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, true) 66 if err != nil { 67 return job.Error(err) 68 } 69 70 logName := repoInfo.LocalName 71 if tag != "" { 72 logName += ":" + tag 73 } 74 75 if len(repoInfo.Index.Mirrors) == 0 && ((repoInfo.Official && repoInfo.Index.Official) || endpoint.Version == registry.APIVersion2) { 76 j := job.Eng.Job("trust_update_base") 77 if err = j.Run(); err != nil { 78 log.Errorf("error updating trust base graph: %s", err) 79 } 80 81 log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) 82 if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { 83 if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { 84 log.Errorf("Error logging event 'pull' for %s: %s", logName, err) 85 } 86 return engine.StatusOK 87 } else if err != registry.ErrDoesNotExist { 88 log.Errorf("Error from V2 registry: %s", err) 89 } 90 91 log.Debug("image does not exist on v2 registry, falling back to v1") 92 } 93 94 log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName) 95 if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil { 96 return job.Error(err) 97 } 98 99 if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { 100 log.Errorf("Error logging event 'pull' for %s: %s", logName, err) 101 } 102 103 return engine.StatusOK 104 } 105 106 func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, askedTag string, sf *utils.StreamFormatter, parallel bool) error { 107 out.Write(sf.FormatStatus("", "Pulling repository %s", repoInfo.CanonicalName)) 108 109 repoData, err := r.GetRepositoryData(repoInfo.RemoteName) 110 if err != nil { 111 if strings.Contains(err.Error(), "HTTP code: 404") { 112 return fmt.Errorf("Error: image %s:%s not found", repoInfo.RemoteName, askedTag) 113 } 114 // Unexpected HTTP error 115 return err 116 } 117 118 log.Debugf("Retrieving the tag list") 119 tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens) 120 if err != nil { 121 log.Errorf("unable to get remote tags: %s", err) 122 return err 123 } 124 125 for tag, id := range tagsList { 126 repoData.ImgList[id] = ®istry.ImgData{ 127 ID: id, 128 Tag: tag, 129 Checksum: "", 130 } 131 } 132 133 log.Debugf("Registering tags") 134 // If no tag has been specified, pull them all 135 var imageId string 136 if askedTag == "" { 137 for tag, id := range tagsList { 138 repoData.ImgList[id].Tag = tag 139 } 140 } else { 141 // Otherwise, check that the tag exists and use only that one 142 id, exists := tagsList[askedTag] 143 if !exists { 144 return fmt.Errorf("Tag %s not found in repository %s", askedTag, repoInfo.CanonicalName) 145 } 146 imageId = id 147 repoData.ImgList[id].Tag = askedTag 148 } 149 150 errors := make(chan error) 151 152 layers_downloaded := false 153 for _, image := range repoData.ImgList { 154 downloadImage := func(img *registry.ImgData) { 155 if askedTag != "" && img.Tag != askedTag { 156 if parallel { 157 errors <- nil 158 } 159 return 160 } 161 162 if img.Tag == "" { 163 log.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) 164 if parallel { 165 errors <- nil 166 } 167 return 168 } 169 170 // ensure no two downloads of the same image happen at the same time 171 if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { 172 if c != nil { 173 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) 174 <-c 175 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 176 } else { 177 log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) 178 } 179 if parallel { 180 errors <- nil 181 } 182 return 183 } 184 defer s.poolRemove("pull", "img:"+img.ID) 185 186 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, repoInfo.CanonicalName), nil)) 187 success := false 188 var lastErr, err error 189 var is_downloaded bool 190 for _, ep := range repoInfo.Index.Mirrors { 191 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) 192 if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { 193 // Don't report errors when pulling from mirrors. 194 log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err) 195 continue 196 } 197 layers_downloaded = layers_downloaded || is_downloaded 198 success = true 199 break 200 } 201 if !success { 202 for _, ep := range repoData.Endpoints { 203 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) 204 if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { 205 // It's not ideal that only the last error is returned, it would be better to concatenate the errors. 206 // As the error is also given to the output stream the user will see the error. 207 lastErr = err 208 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err), nil)) 209 continue 210 } 211 layers_downloaded = layers_downloaded || is_downloaded 212 success = true 213 break 214 } 215 } 216 if !success { 217 err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, repoInfo.CanonicalName, lastErr) 218 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), err.Error(), nil)) 219 if parallel { 220 errors <- err 221 return 222 } 223 } 224 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 225 226 if parallel { 227 errors <- nil 228 } 229 } 230 231 if parallel { 232 go downloadImage(image) 233 } else { 234 downloadImage(image) 235 } 236 } 237 if parallel { 238 var lastError error 239 for i := 0; i < len(repoData.ImgList); i++ { 240 if err := <-errors; err != nil { 241 lastError = err 242 } 243 } 244 if lastError != nil { 245 return lastError 246 } 247 248 } 249 for tag, id := range tagsList { 250 if askedTag != "" && id != imageId { 251 continue 252 } 253 if err := s.Set(repoInfo.LocalName, tag, id, true); err != nil { 254 return err 255 } 256 } 257 258 requestedTag := repoInfo.CanonicalName 259 if len(askedTag) > 0 { 260 requestedTag = repoInfo.CanonicalName + ":" + askedTag 261 } 262 WriteStatus(requestedTag, out, sf, layers_downloaded) 263 return nil 264 } 265 266 func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) (bool, error) { 267 history, err := r.GetRemoteHistory(imgID, endpoint, token) 268 if err != nil { 269 return false, err 270 } 271 out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling dependent layers", nil)) 272 // FIXME: Try to stream the images? 273 // FIXME: Launch the getRemoteImage() in goroutines 274 275 layers_downloaded := false 276 for i := len(history) - 1; i >= 0; i-- { 277 id := history[i] 278 279 // ensure no two downloads of the same layer happen at the same time 280 if c, err := s.poolAdd("pull", "layer:"+id); err != nil { 281 log.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err) 282 <-c 283 } 284 defer s.poolRemove("pull", "layer:"+id) 285 286 if !s.graph.Exists(id) { 287 out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil)) 288 var ( 289 imgJSON []byte 290 imgSize int 291 err error 292 img *image.Image 293 ) 294 retries := 5 295 for j := 1; j <= retries; j++ { 296 imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint, token) 297 if err != nil && j == retries { 298 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) 299 return layers_downloaded, err 300 } else if err != nil { 301 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 302 continue 303 } 304 img, err = image.NewImgJSON(imgJSON) 305 layers_downloaded = true 306 if err != nil && j == retries { 307 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) 308 return layers_downloaded, fmt.Errorf("Failed to parse json: %s", err) 309 } else if err != nil { 310 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 311 continue 312 } else { 313 break 314 } 315 } 316 317 for j := 1; j <= retries; j++ { 318 // Get the layer 319 status := "Pulling fs layer" 320 if j > 1 { 321 status = fmt.Sprintf("Pulling fs layer [retries: %d]", j) 322 } 323 out.Write(sf.FormatProgress(utils.TruncateID(id), status, nil)) 324 layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token, int64(imgSize)) 325 if uerr, ok := err.(*url.Error); ok { 326 err = uerr.Err 327 } 328 if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries { 329 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 330 continue 331 } else if err != nil { 332 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) 333 return layers_downloaded, err 334 } 335 layers_downloaded = true 336 defer layer.Close() 337 338 err = s.graph.Register(img, 339 utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading")) 340 if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries { 341 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 342 continue 343 } else if err != nil { 344 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil)) 345 return layers_downloaded, err 346 } else { 347 break 348 } 349 } 350 } 351 out.Write(sf.FormatProgress(utils.TruncateID(id), "Download complete", nil)) 352 } 353 return layers_downloaded, nil 354 } 355 356 func WriteStatus(requestedTag string, out io.Writer, sf *utils.StreamFormatter, layers_downloaded bool) { 357 if layers_downloaded { 358 out.Write(sf.FormatStatus("", "Status: Downloaded newer image for %s", requestedTag)) 359 } else { 360 out.Write(sf.FormatStatus("", "Status: Image is up to date for %s", requestedTag)) 361 } 362 } 363 364 // downloadInfo is used to pass information from download to extractor 365 type downloadInfo struct { 366 imgJSON []byte 367 img *image.Image 368 tmpFile *os.File 369 length int64 370 downloaded bool 371 err chan error 372 } 373 374 func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { 375 endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) 376 if err != nil { 377 return fmt.Errorf("error getting registry endpoint: %s", err) 378 } 379 auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, true) 380 if err != nil { 381 return fmt.Errorf("error getting authorization: %s", err) 382 } 383 var layersDownloaded bool 384 if tag == "" { 385 log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) 386 tags, err := r.GetV2RemoteTags(endpoint, repoInfo.RemoteName, auth) 387 if err != nil { 388 return err 389 } 390 if len(tags) == 0 { 391 return registry.ErrDoesNotExist 392 } 393 for _, t := range tags { 394 if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, t, sf, parallel, auth); err != nil { 395 return err 396 } else if downloaded { 397 layersDownloaded = true 398 } 399 } 400 } else { 401 if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, tag, sf, parallel, auth); err != nil { 402 return err 403 } else if downloaded { 404 layersDownloaded = true 405 } 406 } 407 408 requestedTag := repoInfo.CanonicalName 409 if len(tag) > 0 { 410 requestedTag = repoInfo.CanonicalName + ":" + tag 411 } 412 WriteStatus(requestedTag, out, sf, layersDownloaded) 413 return nil 414 } 415 416 func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { 417 log.Debugf("Pulling tag from V2 registry: %q", tag) 418 manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) 419 if err != nil { 420 return false, err 421 } 422 423 manifest, verified, err := s.loadManifest(eng, manifestBytes) 424 if err != nil { 425 return false, fmt.Errorf("error verifying manifest: %s", err) 426 } 427 428 if err := checkValidManifest(manifest); err != nil { 429 return false, err 430 } 431 432 if verified { 433 log.Printf("Image manifest for %s:%s has been verified", repoInfo.CanonicalName, tag) 434 } else { 435 out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) 436 } 437 438 downloads := make([]downloadInfo, len(manifest.FSLayers)) 439 440 for i := len(manifest.FSLayers) - 1; i >= 0; i-- { 441 var ( 442 sumStr = manifest.FSLayers[i].BlobSum 443 imgJSON = []byte(manifest.History[i].V1Compatibility) 444 ) 445 446 img, err := image.NewImgJSON(imgJSON) 447 if err != nil { 448 return false, fmt.Errorf("failed to parse json: %s", err) 449 } 450 downloads[i].img = img 451 452 // Check if exists 453 if s.graph.Exists(img.ID) { 454 log.Debugf("Image already exists: %s", img.ID) 455 continue 456 } 457 458 chunks := strings.SplitN(sumStr, ":", 2) 459 if len(chunks) < 2 { 460 return false, fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks) 461 } 462 sumType, checksum := chunks[0], chunks[1] 463 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil)) 464 465 downloadFunc := func(di *downloadInfo) error { 466 log.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID) 467 468 if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { 469 if c != nil { 470 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) 471 <-c 472 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 473 } else { 474 log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) 475 } 476 } else { 477 defer s.poolRemove("pull", "img:"+img.ID) 478 tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob") 479 if err != nil { 480 return err 481 } 482 483 r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, sumType, checksum, auth) 484 if err != nil { 485 return err 486 } 487 defer r.Close() 488 489 // Wrap the reader with the appropriate TarSum reader. 490 tarSumReader, err := tarsum.NewTarSumForLabel(r, true, sumType) 491 if err != nil { 492 return fmt.Errorf("unable to wrap image blob reader with TarSum: %s", err) 493 } 494 495 io.Copy(tmpFile, utils.ProgressReader(ioutil.NopCloser(tarSumReader), int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) 496 497 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Verifying Checksum", nil)) 498 499 if finalChecksum := tarSumReader.Sum(nil); !strings.EqualFold(finalChecksum, sumStr) { 500 return fmt.Errorf("image verification failed: checksum mismatch - expected %q but got %q", sumStr, finalChecksum) 501 } 502 503 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 504 505 log.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name()) 506 di.tmpFile = tmpFile 507 di.length = l 508 di.downloaded = true 509 } 510 di.imgJSON = imgJSON 511 512 return nil 513 } 514 515 if parallel { 516 downloads[i].err = make(chan error) 517 go func(di *downloadInfo) { 518 di.err <- downloadFunc(di) 519 }(&downloads[i]) 520 } else { 521 err := downloadFunc(&downloads[i]) 522 if err != nil { 523 return false, err 524 } 525 } 526 } 527 528 var layersDownloaded bool 529 for i := len(downloads) - 1; i >= 0; i-- { 530 d := &downloads[i] 531 if d.err != nil { 532 err := <-d.err 533 if err != nil { 534 return false, err 535 } 536 } 537 if d.downloaded { 538 // if tmpFile is empty assume download and extracted elsewhere 539 defer os.Remove(d.tmpFile.Name()) 540 defer d.tmpFile.Close() 541 d.tmpFile.Seek(0, 0) 542 if d.tmpFile != nil { 543 err = s.graph.Register(d.img, 544 utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting")) 545 if err != nil { 546 return false, err 547 } 548 549 // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) 550 } 551 out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Pull complete", nil)) 552 layersDownloaded = true 553 } else { 554 out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Already exists", nil)) 555 } 556 557 } 558 559 out.Write(sf.FormatStatus(repoInfo.CanonicalName+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) 560 561 if err = s.Set(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { 562 return false, err 563 } 564 565 return layersDownloaded, nil 566 }