github.com/pritambaral/docker@v1.4.2-0.20150120174542-b2fe1b3dd952/graph/pull.go (about) 1 package graph 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "net" 8 "net/url" 9 "os" 10 "strings" 11 "time" 12 13 log "github.com/Sirupsen/logrus" 14 "github.com/docker/docker/engine" 15 "github.com/docker/docker/image" 16 "github.com/docker/docker/pkg/tarsum" 17 "github.com/docker/docker/registry" 18 "github.com/docker/docker/utils" 19 ) 20 21 func (s *TagStore) CmdPull(job *engine.Job) engine.Status { 22 if n := len(job.Args); n != 1 && n != 2 { 23 return job.Errorf("Usage: %s IMAGE [TAG]", job.Name) 24 } 25 26 var ( 27 localName = job.Args[0] 28 tag string 29 sf = utils.NewStreamFormatter(job.GetenvBool("json")) 30 authConfig = ®istry.AuthConfig{} 31 metaHeaders map[string][]string 32 ) 33 34 // Resolve the Repository name from fqn to RepositoryInfo 35 repoInfo, err := registry.ResolveRepositoryInfo(job, localName) 36 if err != nil { 37 return job.Error(err) 38 } 39 40 if len(job.Args) > 1 { 41 tag = job.Args[1] 42 } 43 44 job.GetenvJson("authConfig", authConfig) 45 job.GetenvJson("metaHeaders", &metaHeaders) 46 47 c, err := s.poolAdd("pull", repoInfo.LocalName+":"+tag) 48 if err != nil { 49 if c != nil { 50 // Another pull of the same repository is already taking place; just wait for it to finish 51 job.Stdout.Write(sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", repoInfo.LocalName)) 52 <-c 53 return engine.StatusOK 54 } 55 return job.Error(err) 56 } 57 defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) 58 59 log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) 60 endpoint, err := repoInfo.GetEndpoint() 61 if err != nil { 62 return job.Error(err) 63 } 64 65 r, err := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, true) 66 if err != nil { 67 return job.Error(err) 68 } 69 70 logName := repoInfo.LocalName 71 if tag != "" { 72 logName += ":" + tag 73 } 74 75 if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { 76 j := job.Eng.Job("trust_update_base") 77 if err = j.Run(); err != nil { 78 log.Errorf("error updating trust base graph: %s", err) 79 } 80 81 log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) 82 if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { 83 if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { 84 log.Errorf("Error logging event 'pull' for %s: %s", logName, err) 85 } 86 return engine.StatusOK 87 } else if err != registry.ErrDoesNotExist { 88 log.Errorf("Error from V2 registry: %s", err) 89 } 90 91 log.Debug("image does not exist on v2 registry, falling back to v1") 92 } 93 94 log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName) 95 if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil { 96 return job.Error(err) 97 } 98 99 if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { 100 log.Errorf("Error logging event 'pull' for %s: %s", logName, err) 101 } 102 103 return engine.StatusOK 104 } 105 106 func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, askedTag string, sf *utils.StreamFormatter, parallel bool) error { 107 out.Write(sf.FormatStatus("", "Pulling repository %s", repoInfo.CanonicalName)) 108 109 repoData, err := r.GetRepositoryData(repoInfo.RemoteName) 110 if err != nil { 111 if strings.Contains(err.Error(), "HTTP code: 404") { 112 return fmt.Errorf("Error: image %s:%s not found", repoInfo.RemoteName, askedTag) 113 } 114 // Unexpected HTTP error 115 return err 116 } 117 118 log.Debugf("Retrieving the tag list") 119 tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens) 120 if err != nil { 121 log.Errorf("unable to get remote tags: %s", err) 122 return err 123 } 124 125 for tag, id := range tagsList { 126 repoData.ImgList[id] = ®istry.ImgData{ 127 ID: id, 128 Tag: tag, 129 Checksum: "", 130 } 131 } 132 133 log.Debugf("Registering tags") 134 // If no tag has been specified, pull them all 135 var imageId string 136 if askedTag == "" { 137 for tag, id := range tagsList { 138 repoData.ImgList[id].Tag = tag 139 } 140 } else { 141 // Otherwise, check that the tag exists and use only that one 142 id, exists := tagsList[askedTag] 143 if !exists { 144 return fmt.Errorf("Tag %s not found in repository %s", askedTag, repoInfo.CanonicalName) 145 } 146 imageId = id 147 repoData.ImgList[id].Tag = askedTag 148 } 149 150 errors := make(chan error) 151 152 layers_downloaded := false 153 for _, image := range repoData.ImgList { 154 downloadImage := func(img *registry.ImgData) { 155 if askedTag != "" && img.Tag != askedTag { 156 log.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) 157 if parallel { 158 errors <- nil 159 } 160 return 161 } 162 163 if img.Tag == "" { 164 log.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) 165 if parallel { 166 errors <- nil 167 } 168 return 169 } 170 171 // ensure no two downloads of the same image happen at the same time 172 if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { 173 if c != nil { 174 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) 175 <-c 176 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 177 } else { 178 log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) 179 } 180 if parallel { 181 errors <- nil 182 } 183 return 184 } 185 defer s.poolRemove("pull", "img:"+img.ID) 186 187 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s", img.Tag, repoInfo.CanonicalName), nil)) 188 success := false 189 var lastErr, err error 190 var is_downloaded bool 191 for _, ep := range repoInfo.Index.Mirrors { 192 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) 193 if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { 194 // Don't report errors when pulling from mirrors. 195 log.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err) 196 continue 197 } 198 layers_downloaded = layers_downloaded || is_downloaded 199 success = true 200 break 201 } 202 if !success { 203 for _, ep := range repoData.Endpoints { 204 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, endpoint: %s", img.Tag, repoInfo.CanonicalName, ep), nil)) 205 if is_downloaded, err = s.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { 206 // It's not ideal that only the last error is returned, it would be better to concatenate the errors. 207 // As the error is also given to the output stream the user will see the error. 208 lastErr = err 209 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), fmt.Sprintf("Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, repoInfo.CanonicalName, ep, err), nil)) 210 continue 211 } 212 layers_downloaded = layers_downloaded || is_downloaded 213 success = true 214 break 215 } 216 } 217 if !success { 218 err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, repoInfo.CanonicalName, lastErr) 219 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), err.Error(), nil)) 220 if parallel { 221 errors <- err 222 return 223 } 224 } 225 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 226 227 if parallel { 228 errors <- nil 229 } 230 } 231 232 if parallel { 233 go downloadImage(image) 234 } else { 235 downloadImage(image) 236 } 237 } 238 if parallel { 239 var lastError error 240 for i := 0; i < len(repoData.ImgList); i++ { 241 if err := <-errors; err != nil { 242 lastError = err 243 } 244 } 245 if lastError != nil { 246 return lastError 247 } 248 249 } 250 for tag, id := range tagsList { 251 if askedTag != "" && id != imageId { 252 continue 253 } 254 if err := s.Set(repoInfo.LocalName, tag, id, true); err != nil { 255 return err 256 } 257 } 258 259 requestedTag := repoInfo.CanonicalName 260 if len(askedTag) > 0 { 261 requestedTag = repoInfo.CanonicalName + ":" + askedTag 262 } 263 WriteStatus(requestedTag, out, sf, layers_downloaded) 264 return nil 265 } 266 267 func (s *TagStore) pullImage(r *registry.Session, out io.Writer, imgID, endpoint string, token []string, sf *utils.StreamFormatter) (bool, error) { 268 history, err := r.GetRemoteHistory(imgID, endpoint, token) 269 if err != nil { 270 return false, err 271 } 272 out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pulling dependent layers", nil)) 273 // FIXME: Try to stream the images? 274 // FIXME: Launch the getRemoteImage() in goroutines 275 276 layers_downloaded := false 277 for i := len(history) - 1; i >= 0; i-- { 278 id := history[i] 279 280 // ensure no two downloads of the same layer happen at the same time 281 if c, err := s.poolAdd("pull", "layer:"+id); err != nil { 282 log.Debugf("Image (id: %s) pull is already running, skipping: %v", id, err) 283 <-c 284 } 285 defer s.poolRemove("pull", "layer:"+id) 286 287 if !s.graph.Exists(id) { 288 out.Write(sf.FormatProgress(utils.TruncateID(id), "Pulling metadata", nil)) 289 var ( 290 imgJSON []byte 291 imgSize int 292 err error 293 img *image.Image 294 ) 295 retries := 5 296 for j := 1; j <= retries; j++ { 297 imgJSON, imgSize, err = r.GetRemoteImageJSON(id, endpoint, token) 298 if err != nil && j == retries { 299 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) 300 return layers_downloaded, err 301 } else if err != nil { 302 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 303 continue 304 } 305 img, err = image.NewImgJSON(imgJSON) 306 layers_downloaded = true 307 if err != nil && j == retries { 308 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) 309 return layers_downloaded, fmt.Errorf("Failed to parse json: %s", err) 310 } else if err != nil { 311 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 312 continue 313 } else { 314 break 315 } 316 } 317 318 for j := 1; j <= retries; j++ { 319 // Get the layer 320 status := "Pulling fs layer" 321 if j > 1 { 322 status = fmt.Sprintf("Pulling fs layer [retries: %d]", j) 323 } 324 out.Write(sf.FormatProgress(utils.TruncateID(id), status, nil)) 325 layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token, int64(imgSize)) 326 if uerr, ok := err.(*url.Error); ok { 327 err = uerr.Err 328 } 329 if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries { 330 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 331 continue 332 } else if err != nil { 333 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error pulling dependent layers", nil)) 334 return layers_downloaded, err 335 } 336 layers_downloaded = true 337 defer layer.Close() 338 339 err = s.graph.Register(img, 340 utils.ProgressReader(layer, imgSize, out, sf, false, utils.TruncateID(id), "Downloading")) 341 if terr, ok := err.(net.Error); ok && terr.Timeout() && j < retries { 342 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 343 continue 344 } else if err != nil { 345 out.Write(sf.FormatProgress(utils.TruncateID(id), "Error downloading dependent layers", nil)) 346 return layers_downloaded, err 347 } else { 348 break 349 } 350 } 351 } 352 out.Write(sf.FormatProgress(utils.TruncateID(id), "Download complete", nil)) 353 } 354 return layers_downloaded, nil 355 } 356 357 func WriteStatus(requestedTag string, out io.Writer, sf *utils.StreamFormatter, layers_downloaded bool) { 358 if layers_downloaded { 359 out.Write(sf.FormatStatus("", "Status: Downloaded newer image for %s", requestedTag)) 360 } else { 361 out.Write(sf.FormatStatus("", "Status: Image is up to date for %s", requestedTag)) 362 } 363 } 364 365 // downloadInfo is used to pass information from download to extractor 366 type downloadInfo struct { 367 imgJSON []byte 368 img *image.Image 369 tmpFile *os.File 370 length int64 371 downloaded bool 372 err chan error 373 } 374 375 func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { 376 endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) 377 if err != nil { 378 return fmt.Errorf("error getting registry endpoint: %s", err) 379 } 380 auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, true) 381 if err != nil { 382 return fmt.Errorf("error getting authorization: %s", err) 383 } 384 var layersDownloaded bool 385 if tag == "" { 386 log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) 387 tags, err := r.GetV2RemoteTags(endpoint, repoInfo.RemoteName, auth) 388 if err != nil { 389 return err 390 } 391 for _, t := range tags { 392 if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, t, sf, parallel, auth); err != nil { 393 return err 394 } else if downloaded { 395 layersDownloaded = true 396 } 397 } 398 } else { 399 if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, tag, sf, parallel, auth); err != nil { 400 return err 401 } else if downloaded { 402 layersDownloaded = true 403 } 404 } 405 406 requestedTag := repoInfo.CanonicalName 407 if len(tag) > 0 { 408 requestedTag = repoInfo.CanonicalName + ":" + tag 409 } 410 WriteStatus(requestedTag, out, sf, layersDownloaded) 411 return nil 412 } 413 414 func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { 415 log.Debugf("Pulling tag from V2 registry: %q", tag) 416 manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) 417 if err != nil { 418 return false, err 419 } 420 421 manifest, verified, err := s.verifyManifest(eng, manifestBytes) 422 if err != nil { 423 return false, fmt.Errorf("error verifying manifest: %s", err) 424 } 425 426 if err := checkValidManifest(manifest); err != nil { 427 return false, err 428 } 429 430 if verified { 431 out.Write(sf.FormatStatus(repoInfo.CanonicalName+":"+tag, "The image you are pulling has been verified")) 432 } else { 433 out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) 434 } 435 downloads := make([]downloadInfo, len(manifest.FSLayers)) 436 437 for i := len(manifest.FSLayers) - 1; i >= 0; i-- { 438 var ( 439 sumStr = manifest.FSLayers[i].BlobSum 440 imgJSON = []byte(manifest.History[i].V1Compatibility) 441 ) 442 443 img, err := image.NewImgJSON(imgJSON) 444 if err != nil { 445 return false, fmt.Errorf("failed to parse json: %s", err) 446 } 447 downloads[i].img = img 448 449 // Check if exists 450 if s.graph.Exists(img.ID) { 451 log.Debugf("Image already exists: %s", img.ID) 452 continue 453 } 454 455 chunks := strings.SplitN(sumStr, ":", 2) 456 if len(chunks) < 2 { 457 return false, fmt.Errorf("expected 2 parts in the sumStr, got %#v", chunks) 458 } 459 sumType, checksum := chunks[0], chunks[1] 460 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Pulling fs layer", nil)) 461 462 downloadFunc := func(di *downloadInfo) error { 463 log.Debugf("pulling blob %q to V1 img %s", sumStr, img.ID) 464 465 if c, err := s.poolAdd("pull", "img:"+img.ID); err != nil { 466 if c != nil { 467 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Layer already being pulled by another client. Waiting.", nil)) 468 <-c 469 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 470 } else { 471 log.Debugf("Image (id: %s) pull is already running, skipping: %v", img.ID, err) 472 } 473 } else { 474 defer s.poolRemove("pull", "img:"+img.ID) 475 tmpFile, err := ioutil.TempFile("", "GetV2ImageBlob") 476 if err != nil { 477 return err 478 } 479 480 r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, sumType, checksum, auth) 481 if err != nil { 482 return err 483 } 484 defer r.Close() 485 486 // Wrap the reader with the appropriate TarSum reader. 487 tarSumReader, err := tarsum.NewTarSumForLabel(r, true, sumType) 488 if err != nil { 489 return fmt.Errorf("unable to wrap image blob reader with TarSum: %s", err) 490 } 491 492 io.Copy(tmpFile, utils.ProgressReader(ioutil.NopCloser(tarSumReader), int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) 493 494 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Verifying Checksum", nil)) 495 496 if finalChecksum := tarSumReader.Sum(nil); !strings.EqualFold(finalChecksum, sumStr) { 497 return fmt.Errorf("image verification failed: checksum mismatch - expected %q but got %q", sumStr, finalChecksum) 498 } 499 500 out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) 501 502 log.Debugf("Downloaded %s to tempfile %s", img.ID, tmpFile.Name()) 503 di.tmpFile = tmpFile 504 di.length = l 505 di.downloaded = true 506 } 507 di.imgJSON = imgJSON 508 509 return nil 510 } 511 512 if parallel { 513 downloads[i].err = make(chan error) 514 go func(di *downloadInfo) { 515 di.err <- downloadFunc(di) 516 }(&downloads[i]) 517 } else { 518 err := downloadFunc(&downloads[i]) 519 if err != nil { 520 return false, err 521 } 522 } 523 } 524 525 var layersDownloaded bool 526 for i := len(downloads) - 1; i >= 0; i-- { 527 d := &downloads[i] 528 if d.err != nil { 529 err := <-d.err 530 if err != nil { 531 return false, err 532 } 533 } 534 if d.downloaded { 535 // if tmpFile is empty assume download and extracted elsewhere 536 defer os.Remove(d.tmpFile.Name()) 537 defer d.tmpFile.Close() 538 d.tmpFile.Seek(0, 0) 539 if d.tmpFile != nil { 540 err = s.graph.Register(d.img, 541 utils.ProgressReader(d.tmpFile, int(d.length), out, sf, false, utils.TruncateID(d.img.ID), "Extracting")) 542 if err != nil { 543 return false, err 544 } 545 546 // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) 547 } 548 out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Pull complete", nil)) 549 layersDownloaded = true 550 } else { 551 out.Write(sf.FormatProgress(utils.TruncateID(d.img.ID), "Already exists", nil)) 552 } 553 554 } 555 556 if err = s.Set(repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { 557 return false, err 558 } 559 560 return layersDownloaded, nil 561 }