github.com/fmzhen/docker@v1.6.0-rc2/graph/push.go (about) 1 package graph 2 3 import ( 4 "crypto/sha256" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "io" 9 "io/ioutil" 10 "os" 11 "path" 12 "strings" 13 "sync" 14 15 log "github.com/Sirupsen/logrus" 16 "github.com/docker/distribution/digest" 17 "github.com/docker/docker/engine" 18 "github.com/docker/docker/image" 19 "github.com/docker/docker/pkg/common" 20 "github.com/docker/docker/pkg/progressreader" 21 "github.com/docker/docker/registry" 22 "github.com/docker/docker/runconfig" 23 "github.com/docker/docker/utils" 24 "github.com/docker/libtrust" 25 ) 26 27 var ErrV2RegistryUnavailable = errors.New("error v2 registry unavailable") 28 29 // Retrieve the all the images to be uploaded in the correct order 30 func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string) ([]string, map[string][]string, error) { 31 var ( 32 imageList []string 33 imagesSeen = make(map[string]bool) 34 tagsByImage = make(map[string][]string) 35 ) 36 37 for tag, id := range localRepo { 38 if requestedTag != "" && requestedTag != tag { 39 // Include only the requested tag. 40 continue 41 } 42 43 if utils.DigestReference(tag) { 44 // Ignore digest references. 45 continue 46 } 47 48 var imageListForThisTag []string 49 50 tagsByImage[id] = append(tagsByImage[id], tag) 51 52 for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() { 53 if err != nil { 54 return nil, nil, err 55 } 56 57 if imagesSeen[img.ID] { 58 // This image is already on the list, we can ignore it and all its parents 59 break 60 } 61 62 imagesSeen[img.ID] = true 63 imageListForThisTag = append(imageListForThisTag, img.ID) 64 } 65 66 // reverse the image list for this tag (so the "most"-parent image is first) 67 for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 { 68 imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i] 69 } 70 71 // append to main image list 72 imageList = append(imageList, imageListForThisTag...) 73 } 74 if len(imageList) == 0 { 75 return nil, nil, fmt.Errorf("No images found for the requested repository / tag") 76 } 77 log.Debugf("Image list: %v", imageList) 78 log.Debugf("Tags by image: %v", tagsByImage) 79 80 return imageList, tagsByImage, nil 81 } 82 83 func (s *TagStore) getImageTags(localRepo map[string]string, askedTag string) ([]string, error) { 84 log.Debugf("Checking %s against %#v", askedTag, localRepo) 85 if len(askedTag) > 0 { 86 if _, ok := localRepo[askedTag]; !ok || utils.DigestReference(askedTag) { 87 return nil, fmt.Errorf("Tag does not exist: %s", askedTag) 88 } 89 return []string{askedTag}, nil 90 } 91 var tags []string 92 for tag := range localRepo { 93 if !utils.DigestReference(tag) { 94 tags = append(tags, tag) 95 } 96 } 97 return tags, nil 98 } 99 100 // createImageIndex returns an index of an image's layer IDs and tags. 101 func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData { 102 var imageIndex []*registry.ImgData 103 for _, id := range images { 104 if tags, hasTags := tags[id]; hasTags { 105 // If an image has tags you must add an entry in the image index 106 // for each tag 107 for _, tag := range tags { 108 imageIndex = append(imageIndex, ®istry.ImgData{ 109 ID: id, 110 Tag: tag, 111 }) 112 } 113 continue 114 } 115 // If the image does not have a tag it still needs to be sent to the 116 // registry with an empty tag so that it is accociated with the repository 117 imageIndex = append(imageIndex, ®istry.ImgData{ 118 ID: id, 119 Tag: "", 120 }) 121 } 122 return imageIndex 123 } 124 125 type imagePushData struct { 126 id string 127 endpoint string 128 tokens []string 129 } 130 131 // lookupImageOnEndpoint checks the specified endpoint to see if an image exists 132 // and if it is absent then it sends the image id to the channel to be pushed. 133 func lookupImageOnEndpoint(wg *sync.WaitGroup, r *registry.Session, out io.Writer, sf *utils.StreamFormatter, 134 images chan imagePushData, imagesToPush chan string) { 135 defer wg.Done() 136 for image := range images { 137 if err := r.LookupRemoteImage(image.id, image.endpoint, image.tokens); err != nil { 138 log.Errorf("Error in LookupRemoteImage: %s", err) 139 imagesToPush <- image.id 140 continue 141 } 142 out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", common.TruncateID(image.id))) 143 } 144 } 145 146 func (s *TagStore) pushImageToEndpoint(endpoint string, out io.Writer, remoteName string, imageIDs []string, 147 tags map[string][]string, repo *registry.RepositoryData, sf *utils.StreamFormatter, r *registry.Session) error { 148 workerCount := len(imageIDs) 149 // start a maximum of 5 workers to check if images exist on the specified endpoint. 150 if workerCount > 5 { 151 workerCount = 5 152 } 153 var ( 154 wg = &sync.WaitGroup{} 155 imageData = make(chan imagePushData, workerCount*2) 156 imagesToPush = make(chan string, workerCount*2) 157 pushes = make(chan map[string]struct{}, 1) 158 ) 159 for i := 0; i < workerCount; i++ { 160 wg.Add(1) 161 go lookupImageOnEndpoint(wg, r, out, sf, imageData, imagesToPush) 162 } 163 // start a go routine that consumes the images to push 164 go func() { 165 shouldPush := make(map[string]struct{}) 166 for id := range imagesToPush { 167 shouldPush[id] = struct{}{} 168 } 169 pushes <- shouldPush 170 }() 171 for _, id := range imageIDs { 172 imageData <- imagePushData{ 173 id: id, 174 endpoint: endpoint, 175 tokens: repo.Tokens, 176 } 177 } 178 // close the channel to notify the workers that there will be no more images to check. 179 close(imageData) 180 wg.Wait() 181 close(imagesToPush) 182 // wait for all the images that require pushes to be collected into a consumable map. 183 shouldPush := <-pushes 184 // finish by pushing any images and tags to the endpoint. The order that the images are pushed 185 // is very important that is why we are still iterating over the ordered list of imageIDs. 186 for _, id := range imageIDs { 187 if _, push := shouldPush[id]; push { 188 if _, err := s.pushImage(r, out, id, endpoint, repo.Tokens, sf); err != nil { 189 // FIXME: Continue on error? 190 return err 191 } 192 } 193 for _, tag := range tags[id] { 194 out.Write(sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", common.TruncateID(id), endpoint+"repositories/"+remoteName+"/tags/"+tag)) 195 if err := r.PushRegistryTag(remoteName, id, tag, endpoint, repo.Tokens); err != nil { 196 return err 197 } 198 } 199 } 200 return nil 201 } 202 203 // pushRepository pushes layers that do not already exist on the registry. 204 func (s *TagStore) pushRepository(r *registry.Session, out io.Writer, 205 repoInfo *registry.RepositoryInfo, localRepo map[string]string, 206 tag string, sf *utils.StreamFormatter) error { 207 log.Debugf("Local repo: %s", localRepo) 208 out = utils.NewWriteFlusher(out) 209 imgList, tags, err := s.getImageList(localRepo, tag) 210 if err != nil { 211 return err 212 } 213 out.Write(sf.FormatStatus("", "Sending image list")) 214 215 imageIndex := s.createImageIndex(imgList, tags) 216 log.Debugf("Preparing to push %s with the following images and tags", localRepo) 217 for _, data := range imageIndex { 218 log.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag) 219 } 220 // Register all the images in a repository with the registry 221 // If an image is not in this list it will not be associated with the repository 222 repoData, err := r.PushImageJSONIndex(repoInfo.RemoteName, imageIndex, false, nil) 223 if err != nil { 224 return err 225 } 226 nTag := 1 227 if tag == "" { 228 nTag = len(localRepo) 229 } 230 out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", repoInfo.CanonicalName, nTag)) 231 // push the repository to each of the endpoints only if it does not exist. 232 for _, endpoint := range repoData.Endpoints { 233 if err := s.pushImageToEndpoint(endpoint, out, repoInfo.RemoteName, imgList, tags, repoData, sf, r); err != nil { 234 return err 235 } 236 } 237 _, err = r.PushImageJSONIndex(repoInfo.RemoteName, imageIndex, true, repoData.Endpoints) 238 return err 239 } 240 241 func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { 242 out = utils.NewWriteFlusher(out) 243 jsonRaw, err := ioutil.ReadFile(path.Join(s.graph.Root, imgID, "json")) 244 if err != nil { 245 return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) 246 } 247 out.Write(sf.FormatProgress(common.TruncateID(imgID), "Pushing", nil)) 248 249 imgData := ®istry.ImgData{ 250 ID: imgID, 251 } 252 253 // Send the json 254 if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { 255 if err == registry.ErrAlreadyExists { 256 out.Write(sf.FormatProgress(common.TruncateID(imgData.ID), "Image already pushed, skipping", nil)) 257 return "", nil 258 } 259 return "", err 260 } 261 262 layerData, err := s.graph.TempLayerArchive(imgID, sf, out) 263 if err != nil { 264 return "", fmt.Errorf("Failed to generate layer archive: %s", err) 265 } 266 defer os.RemoveAll(layerData.Name()) 267 268 // Send the layer 269 log.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size) 270 271 checksum, checksumPayload, err := r.PushImageLayerRegistry(imgData.ID, 272 progressreader.New(progressreader.Config{ 273 In: layerData, 274 Out: out, 275 Formatter: sf, 276 Size: int(layerData.Size), 277 NewLines: false, 278 ID: common.TruncateID(imgData.ID), 279 Action: "Pushing", 280 }), ep, token, jsonRaw) 281 if err != nil { 282 return "", err 283 } 284 imgData.Checksum = checksum 285 imgData.ChecksumPayload = checksumPayload 286 // Send the checksum 287 if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { 288 return "", err 289 } 290 291 out.Write(sf.FormatProgress(common.TruncateID(imgData.ID), "Image successfully pushed", nil)) 292 return imgData.Checksum, nil 293 } 294 295 func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter) error { 296 endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) 297 if err != nil { 298 if repoInfo.Index.Official { 299 log.Debugf("Unable to push to V2 registry, falling back to v1: %s", err) 300 return ErrV2RegistryUnavailable 301 } 302 return fmt.Errorf("error getting registry endpoint: %s", err) 303 } 304 305 tags, err := s.getImageTags(localRepo, tag) 306 if err != nil { 307 return err 308 } 309 if len(tags) == 0 { 310 return fmt.Errorf("No tags to push for %s", repoInfo.LocalName) 311 } 312 313 auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, false) 314 if err != nil { 315 return fmt.Errorf("error getting authorization: %s", err) 316 } 317 318 for _, tag := range tags { 319 log.Debugf("Pushing repository: %s:%s", repoInfo.CanonicalName, tag) 320 321 layerId, exists := localRepo[tag] 322 if !exists { 323 return fmt.Errorf("tag does not exist: %s", tag) 324 } 325 326 layer, err := s.graph.Get(layerId) 327 if err != nil { 328 return err 329 } 330 331 m := ®istry.ManifestData{ 332 SchemaVersion: 1, 333 Name: repoInfo.RemoteName, 334 Tag: tag, 335 Architecture: layer.Architecture, 336 } 337 var metadata runconfig.Config 338 if layer.Config != nil { 339 metadata = *layer.Config 340 } 341 342 layersSeen := make(map[string]bool) 343 layers := []*image.Image{layer} 344 for ; layer != nil; layer, err = layer.GetParent() { 345 if err != nil { 346 return err 347 } 348 349 if layersSeen[layer.ID] { 350 break 351 } 352 layers = append(layers, layer) 353 layersSeen[layer.ID] = true 354 } 355 m.FSLayers = make([]*registry.FSLayer, len(layers)) 356 m.History = make([]*registry.ManifestHistory, len(layers)) 357 358 // Schema version 1 requires layer ordering from top to root 359 for i, layer := range layers { 360 log.Debugf("Pushing layer: %s", layer.ID) 361 362 if layer.Config != nil && metadata.Image != layer.ID { 363 err = runconfig.Merge(&metadata, layer.Config) 364 if err != nil { 365 return err 366 } 367 } 368 jsonData, err := layer.RawJson() 369 if err != nil { 370 return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err) 371 } 372 373 checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID)) 374 if err != nil { 375 return fmt.Errorf("error getting image checksum: %s", err) 376 } 377 378 var exists bool 379 if len(checksum) > 0 { 380 sumParts := strings.SplitN(checksum, ":", 2) 381 if len(sumParts) < 2 { 382 return fmt.Errorf("Invalid checksum: %s", checksum) 383 } 384 385 // Call mount blob 386 exists, err = r.HeadV2ImageBlob(endpoint, repoInfo.RemoteName, sumParts[0], sumParts[1], auth) 387 if err != nil { 388 out.Write(sf.FormatProgress(common.TruncateID(layer.ID), "Image push failed", nil)) 389 return err 390 } 391 } 392 if !exists { 393 if cs, err := s.pushV2Image(r, layer, endpoint, repoInfo.RemoteName, sf, out, auth); err != nil { 394 return err 395 } else if cs != checksum { 396 // Cache new checksum 397 if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), cs); err != nil { 398 return err 399 } 400 checksum = cs 401 } 402 } else { 403 out.Write(sf.FormatProgress(common.TruncateID(layer.ID), "Image already exists", nil)) 404 } 405 m.FSLayers[i] = ®istry.FSLayer{BlobSum: checksum} 406 m.History[i] = ®istry.ManifestHistory{V1Compatibility: string(jsonData)} 407 } 408 409 if err := checkValidManifest(m); err != nil { 410 return fmt.Errorf("invalid manifest: %s", err) 411 } 412 413 log.Debugf("Pushing %s:%s to v2 repository", repoInfo.LocalName, tag) 414 mBytes, err := json.MarshalIndent(m, "", " ") 415 if err != nil { 416 return err 417 } 418 js, err := libtrust.NewJSONSignature(mBytes) 419 if err != nil { 420 return err 421 } 422 423 if err = js.Sign(s.trustKey); err != nil { 424 return err 425 } 426 427 signedBody, err := js.PrettySignature("signatures") 428 if err != nil { 429 return err 430 } 431 log.Infof("Signed manifest for %s:%s using daemon's key: %s", repoInfo.LocalName, tag, s.trustKey.KeyID()) 432 433 // push the manifest 434 digest, err := r.PutV2ImageManifest(endpoint, repoInfo.RemoteName, tag, signedBody, mBytes, auth) 435 if err != nil { 436 return err 437 } 438 439 out.Write(sf.FormatStatus("", "Digest: %s", digest)) 440 } 441 return nil 442 } 443 444 // PushV2Image pushes the image content to the v2 registry, first buffering the contents to disk 445 func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint *registry.Endpoint, imageName string, sf *utils.StreamFormatter, out io.Writer, auth *registry.RequestAuthorization) (string, error) { 446 out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Buffering to Disk", nil)) 447 448 image, err := s.graph.Get(img.ID) 449 if err != nil { 450 return "", err 451 } 452 arch, err := image.TarLayer() 453 if err != nil { 454 return "", err 455 } 456 defer arch.Close() 457 458 tf, err := s.graph.newTempFile() 459 if err != nil { 460 return "", err 461 } 462 defer func() { 463 tf.Close() 464 os.Remove(tf.Name()) 465 }() 466 467 h := sha256.New() 468 size, err := bufferToFile(tf, io.TeeReader(arch, h)) 469 if err != nil { 470 return "", err 471 } 472 dgst := digest.NewDigest("sha256", h) 473 474 // Send the layer 475 log.Debugf("rendered layer for %s of [%d] size", img.ID, size) 476 477 if err := r.PutV2ImageBlob(endpoint, imageName, dgst.Algorithm(), dgst.Hex(), 478 progressreader.New(progressreader.Config{ 479 In: tf, 480 Out: out, 481 Formatter: sf, 482 Size: int(size), 483 NewLines: false, 484 ID: common.TruncateID(img.ID), 485 Action: "Pushing", 486 }), auth); err != nil { 487 out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Image push failed", nil)) 488 return "", err 489 } 490 out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Image successfully pushed", nil)) 491 return dgst.String(), nil 492 } 493 494 // FIXME: Allow to interrupt current push when new push of same image is done. 495 func (s *TagStore) CmdPush(job *engine.Job) engine.Status { 496 if n := len(job.Args); n != 1 { 497 return job.Errorf("Usage: %s IMAGE", job.Name) 498 } 499 var ( 500 localName = job.Args[0] 501 sf = utils.NewStreamFormatter(job.GetenvBool("json")) 502 authConfig = ®istry.AuthConfig{} 503 metaHeaders map[string][]string 504 ) 505 506 // Resolve the Repository name from fqn to RepositoryInfo 507 repoInfo, err := registry.ResolveRepositoryInfo(job, localName) 508 if err != nil { 509 return job.Error(err) 510 } 511 512 tag := job.Getenv("tag") 513 job.GetenvJson("authConfig", authConfig) 514 job.GetenvJson("metaHeaders", &metaHeaders) 515 516 if _, err := s.poolAdd("push", repoInfo.LocalName); err != nil { 517 return job.Error(err) 518 } 519 defer s.poolRemove("push", repoInfo.LocalName) 520 521 endpoint, err := repoInfo.GetEndpoint() 522 if err != nil { 523 return job.Error(err) 524 } 525 526 r, err := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, false) 527 if err != nil { 528 return job.Error(err) 529 } 530 531 reposLen := 1 532 if tag == "" { 533 reposLen = len(s.Repositories[repoInfo.LocalName]) 534 } 535 job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) 536 // If it fails, try to get the repository 537 localRepo, exists := s.Repositories[repoInfo.LocalName] 538 if !exists { 539 return job.Errorf("Repository does not exist: %s", repoInfo.LocalName) 540 } 541 542 if repoInfo.Index.Official || endpoint.Version == registry.APIVersion2 { 543 err := s.pushV2Repository(r, localRepo, job.Stdout, repoInfo, tag, sf) 544 if err == nil { 545 return engine.StatusOK 546 } 547 548 if err != ErrV2RegistryUnavailable { 549 return job.Errorf("Error pushing to registry: %s", err) 550 } 551 } 552 553 if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { 554 return job.Error(err) 555 } 556 return engine.StatusOK 557 558 }