github.com/alok87/docker@v1.6.0/graph/push.go (about) 1 package graph 2 3 import ( 4 "encoding/json" 5 "errors" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "os" 10 "path" 11 "strings" 12 "sync" 13 14 log "github.com/Sirupsen/logrus" 15 "github.com/docker/docker/engine" 16 "github.com/docker/docker/image" 17 "github.com/docker/docker/pkg/common" 18 "github.com/docker/docker/pkg/progressreader" 19 "github.com/docker/docker/registry" 20 "github.com/docker/docker/runconfig" 21 "github.com/docker/docker/utils" 22 "github.com/docker/libtrust" 23 ) 24 25 var ErrV2RegistryUnavailable = errors.New("error v2 registry unavailable") 26 27 // Retrieve the all the images to be uploaded in the correct order 28 func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string) ([]string, map[string][]string, error) { 29 var ( 30 imageList []string 31 imagesSeen = make(map[string]bool) 32 tagsByImage = make(map[string][]string) 33 ) 34 35 for tag, id := range localRepo { 36 if requestedTag != "" && requestedTag != tag { 37 // Include only the requested tag. 38 continue 39 } 40 41 if utils.DigestReference(tag) { 42 // Ignore digest references. 43 continue 44 } 45 46 var imageListForThisTag []string 47 48 tagsByImage[id] = append(tagsByImage[id], tag) 49 50 for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() { 51 if err != nil { 52 return nil, nil, err 53 } 54 55 if imagesSeen[img.ID] { 56 // This image is already on the list, we can ignore it and all its parents 57 break 58 } 59 60 imagesSeen[img.ID] = true 61 imageListForThisTag = append(imageListForThisTag, img.ID) 62 } 63 64 // reverse the image list for this tag (so the "most"-parent image is first) 65 for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 { 66 imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i] 67 } 68 69 // append to main image list 70 imageList = append(imageList, imageListForThisTag...) 71 } 72 if len(imageList) == 0 { 73 return nil, nil, fmt.Errorf("No images found for the requested repository / tag") 74 } 75 log.Debugf("Image list: %v", imageList) 76 log.Debugf("Tags by image: %v", tagsByImage) 77 78 return imageList, tagsByImage, nil 79 } 80 81 func (s *TagStore) getImageTags(localRepo map[string]string, askedTag string) ([]string, error) { 82 log.Debugf("Checking %s against %#v", askedTag, localRepo) 83 if len(askedTag) > 0 { 84 if _, ok := localRepo[askedTag]; !ok || utils.DigestReference(askedTag) { 85 return nil, fmt.Errorf("Tag does not exist: %s", askedTag) 86 } 87 return []string{askedTag}, nil 88 } 89 var tags []string 90 for tag := range localRepo { 91 if !utils.DigestReference(tag) { 92 tags = append(tags, tag) 93 } 94 } 95 return tags, nil 96 } 97 98 // createImageIndex returns an index of an image's layer IDs and tags. 99 func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData { 100 var imageIndex []*registry.ImgData 101 for _, id := range images { 102 if tags, hasTags := tags[id]; hasTags { 103 // If an image has tags you must add an entry in the image index 104 // for each tag 105 for _, tag := range tags { 106 imageIndex = append(imageIndex, ®istry.ImgData{ 107 ID: id, 108 Tag: tag, 109 }) 110 } 111 continue 112 } 113 // If the image does not have a tag it still needs to be sent to the 114 // registry with an empty tag so that it is accociated with the repository 115 imageIndex = append(imageIndex, ®istry.ImgData{ 116 ID: id, 117 Tag: "", 118 }) 119 } 120 return imageIndex 121 } 122 123 type imagePushData struct { 124 id string 125 endpoint string 126 tokens []string 127 } 128 129 // lookupImageOnEndpoint checks the specified endpoint to see if an image exists 130 // and if it is absent then it sends the image id to the channel to be pushed. 131 func lookupImageOnEndpoint(wg *sync.WaitGroup, r *registry.Session, out io.Writer, sf *utils.StreamFormatter, 132 images chan imagePushData, imagesToPush chan string) { 133 defer wg.Done() 134 for image := range images { 135 if err := r.LookupRemoteImage(image.id, image.endpoint, image.tokens); err != nil { 136 log.Errorf("Error in LookupRemoteImage: %s", err) 137 imagesToPush <- image.id 138 continue 139 } 140 out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", common.TruncateID(image.id))) 141 } 142 } 143 144 func (s *TagStore) pushImageToEndpoint(endpoint string, out io.Writer, remoteName string, imageIDs []string, 145 tags map[string][]string, repo *registry.RepositoryData, sf *utils.StreamFormatter, r *registry.Session) error { 146 workerCount := len(imageIDs) 147 // start a maximum of 5 workers to check if images exist on the specified endpoint. 148 if workerCount > 5 { 149 workerCount = 5 150 } 151 var ( 152 wg = &sync.WaitGroup{} 153 imageData = make(chan imagePushData, workerCount*2) 154 imagesToPush = make(chan string, workerCount*2) 155 pushes = make(chan map[string]struct{}, 1) 156 ) 157 for i := 0; i < workerCount; i++ { 158 wg.Add(1) 159 go lookupImageOnEndpoint(wg, r, out, sf, imageData, imagesToPush) 160 } 161 // start a go routine that consumes the images to push 162 go func() { 163 shouldPush := make(map[string]struct{}) 164 for id := range imagesToPush { 165 shouldPush[id] = struct{}{} 166 } 167 pushes <- shouldPush 168 }() 169 for _, id := range imageIDs { 170 imageData <- imagePushData{ 171 id: id, 172 endpoint: endpoint, 173 tokens: repo.Tokens, 174 } 175 } 176 // close the channel to notify the workers that there will be no more images to check. 177 close(imageData) 178 wg.Wait() 179 close(imagesToPush) 180 // wait for all the images that require pushes to be collected into a consumable map. 181 shouldPush := <-pushes 182 // finish by pushing any images and tags to the endpoint. The order that the images are pushed 183 // is very important that is why we are still iterating over the ordered list of imageIDs. 184 for _, id := range imageIDs { 185 if _, push := shouldPush[id]; push { 186 if _, err := s.pushImage(r, out, id, endpoint, repo.Tokens, sf); err != nil { 187 // FIXME: Continue on error? 188 return err 189 } 190 } 191 for _, tag := range tags[id] { 192 out.Write(sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", common.TruncateID(id), endpoint+"repositories/"+remoteName+"/tags/"+tag)) 193 if err := r.PushRegistryTag(remoteName, id, tag, endpoint, repo.Tokens); err != nil { 194 return err 195 } 196 } 197 } 198 return nil 199 } 200 201 // pushRepository pushes layers that do not already exist on the registry. 202 func (s *TagStore) pushRepository(r *registry.Session, out io.Writer, 203 repoInfo *registry.RepositoryInfo, localRepo map[string]string, 204 tag string, sf *utils.StreamFormatter) error { 205 log.Debugf("Local repo: %s", localRepo) 206 out = utils.NewWriteFlusher(out) 207 imgList, tags, err := s.getImageList(localRepo, tag) 208 if err != nil { 209 return err 210 } 211 out.Write(sf.FormatStatus("", "Sending image list")) 212 213 imageIndex := s.createImageIndex(imgList, tags) 214 log.Debugf("Preparing to push %s with the following images and tags", localRepo) 215 for _, data := range imageIndex { 216 log.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag) 217 } 218 // Register all the images in a repository with the registry 219 // If an image is not in this list it will not be associated with the repository 220 repoData, err := r.PushImageJSONIndex(repoInfo.RemoteName, imageIndex, false, nil) 221 if err != nil { 222 return err 223 } 224 nTag := 1 225 if tag == "" { 226 nTag = len(localRepo) 227 } 228 out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", repoInfo.CanonicalName, nTag)) 229 // push the repository to each of the endpoints only if it does not exist. 230 for _, endpoint := range repoData.Endpoints { 231 if err := s.pushImageToEndpoint(endpoint, out, repoInfo.RemoteName, imgList, tags, repoData, sf, r); err != nil { 232 return err 233 } 234 } 235 _, err = r.PushImageJSONIndex(repoInfo.RemoteName, imageIndex, true, repoData.Endpoints) 236 return err 237 } 238 239 func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { 240 out = utils.NewWriteFlusher(out) 241 jsonRaw, err := ioutil.ReadFile(path.Join(s.graph.Root, imgID, "json")) 242 if err != nil { 243 return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) 244 } 245 out.Write(sf.FormatProgress(common.TruncateID(imgID), "Pushing", nil)) 246 247 imgData := ®istry.ImgData{ 248 ID: imgID, 249 } 250 251 // Send the json 252 if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { 253 if err == registry.ErrAlreadyExists { 254 out.Write(sf.FormatProgress(common.TruncateID(imgData.ID), "Image already pushed, skipping", nil)) 255 return "", nil 256 } 257 return "", err 258 } 259 260 layerData, err := s.graph.TempLayerArchive(imgID, sf, out) 261 if err != nil { 262 return "", fmt.Errorf("Failed to generate layer archive: %s", err) 263 } 264 defer os.RemoveAll(layerData.Name()) 265 266 // Send the layer 267 log.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size) 268 269 checksum, checksumPayload, err := r.PushImageLayerRegistry(imgData.ID, 270 progressreader.New(progressreader.Config{ 271 In: layerData, 272 Out: out, 273 Formatter: sf, 274 Size: int(layerData.Size), 275 NewLines: false, 276 ID: common.TruncateID(imgData.ID), 277 Action: "Pushing", 278 }), ep, token, jsonRaw) 279 if err != nil { 280 return "", err 281 } 282 imgData.Checksum = checksum 283 imgData.ChecksumPayload = checksumPayload 284 // Send the checksum 285 if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { 286 return "", err 287 } 288 289 out.Write(sf.FormatProgress(common.TruncateID(imgData.ID), "Image successfully pushed", nil)) 290 return imgData.Checksum, nil 291 } 292 293 func (s *TagStore) pushV2Repository(r *registry.Session, localRepo Repository, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter) error { 294 endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) 295 if err != nil { 296 if repoInfo.Index.Official { 297 log.Debugf("Unable to push to V2 registry, falling back to v1: %s", err) 298 return ErrV2RegistryUnavailable 299 } 300 return fmt.Errorf("error getting registry endpoint: %s", err) 301 } 302 303 tags, err := s.getImageTags(localRepo, tag) 304 if err != nil { 305 return err 306 } 307 if len(tags) == 0 { 308 return fmt.Errorf("No tags to push for %s", repoInfo.LocalName) 309 } 310 311 auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, false) 312 if err != nil { 313 return fmt.Errorf("error getting authorization: %s", err) 314 } 315 316 for _, tag := range tags { 317 log.Debugf("Pushing repository: %s:%s", repoInfo.CanonicalName, tag) 318 319 layerId, exists := localRepo[tag] 320 if !exists { 321 return fmt.Errorf("tag does not exist: %s", tag) 322 } 323 324 layer, err := s.graph.Get(layerId) 325 if err != nil { 326 return err 327 } 328 329 m := ®istry.ManifestData{ 330 SchemaVersion: 1, 331 Name: repoInfo.RemoteName, 332 Tag: tag, 333 Architecture: layer.Architecture, 334 } 335 var metadata runconfig.Config 336 if layer.Config != nil { 337 metadata = *layer.Config 338 } 339 340 layersSeen := make(map[string]bool) 341 layers := []*image.Image{layer} 342 for ; layer != nil; layer, err = layer.GetParent() { 343 if err != nil { 344 return err 345 } 346 347 if layersSeen[layer.ID] { 348 break 349 } 350 layers = append(layers, layer) 351 layersSeen[layer.ID] = true 352 } 353 m.FSLayers = make([]*registry.FSLayer, len(layers)) 354 m.History = make([]*registry.ManifestHistory, len(layers)) 355 356 // Schema version 1 requires layer ordering from top to root 357 for i, layer := range layers { 358 log.Debugf("Pushing layer: %s", layer.ID) 359 360 if layer.Config != nil && metadata.Image != layer.ID { 361 err = runconfig.Merge(&metadata, layer.Config) 362 if err != nil { 363 return err 364 } 365 } 366 jsonData, err := layer.RawJson() 367 if err != nil { 368 return fmt.Errorf("cannot retrieve the path for %s: %s", layer.ID, err) 369 } 370 371 checksum, err := layer.GetCheckSum(s.graph.ImageRoot(layer.ID)) 372 if err != nil { 373 return fmt.Errorf("error getting image checksum: %s", err) 374 } 375 376 var exists bool 377 if len(checksum) > 0 { 378 sumParts := strings.SplitN(checksum, ":", 2) 379 if len(sumParts) < 2 { 380 return fmt.Errorf("Invalid checksum: %s", checksum) 381 } 382 383 // Call mount blob 384 exists, err = r.HeadV2ImageBlob(endpoint, repoInfo.RemoteName, sumParts[0], sumParts[1], auth) 385 if err != nil { 386 out.Write(sf.FormatProgress(common.TruncateID(layer.ID), "Image push failed", nil)) 387 return err 388 } 389 } 390 if !exists { 391 if cs, err := s.pushV2Image(r, layer, endpoint, repoInfo.RemoteName, sf, out, auth); err != nil { 392 return err 393 } else if cs != checksum { 394 // Cache new checksum 395 if err := layer.SaveCheckSum(s.graph.ImageRoot(layer.ID), cs); err != nil { 396 return err 397 } 398 checksum = cs 399 } 400 } else { 401 out.Write(sf.FormatProgress(common.TruncateID(layer.ID), "Image already exists", nil)) 402 } 403 m.FSLayers[i] = ®istry.FSLayer{BlobSum: checksum} 404 m.History[i] = ®istry.ManifestHistory{V1Compatibility: string(jsonData)} 405 } 406 407 if err := checkValidManifest(m); err != nil { 408 return fmt.Errorf("invalid manifest: %s", err) 409 } 410 411 log.Debugf("Pushing %s:%s to v2 repository", repoInfo.LocalName, tag) 412 mBytes, err := json.MarshalIndent(m, "", " ") 413 if err != nil { 414 return err 415 } 416 js, err := libtrust.NewJSONSignature(mBytes) 417 if err != nil { 418 return err 419 } 420 421 if err = js.Sign(s.trustKey); err != nil { 422 return err 423 } 424 425 signedBody, err := js.PrettySignature("signatures") 426 if err != nil { 427 return err 428 } 429 log.Infof("Signed manifest for %s:%s using daemon's key: %s", repoInfo.LocalName, tag, s.trustKey.KeyID()) 430 431 // push the manifest 432 digest, err := r.PutV2ImageManifest(endpoint, repoInfo.RemoteName, tag, signedBody, mBytes, auth) 433 if err != nil { 434 return err 435 } 436 437 out.Write(sf.FormatStatus("", "Digest: %s", digest)) 438 } 439 return nil 440 } 441 442 // PushV2Image pushes the image content to the v2 registry, first buffering the contents to disk 443 func (s *TagStore) pushV2Image(r *registry.Session, img *image.Image, endpoint *registry.Endpoint, imageName string, sf *utils.StreamFormatter, out io.Writer, auth *registry.RequestAuthorization) (string, error) { 444 out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Buffering to Disk", nil)) 445 446 image, err := s.graph.Get(img.ID) 447 if err != nil { 448 return "", err 449 } 450 arch, err := image.TarLayer() 451 if err != nil { 452 return "", err 453 } 454 defer arch.Close() 455 456 tf, err := s.graph.newTempFile() 457 if err != nil { 458 return "", err 459 } 460 defer func() { 461 tf.Close() 462 os.Remove(tf.Name()) 463 }() 464 465 size, dgst, err := bufferToFile(tf, arch) 466 467 // Send the layer 468 log.Debugf("rendered layer for %s of [%d] size", img.ID, size) 469 470 if err := r.PutV2ImageBlob(endpoint, imageName, dgst.Algorithm(), dgst.Hex(), 471 progressreader.New(progressreader.Config{ 472 In: tf, 473 Out: out, 474 Formatter: sf, 475 Size: int(size), 476 NewLines: false, 477 ID: common.TruncateID(img.ID), 478 Action: "Pushing", 479 }), auth); err != nil { 480 out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Image push failed", nil)) 481 return "", err 482 } 483 out.Write(sf.FormatProgress(common.TruncateID(img.ID), "Image successfully pushed", nil)) 484 return dgst.String(), nil 485 } 486 487 // FIXME: Allow to interrupt current push when new push of same image is done. 488 func (s *TagStore) CmdPush(job *engine.Job) engine.Status { 489 if n := len(job.Args); n != 1 { 490 return job.Errorf("Usage: %s IMAGE", job.Name) 491 } 492 var ( 493 localName = job.Args[0] 494 sf = utils.NewStreamFormatter(job.GetenvBool("json")) 495 authConfig = ®istry.AuthConfig{} 496 metaHeaders map[string][]string 497 ) 498 499 // Resolve the Repository name from fqn to RepositoryInfo 500 repoInfo, err := registry.ResolveRepositoryInfo(job, localName) 501 if err != nil { 502 return job.Error(err) 503 } 504 505 tag := job.Getenv("tag") 506 job.GetenvJson("authConfig", authConfig) 507 job.GetenvJson("metaHeaders", &metaHeaders) 508 509 if _, err := s.poolAdd("push", repoInfo.LocalName); err != nil { 510 return job.Error(err) 511 } 512 defer s.poolRemove("push", repoInfo.LocalName) 513 514 endpoint, err := repoInfo.GetEndpoint() 515 if err != nil { 516 return job.Error(err) 517 } 518 519 r, err := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, false) 520 if err != nil { 521 return job.Error(err) 522 } 523 524 reposLen := 1 525 if tag == "" { 526 reposLen = len(s.Repositories[repoInfo.LocalName]) 527 } 528 job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) 529 // If it fails, try to get the repository 530 localRepo, exists := s.Repositories[repoInfo.LocalName] 531 if !exists { 532 return job.Errorf("Repository does not exist: %s", repoInfo.LocalName) 533 } 534 535 if repoInfo.Index.Official || endpoint.Version == registry.APIVersion2 { 536 err := s.pushV2Repository(r, localRepo, job.Stdout, repoInfo, tag, sf) 537 if err == nil { 538 return engine.StatusOK 539 } 540 541 if err != ErrV2RegistryUnavailable { 542 return job.Errorf("Error pushing to registry: %s", err) 543 } 544 } 545 546 if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { 547 return job.Error(err) 548 } 549 return engine.StatusOK 550 551 }