github.com/adxhyt/docker@v1.4.2-0.20150117221845-467b7c821390/graph/push.go (about) 1 package graph 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "os" 8 "path" 9 "sync" 10 11 log "github.com/Sirupsen/logrus" 12 "github.com/docker/docker/engine" 13 "github.com/docker/docker/pkg/archive" 14 "github.com/docker/docker/registry" 15 "github.com/docker/docker/utils" 16 ) 17 18 // Retrieve the all the images to be uploaded in the correct order 19 func (s *TagStore) getImageList(localRepo map[string]string, requestedTag string) ([]string, map[string][]string, error) { 20 var ( 21 imageList []string 22 imagesSeen = make(map[string]bool) 23 tagsByImage = make(map[string][]string) 24 ) 25 26 for tag, id := range localRepo { 27 if requestedTag != "" && requestedTag != tag { 28 continue 29 } 30 var imageListForThisTag []string 31 32 tagsByImage[id] = append(tagsByImage[id], tag) 33 34 for img, err := s.graph.Get(id); img != nil; img, err = img.GetParent() { 35 if err != nil { 36 return nil, nil, err 37 } 38 39 if imagesSeen[img.ID] { 40 // This image is already on the list, we can ignore it and all its parents 41 break 42 } 43 44 imagesSeen[img.ID] = true 45 imageListForThisTag = append(imageListForThisTag, img.ID) 46 } 47 48 // reverse the image list for this tag (so the "most"-parent image is first) 49 for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 { 50 imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i] 51 } 52 53 // append to main image list 54 imageList = append(imageList, imageListForThisTag...) 55 } 56 if len(imageList) == 0 { 57 return nil, nil, fmt.Errorf("No images found for the requested repository / tag") 58 } 59 log.Debugf("Image list: %v", imageList) 60 log.Debugf("Tags by image: %v", tagsByImage) 61 62 return imageList, tagsByImage, nil 63 } 64 65 // createImageIndex returns an index of an image's layer IDs and tags. 66 func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData { 67 var imageIndex []*registry.ImgData 68 for _, id := range images { 69 if tags, hasTags := tags[id]; hasTags { 70 // If an image has tags you must add an entry in the image index 71 // for each tag 72 for _, tag := range tags { 73 imageIndex = append(imageIndex, ®istry.ImgData{ 74 ID: id, 75 Tag: tag, 76 }) 77 } 78 continue 79 } 80 // If the image does not have a tag it still needs to be sent to the 81 // registry with an empty tag so that it is accociated with the repository 82 imageIndex = append(imageIndex, ®istry.ImgData{ 83 ID: id, 84 Tag: "", 85 }) 86 } 87 return imageIndex 88 } 89 90 type imagePushData struct { 91 id string 92 endpoint string 93 tokens []string 94 } 95 96 // lookupImageOnEndpoint checks the specified endpoint to see if an image exists 97 // and if it is absent then it sends the image id to the channel to be pushed. 98 func lookupImageOnEndpoint(wg *sync.WaitGroup, r *registry.Session, out io.Writer, sf *utils.StreamFormatter, 99 images chan imagePushData, imagesToPush chan string) { 100 defer wg.Done() 101 for image := range images { 102 if err := r.LookupRemoteImage(image.id, image.endpoint, image.tokens); err != nil { 103 log.Errorf("Error in LookupRemoteImage: %s", err) 104 imagesToPush <- image.id 105 continue 106 } 107 out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", utils.TruncateID(image.id))) 108 } 109 } 110 111 func (s *TagStore) pushImageToEndpoint(endpoint string, out io.Writer, remoteName string, imageIDs []string, 112 tags map[string][]string, repo *registry.RepositoryData, sf *utils.StreamFormatter, r *registry.Session) error { 113 workerCount := len(imageIDs) 114 // start a maximum of 5 workers to check if images exist on the specified endpoint. 115 if workerCount > 5 { 116 workerCount = 5 117 } 118 var ( 119 wg = &sync.WaitGroup{} 120 imageData = make(chan imagePushData, workerCount*2) 121 imagesToPush = make(chan string, workerCount*2) 122 pushes = make(chan map[string]struct{}, 1) 123 ) 124 for i := 0; i < workerCount; i++ { 125 wg.Add(1) 126 go lookupImageOnEndpoint(wg, r, out, sf, imageData, imagesToPush) 127 } 128 // start a go routine that consumes the images to push 129 go func() { 130 shouldPush := make(map[string]struct{}) 131 for id := range imagesToPush { 132 shouldPush[id] = struct{}{} 133 } 134 pushes <- shouldPush 135 }() 136 for _, id := range imageIDs { 137 imageData <- imagePushData{ 138 id: id, 139 endpoint: endpoint, 140 tokens: repo.Tokens, 141 } 142 } 143 // close the channel to notify the workers that there will be no more images to check. 144 close(imageData) 145 wg.Wait() 146 close(imagesToPush) 147 // wait for all the images that require pushes to be collected into a consumable map. 148 shouldPush := <-pushes 149 // finish by pushing any images and tags to the endpoint. The order that the images are pushed 150 // is very important that is why we are still itterating over the ordered list of imageIDs. 151 for _, id := range imageIDs { 152 if _, push := shouldPush[id]; push { 153 if _, err := s.pushImage(r, out, id, endpoint, repo.Tokens, sf); err != nil { 154 // FIXME: Continue on error? 155 return err 156 } 157 } 158 for _, tag := range tags[id] { 159 out.Write(sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", utils.TruncateID(id), endpoint+"repositories/"+remoteName+"/tags/"+tag)) 160 if err := r.PushRegistryTag(remoteName, id, tag, endpoint, repo.Tokens); err != nil { 161 return err 162 } 163 } 164 } 165 return nil 166 } 167 168 // pushRepository pushes layers that do not already exist on the registry. 169 func (s *TagStore) pushRepository(r *registry.Session, out io.Writer, 170 repoInfo *registry.RepositoryInfo, localRepo map[string]string, 171 tag string, sf *utils.StreamFormatter) error { 172 log.Debugf("Local repo: %s", localRepo) 173 out = utils.NewWriteFlusher(out) 174 imgList, tags, err := s.getImageList(localRepo, tag) 175 if err != nil { 176 return err 177 } 178 out.Write(sf.FormatStatus("", "Sending image list")) 179 180 imageIndex := s.createImageIndex(imgList, tags) 181 log.Debugf("Preparing to push %s with the following images and tags", localRepo) 182 for _, data := range imageIndex { 183 log.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag) 184 } 185 // Register all the images in a repository with the registry 186 // If an image is not in this list it will not be associated with the repository 187 repoData, err := r.PushImageJSONIndex(repoInfo.RemoteName, imageIndex, false, nil) 188 if err != nil { 189 return err 190 } 191 nTag := 1 192 if tag == "" { 193 nTag = len(localRepo) 194 } 195 out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", repoInfo.CanonicalName, nTag)) 196 // push the repository to each of the endpoints only if it does not exist. 197 for _, endpoint := range repoData.Endpoints { 198 if err := s.pushImageToEndpoint(endpoint, out, repoInfo.RemoteName, imgList, tags, repoData, sf, r); err != nil { 199 return err 200 } 201 } 202 _, err = r.PushImageJSONIndex(repoInfo.RemoteName, imageIndex, true, repoData.Endpoints) 203 return err 204 } 205 206 func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { 207 out = utils.NewWriteFlusher(out) 208 jsonRaw, err := ioutil.ReadFile(path.Join(s.graph.Root, imgID, "json")) 209 if err != nil { 210 return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) 211 } 212 out.Write(sf.FormatProgress(utils.TruncateID(imgID), "Pushing", nil)) 213 214 imgData := ®istry.ImgData{ 215 ID: imgID, 216 } 217 218 // Send the json 219 if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { 220 if err == registry.ErrAlreadyExists { 221 out.Write(sf.FormatProgress(utils.TruncateID(imgData.ID), "Image already pushed, skipping", nil)) 222 return "", nil 223 } 224 return "", err 225 } 226 227 layerData, err := s.graph.TempLayerArchive(imgID, archive.Uncompressed, sf, out) 228 if err != nil { 229 return "", fmt.Errorf("Failed to generate layer archive: %s", err) 230 } 231 defer os.RemoveAll(layerData.Name()) 232 233 // Send the layer 234 log.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size) 235 236 checksum, checksumPayload, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf, false, utils.TruncateID(imgData.ID), "Pushing"), ep, token, jsonRaw) 237 if err != nil { 238 return "", err 239 } 240 imgData.Checksum = checksum 241 imgData.ChecksumPayload = checksumPayload 242 // Send the checksum 243 if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { 244 return "", err 245 } 246 247 out.Write(sf.FormatProgress(utils.TruncateID(imgData.ID), "Image successfully pushed", nil)) 248 return imgData.Checksum, nil 249 } 250 251 // FIXME: Allow to interrupt current push when new push of same image is done. 252 func (s *TagStore) CmdPush(job *engine.Job) engine.Status { 253 if n := len(job.Args); n != 1 { 254 return job.Errorf("Usage: %s IMAGE", job.Name) 255 } 256 var ( 257 localName = job.Args[0] 258 sf = utils.NewStreamFormatter(job.GetenvBool("json")) 259 authConfig = ®istry.AuthConfig{} 260 metaHeaders map[string][]string 261 ) 262 263 // Resolve the Repository name from fqn to RepositoryInfo 264 repoInfo, err := registry.ResolveRepositoryInfo(job, localName) 265 if err != nil { 266 return job.Error(err) 267 } 268 269 tag := job.Getenv("tag") 270 job.GetenvJson("authConfig", authConfig) 271 job.GetenvJson("metaHeaders", &metaHeaders) 272 273 if _, err := s.poolAdd("push", repoInfo.LocalName); err != nil { 274 return job.Error(err) 275 } 276 defer s.poolRemove("push", repoInfo.LocalName) 277 278 endpoint, err := repoInfo.GetEndpoint() 279 if err != nil { 280 return job.Error(err) 281 } 282 283 img, err := s.graph.Get(repoInfo.LocalName) 284 r, err2 := registry.NewSession(authConfig, registry.HTTPRequestFactory(metaHeaders), endpoint, false) 285 if err2 != nil { 286 return job.Error(err2) 287 } 288 289 if err != nil { 290 reposLen := 1 291 if tag == "" { 292 reposLen = len(s.Repositories[repoInfo.LocalName]) 293 } 294 job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) 295 // If it fails, try to get the repository 296 if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { 297 if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { 298 return job.Error(err) 299 } 300 return engine.StatusOK 301 } 302 return job.Error(err) 303 } 304 305 var token []string 306 job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) 307 if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { 308 return job.Error(err) 309 } 310 return engine.StatusOK 311 }