github.com/mheon/docker@v0.11.2-0.20150922122814-44f47903a831/graph/push_v1.go (about) 1 package graph 2 3 import ( 4 "fmt" 5 "io" 6 "os" 7 "sync" 8 9 "github.com/Sirupsen/logrus" 10 "github.com/docker/distribution/registry/client/transport" 11 "github.com/docker/docker/pkg/ioutils" 12 "github.com/docker/docker/pkg/progressreader" 13 "github.com/docker/docker/pkg/streamformatter" 14 "github.com/docker/docker/pkg/stringid" 15 "github.com/docker/docker/registry" 16 "github.com/docker/docker/utils" 17 ) 18 19 type v1Pusher struct { 20 *TagStore 21 endpoint registry.APIEndpoint 22 localRepo Repository 23 repoInfo *registry.RepositoryInfo 24 config *ImagePushConfig 25 sf *streamformatter.StreamFormatter 26 session *registry.Session 27 28 out io.Writer 29 } 30 31 func (p *v1Pusher) Push() (fallback bool, err error) { 32 tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name) 33 if err != nil { 34 return false, err 35 } 36 // Adds Docker-specific headers as well as user-specified headers (metaHeaders) 37 tr := transport.NewTransport( 38 // TODO(tiborvass): was NoTimeout 39 registry.NewTransport(tlsConfig), 40 registry.DockerHeaders(p.config.MetaHeaders)..., 41 ) 42 client := registry.HTTPClient(tr) 43 v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders) 44 if err != nil { 45 logrus.Debugf("Could not get v1 endpoint: %v", err) 46 return true, err 47 } 48 p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint) 49 if err != nil { 50 // TODO(dmcgowan): Check if should fallback 51 return true, err 52 } 53 if err := p.pushRepository(p.config.Tag); err != nil { 54 // TODO(dmcgowan): Check if should fallback 55 return false, err 56 } 57 return false, nil 58 } 59 60 // Retrieve the all the images to be uploaded in the correct order 61 func (p *v1Pusher) getImageList(requestedTag string) ([]string, map[string][]string, error) { 62 var ( 63 imageList []string 64 imagesSeen = make(map[string]bool) 65 tagsByImage = make(map[string][]string) 66 ) 67 68 for tag, id := range p.localRepo { 69 if requestedTag != "" && requestedTag != tag { 70 // Include only the requested tag. 71 continue 72 } 73 74 if utils.DigestReference(tag) { 75 // Ignore digest references. 76 continue 77 } 78 79 var imageListForThisTag []string 80 81 tagsByImage[id] = append(tagsByImage[id], tag) 82 83 for img, err := p.graph.Get(id); img != nil; img, err = p.graph.GetParent(img) { 84 if err != nil { 85 return nil, nil, err 86 } 87 88 if imagesSeen[img.ID] { 89 // This image is already on the list, we can ignore it and all its parents 90 break 91 } 92 93 imagesSeen[img.ID] = true 94 imageListForThisTag = append(imageListForThisTag, img.ID) 95 } 96 97 // reverse the image list for this tag (so the "most"-parent image is first) 98 for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 { 99 imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i] 100 } 101 102 // append to main image list 103 imageList = append(imageList, imageListForThisTag...) 104 } 105 if len(imageList) == 0 { 106 return nil, nil, fmt.Errorf("No images found for the requested repository / tag") 107 } 108 logrus.Debugf("Image list: %v", imageList) 109 logrus.Debugf("Tags by image: %v", tagsByImage) 110 111 return imageList, tagsByImage, nil 112 } 113 114 // createImageIndex returns an index of an image's layer IDs and tags. 115 func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData { 116 var imageIndex []*registry.ImgData 117 for _, id := range images { 118 if tags, hasTags := tags[id]; hasTags { 119 // If an image has tags you must add an entry in the image index 120 // for each tag 121 for _, tag := range tags { 122 imageIndex = append(imageIndex, ®istry.ImgData{ 123 ID: id, 124 Tag: tag, 125 }) 126 } 127 continue 128 } 129 // If the image does not have a tag it still needs to be sent to the 130 // registry with an empty tag so that it is accociated with the repository 131 imageIndex = append(imageIndex, ®istry.ImgData{ 132 ID: id, 133 Tag: "", 134 }) 135 } 136 return imageIndex 137 } 138 139 type imagePushData struct { 140 id string 141 endpoint string 142 } 143 144 // lookupImageOnEndpoint checks the specified endpoint to see if an image exists 145 // and if it is absent then it sends the image id to the channel to be pushed. 146 func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) { 147 defer wg.Done() 148 for image := range images { 149 if err := p.session.LookupRemoteImage(image.id, image.endpoint); err != nil { 150 logrus.Errorf("Error in LookupRemoteImage: %s", err) 151 imagesToPush <- image.id 152 continue 153 } 154 p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(image.id))) 155 } 156 } 157 158 func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags map[string][]string, repo *registry.RepositoryData) error { 159 workerCount := len(imageIDs) 160 // start a maximum of 5 workers to check if images exist on the specified endpoint. 161 if workerCount > 5 { 162 workerCount = 5 163 } 164 var ( 165 wg = &sync.WaitGroup{} 166 imageData = make(chan imagePushData, workerCount*2) 167 imagesToPush = make(chan string, workerCount*2) 168 pushes = make(chan map[string]struct{}, 1) 169 ) 170 for i := 0; i < workerCount; i++ { 171 wg.Add(1) 172 go p.lookupImageOnEndpoint(wg, imageData, imagesToPush) 173 } 174 // start a go routine that consumes the images to push 175 go func() { 176 shouldPush := make(map[string]struct{}) 177 for id := range imagesToPush { 178 shouldPush[id] = struct{}{} 179 } 180 pushes <- shouldPush 181 }() 182 for _, id := range imageIDs { 183 imageData <- imagePushData{ 184 id: id, 185 endpoint: endpoint, 186 } 187 } 188 // close the channel to notify the workers that there will be no more images to check. 189 close(imageData) 190 wg.Wait() 191 close(imagesToPush) 192 // wait for all the images that require pushes to be collected into a consumable map. 193 shouldPush := <-pushes 194 // finish by pushing any images and tags to the endpoint. The order that the images are pushed 195 // is very important that is why we are still iterating over the ordered list of imageIDs. 196 for _, id := range imageIDs { 197 if _, push := shouldPush[id]; push { 198 if _, err := p.pushImage(id, endpoint); err != nil { 199 // FIXME: Continue on error? 200 return err 201 } 202 } 203 for _, tag := range tags[id] { 204 p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag)) 205 if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, id, tag, endpoint); err != nil { 206 return err 207 } 208 } 209 } 210 return nil 211 } 212 213 // pushRepository pushes layers that do not already exist on the registry. 214 func (p *v1Pusher) pushRepository(tag string) error { 215 logrus.Debugf("Local repo: %s", p.localRepo) 216 p.out = ioutils.NewWriteFlusher(p.config.OutStream) 217 imgList, tags, err := p.getImageList(tag) 218 if err != nil { 219 return err 220 } 221 p.out.Write(p.sf.FormatStatus("", "Sending image list")) 222 223 imageIndex := p.createImageIndex(imgList, tags) 224 logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo) 225 for _, data := range imageIndex { 226 logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag) 227 } 228 229 if _, found := p.poolAdd("push", p.repoInfo.LocalName); found { 230 return fmt.Errorf("push or pull %s is already in progress", p.repoInfo.LocalName) 231 } 232 defer p.poolRemove("push", p.repoInfo.LocalName) 233 234 // Register all the images in a repository with the registry 235 // If an image is not in this list it will not be associated with the repository 236 repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil) 237 if err != nil { 238 return err 239 } 240 nTag := 1 241 if tag == "" { 242 nTag = len(p.localRepo) 243 } 244 p.out.Write(p.sf.FormatStatus("", "Pushing repository %s (%d tags)", p.repoInfo.CanonicalName, nTag)) 245 // push the repository to each of the endpoints only if it does not exist. 246 for _, endpoint := range repoData.Endpoints { 247 if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil { 248 return err 249 } 250 } 251 _, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints) 252 return err 253 } 254 255 func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) { 256 jsonRaw, err := p.graph.RawJSON(imgID) 257 if err != nil { 258 return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) 259 } 260 p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil)) 261 262 imgData := ®istry.ImgData{ 263 ID: imgID, 264 } 265 266 // Send the json 267 if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil { 268 if err == registry.ErrAlreadyExists { 269 p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image already pushed, skipping", nil)) 270 return "", nil 271 } 272 return "", err 273 } 274 275 layerData, err := p.graph.TempLayerArchive(imgID, p.sf, p.out) 276 if err != nil { 277 return "", fmt.Errorf("Failed to generate layer archive: %s", err) 278 } 279 defer os.RemoveAll(layerData.Name()) 280 281 // Send the layer 282 logrus.Debugf("rendered layer for %s of [%d] size", imgData.ID, layerData.Size) 283 284 checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID, 285 progressreader.New(progressreader.Config{ 286 In: layerData, 287 Out: p.out, 288 Formatter: p.sf, 289 Size: layerData.Size, 290 NewLines: false, 291 ID: stringid.TruncateID(imgData.ID), 292 Action: "Pushing", 293 }), ep, jsonRaw) 294 if err != nil { 295 return "", err 296 } 297 imgData.Checksum = checksum 298 imgData.ChecksumPayload = checksumPayload 299 // Send the checksum 300 if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil { 301 return "", err 302 } 303 304 p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgData.ID), "Image successfully pushed", nil)) 305 return imgData.Checksum, nil 306 }