github.com/yamamoto-febc/docker@v1.9.0/graph/push_v1.go (about) 1 package graph 2 3 import ( 4 "fmt" 5 "io" 6 "os" 7 "sync" 8 9 "github.com/Sirupsen/logrus" 10 "github.com/docker/distribution/registry/client/transport" 11 "github.com/docker/docker/image" 12 "github.com/docker/docker/pkg/ioutils" 13 "github.com/docker/docker/pkg/progressreader" 14 "github.com/docker/docker/pkg/streamformatter" 15 "github.com/docker/docker/pkg/stringid" 16 "github.com/docker/docker/registry" 17 "github.com/docker/docker/utils" 18 ) 19 20 type v1Pusher struct { 21 *TagStore 22 endpoint registry.APIEndpoint 23 localRepo Repository 24 repoInfo *registry.RepositoryInfo 25 config *ImagePushConfig 26 sf *streamformatter.StreamFormatter 27 session *registry.Session 28 29 out io.Writer 30 } 31 32 func (p *v1Pusher) Push() (fallback bool, err error) { 33 tlsConfig, err := p.registryService.TLSConfig(p.repoInfo.Index.Name) 34 if err != nil { 35 return false, err 36 } 37 // Adds Docker-specific headers as well as user-specified headers (metaHeaders) 38 tr := transport.NewTransport( 39 // TODO(tiborvass): was NoTimeout 40 registry.NewTransport(tlsConfig), 41 registry.DockerHeaders(p.config.MetaHeaders)..., 42 ) 43 client := registry.HTTPClient(tr) 44 v1Endpoint, err := p.endpoint.ToV1Endpoint(p.config.MetaHeaders) 45 if err != nil { 46 logrus.Debugf("Could not get v1 endpoint: %v", err) 47 return true, err 48 } 49 p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint) 50 if err != nil { 51 // TODO(dmcgowan): Check if should fallback 52 return true, err 53 } 54 if err := p.pushRepository(p.config.Tag); err != nil { 55 // TODO(dmcgowan): Check if should fallback 56 return false, err 57 } 58 return false, nil 59 } 60 61 // Retrieve the all the images to be uploaded in the correct order 62 func (p *v1Pusher) getImageList(requestedTag string) ([]string, map[string][]string, error) { 63 var ( 64 imageList []string 65 imagesSeen = make(map[string]bool) 66 tagsByImage = make(map[string][]string) 67 ) 68 69 for tag, id := range p.localRepo { 70 if requestedTag != "" && requestedTag != tag { 71 // Include only the requested tag. 72 continue 73 } 74 75 if utils.DigestReference(tag) { 76 // Ignore digest references. 77 continue 78 } 79 80 var imageListForThisTag []string 81 82 tagsByImage[id] = append(tagsByImage[id], tag) 83 84 for img, err := p.graph.Get(id); img != nil; img, err = p.graph.GetParent(img) { 85 if err != nil { 86 return nil, nil, err 87 } 88 89 if imagesSeen[img.ID] { 90 // This image is already on the list, we can ignore it and all its parents 91 break 92 } 93 94 imagesSeen[img.ID] = true 95 imageListForThisTag = append(imageListForThisTag, img.ID) 96 } 97 98 // reverse the image list for this tag (so the "most"-parent image is first) 99 for i, j := 0, len(imageListForThisTag)-1; i < j; i, j = i+1, j-1 { 100 imageListForThisTag[i], imageListForThisTag[j] = imageListForThisTag[j], imageListForThisTag[i] 101 } 102 103 // append to main image list 104 imageList = append(imageList, imageListForThisTag...) 105 } 106 if len(imageList) == 0 { 107 return nil, nil, fmt.Errorf("No images found for the requested repository / tag") 108 } 109 logrus.Debugf("Image list: %v", imageList) 110 logrus.Debugf("Tags by image: %v", tagsByImage) 111 112 return imageList, tagsByImage, nil 113 } 114 115 // createImageIndex returns an index of an image's layer IDs and tags. 116 func (s *TagStore) createImageIndex(images []string, tags map[string][]string) []*registry.ImgData { 117 var imageIndex []*registry.ImgData 118 for _, id := range images { 119 if tags, hasTags := tags[id]; hasTags { 120 // If an image has tags you must add an entry in the image index 121 // for each tag 122 for _, tag := range tags { 123 imageIndex = append(imageIndex, ®istry.ImgData{ 124 ID: id, 125 Tag: tag, 126 }) 127 } 128 continue 129 } 130 // If the image does not have a tag it still needs to be sent to the 131 // registry with an empty tag so that it is associated with the repository 132 imageIndex = append(imageIndex, ®istry.ImgData{ 133 ID: id, 134 Tag: "", 135 }) 136 } 137 return imageIndex 138 } 139 140 type imagePushData struct { 141 id string 142 compatibilityID string 143 endpoint string 144 } 145 146 // lookupImageOnEndpoint checks the specified endpoint to see if an image exists 147 // and if it is absent then it sends the image id to the channel to be pushed. 148 func (p *v1Pusher) lookupImageOnEndpoint(wg *sync.WaitGroup, images chan imagePushData, imagesToPush chan string) { 149 defer wg.Done() 150 for image := range images { 151 if err := p.session.LookupRemoteImage(image.compatibilityID, image.endpoint); err != nil { 152 logrus.Errorf("Error in LookupRemoteImage: %s", err) 153 imagesToPush <- image.id 154 continue 155 } 156 p.out.Write(p.sf.FormatStatus("", "Image %s already pushed, skipping", stringid.TruncateID(image.id))) 157 } 158 } 159 160 func (p *v1Pusher) pushImageToEndpoint(endpoint string, imageIDs []string, tags map[string][]string, repo *registry.RepositoryData) error { 161 workerCount := len(imageIDs) 162 // start a maximum of 5 workers to check if images exist on the specified endpoint. 163 if workerCount > 5 { 164 workerCount = 5 165 } 166 var ( 167 wg = &sync.WaitGroup{} 168 imageData = make(chan imagePushData, workerCount*2) 169 imagesToPush = make(chan string, workerCount*2) 170 pushes = make(chan map[string]struct{}, 1) 171 ) 172 for i := 0; i < workerCount; i++ { 173 wg.Add(1) 174 go p.lookupImageOnEndpoint(wg, imageData, imagesToPush) 175 } 176 // start a go routine that consumes the images to push 177 go func() { 178 shouldPush := make(map[string]struct{}) 179 for id := range imagesToPush { 180 shouldPush[id] = struct{}{} 181 } 182 pushes <- shouldPush 183 }() 184 for _, id := range imageIDs { 185 compatibilityID, err := p.getV1ID(id) 186 if err != nil { 187 return err 188 } 189 imageData <- imagePushData{ 190 id: id, 191 compatibilityID: compatibilityID, 192 endpoint: endpoint, 193 } 194 } 195 // close the channel to notify the workers that there will be no more images to check. 196 close(imageData) 197 wg.Wait() 198 close(imagesToPush) 199 // wait for all the images that require pushes to be collected into a consumable map. 200 shouldPush := <-pushes 201 // finish by pushing any images and tags to the endpoint. The order that the images are pushed 202 // is very important that is why we are still iterating over the ordered list of imageIDs. 203 for _, id := range imageIDs { 204 if _, push := shouldPush[id]; push { 205 if _, err := p.pushImage(id, endpoint); err != nil { 206 // FIXME: Continue on error? 207 return err 208 } 209 } 210 for _, tag := range tags[id] { 211 p.out.Write(p.sf.FormatStatus("", "Pushing tag for rev [%s] on {%s}", stringid.TruncateID(id), endpoint+"repositories/"+p.repoInfo.RemoteName+"/tags/"+tag)) 212 compatibilityID, err := p.getV1ID(id) 213 if err != nil { 214 return err 215 } 216 if err := p.session.PushRegistryTag(p.repoInfo.RemoteName, compatibilityID, tag, endpoint); err != nil { 217 return err 218 } 219 } 220 } 221 return nil 222 } 223 224 // pushRepository pushes layers that do not already exist on the registry. 225 func (p *v1Pusher) pushRepository(tag string) error { 226 logrus.Debugf("Local repo: %s", p.localRepo) 227 p.out = ioutils.NewWriteFlusher(p.config.OutStream) 228 imgList, tags, err := p.getImageList(tag) 229 if err != nil { 230 return err 231 } 232 p.out.Write(p.sf.FormatStatus("", "Sending image list")) 233 234 imageIndex := p.createImageIndex(imgList, tags) 235 logrus.Debugf("Preparing to push %s with the following images and tags", p.localRepo) 236 for _, data := range imageIndex { 237 logrus.Debugf("Pushing ID: %s with Tag: %s", data.ID, data.Tag) 238 239 // convert IDs to compatibilityIDs, imageIndex only used in registry calls 240 data.ID, err = p.getV1ID(data.ID) 241 if err != nil { 242 return err 243 } 244 } 245 246 if _, found := p.poolAdd("push", p.repoInfo.LocalName); found { 247 return fmt.Errorf("push or pull %s is already in progress", p.repoInfo.LocalName) 248 } 249 defer p.poolRemove("push", p.repoInfo.LocalName) 250 251 // Register all the images in a repository with the registry 252 // If an image is not in this list it will not be associated with the repository 253 repoData, err := p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, false, nil) 254 if err != nil { 255 return err 256 } 257 nTag := 1 258 if tag == "" { 259 nTag = len(p.localRepo) 260 } 261 p.out.Write(p.sf.FormatStatus("", "Pushing repository %s (%d tags)", p.repoInfo.CanonicalName, nTag)) 262 // push the repository to each of the endpoints only if it does not exist. 263 for _, endpoint := range repoData.Endpoints { 264 if err := p.pushImageToEndpoint(endpoint, imgList, tags, repoData); err != nil { 265 return err 266 } 267 } 268 _, err = p.session.PushImageJSONIndex(p.repoInfo.RemoteName, imageIndex, true, repoData.Endpoints) 269 return err 270 } 271 272 func (p *v1Pusher) pushImage(imgID, ep string) (checksum string, err error) { 273 jsonRaw, err := p.getV1Config(imgID) 274 if err != nil { 275 return "", fmt.Errorf("Cannot retrieve the path for {%s}: %s", imgID, err) 276 } 277 p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Pushing", nil)) 278 279 compatibilityID, err := p.getV1ID(imgID) 280 if err != nil { 281 return "", err 282 } 283 284 // General rule is to use ID for graph accesses and compatibilityID for 285 // calls to session.registry() 286 imgData := ®istry.ImgData{ 287 ID: compatibilityID, 288 } 289 290 // Send the json 291 if err := p.session.PushImageJSONRegistry(imgData, jsonRaw, ep); err != nil { 292 if err == registry.ErrAlreadyExists { 293 p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image already pushed, skipping", nil)) 294 return "", nil 295 } 296 return "", err 297 } 298 299 layerData, err := p.graph.TempLayerArchive(imgID, p.sf, p.out) 300 if err != nil { 301 return "", fmt.Errorf("Failed to generate layer archive: %s", err) 302 } 303 defer os.RemoveAll(layerData.Name()) 304 305 // Send the layer 306 logrus.Debugf("rendered layer for %s of [%d] size", imgID, layerData.Size) 307 308 checksum, checksumPayload, err := p.session.PushImageLayerRegistry(imgData.ID, 309 progressreader.New(progressreader.Config{ 310 In: layerData, 311 Out: p.out, 312 Formatter: p.sf, 313 Size: layerData.Size, 314 NewLines: false, 315 ID: stringid.TruncateID(imgID), 316 Action: "Pushing", 317 }), ep, jsonRaw) 318 if err != nil { 319 return "", err 320 } 321 imgData.Checksum = checksum 322 imgData.ChecksumPayload = checksumPayload 323 // Send the checksum 324 if err := p.session.PushImageChecksumRegistry(imgData, ep); err != nil { 325 return "", err 326 } 327 328 p.out.Write(p.sf.FormatProgress(stringid.TruncateID(imgID), "Image successfully pushed", nil)) 329 return imgData.Checksum, nil 330 } 331 332 // getV1ID returns the compatibilityID for the ID in the graph. compatibilityID 333 // is read from from the v1Compatibility config file in the disk. 334 func (p *v1Pusher) getV1ID(id string) (string, error) { 335 jsonData, err := p.getV1Config(id) 336 if err != nil { 337 return "", err 338 } 339 img, err := image.NewImgJSON(jsonData) 340 if err != nil { 341 return "", err 342 } 343 return img.ID, nil 344 } 345 346 // getV1Config returns v1Compatibility config for the image in the graph. If 347 // there is no v1Compatibility file on disk for the image 348 func (p *v1Pusher) getV1Config(id string) ([]byte, error) { 349 jsonData, err := p.graph.GenerateV1CompatibilityChain(id) 350 if err != nil { 351 return nil, err 352 } 353 return jsonData, nil 354 }