github.com/dinever/docker@v1.11.1/distribution/pull_v1.go (about) 1 package distribution 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "net" 9 "net/url" 10 "os" 11 "strings" 12 "time" 13 14 "github.com/Sirupsen/logrus" 15 "github.com/docker/distribution/registry/client/transport" 16 "github.com/docker/docker/distribution/metadata" 17 "github.com/docker/docker/distribution/xfer" 18 "github.com/docker/docker/dockerversion" 19 "github.com/docker/docker/image" 20 "github.com/docker/docker/image/v1" 21 "github.com/docker/docker/layer" 22 "github.com/docker/docker/pkg/ioutils" 23 "github.com/docker/docker/pkg/progress" 24 "github.com/docker/docker/pkg/stringid" 25 "github.com/docker/docker/reference" 26 "github.com/docker/docker/registry" 27 "golang.org/x/net/context" 28 ) 29 30 type v1Puller struct { 31 v1IDService *metadata.V1IDService 32 endpoint registry.APIEndpoint 33 config *ImagePullConfig 34 repoInfo *registry.RepositoryInfo 35 session *registry.Session 36 } 37 38 func (p *v1Puller) Pull(ctx context.Context, ref reference.Named) error { 39 if _, isCanonical := ref.(reference.Canonical); isCanonical { 40 // Allowing fallback, because HTTPS v1 is before HTTP v2 41 return fallbackError{err: ErrNoSupport{Err: errors.New("Cannot pull by digest with v1 registry")}} 42 } 43 44 tlsConfig, err := p.config.RegistryService.TLSConfig(p.repoInfo.Index.Name) 45 if err != nil { 46 return err 47 } 48 // Adds Docker-specific headers as well as user-specified headers (metaHeaders) 49 tr := transport.NewTransport( 50 // TODO(tiborvass): was ReceiveTimeout 51 registry.NewTransport(tlsConfig), 52 registry.DockerHeaders(dockerversion.DockerUserAgent(ctx), p.config.MetaHeaders)..., 53 ) 54 client := registry.HTTPClient(tr) 55 v1Endpoint, err := p.endpoint.ToV1Endpoint(dockerversion.DockerUserAgent(ctx), p.config.MetaHeaders) 56 if err != nil { 57 logrus.Debugf("Could not get v1 endpoint: %v", err) 58 return fallbackError{err: err} 59 } 60 p.session, err = registry.NewSession(client, p.config.AuthConfig, v1Endpoint) 61 if err != nil { 62 // TODO(dmcgowan): Check if should fallback 63 logrus.Debugf("Fallback from error: %s", err) 64 return fallbackError{err: err} 65 } 66 if err := p.pullRepository(ctx, ref); err != nil { 67 // TODO(dmcgowan): Check if should fallback 68 return err 69 } 70 progress.Message(p.config.ProgressOutput, "", p.repoInfo.FullName()+": this image was pulled from a legacy registry. Important: This registry version will not be supported in future versions of docker.") 71 72 return nil 73 } 74 75 func (p *v1Puller) pullRepository(ctx context.Context, ref reference.Named) error { 76 progress.Message(p.config.ProgressOutput, "", "Pulling repository "+p.repoInfo.FullName()) 77 78 repoData, err := p.session.GetRepositoryData(p.repoInfo) 79 if err != nil { 80 if strings.Contains(err.Error(), "HTTP code: 404") { 81 return fmt.Errorf("Error: image %s not found", p.repoInfo.RemoteName()) 82 } 83 // Unexpected HTTP error 84 return err 85 } 86 87 logrus.Debugf("Retrieving the tag list") 88 var tagsList map[string]string 89 tagged, isTagged := ref.(reference.NamedTagged) 90 if !isTagged { 91 tagsList, err = p.session.GetRemoteTags(repoData.Endpoints, p.repoInfo) 92 } else { 93 var tagID string 94 tagsList = make(map[string]string) 95 tagID, err = p.session.GetRemoteTag(repoData.Endpoints, p.repoInfo, tagged.Tag()) 96 if err == registry.ErrRepoNotFound { 97 return fmt.Errorf("Tag %s not found in repository %s", tagged.Tag(), p.repoInfo.FullName()) 98 } 99 tagsList[tagged.Tag()] = tagID 100 } 101 if err != nil { 102 logrus.Errorf("unable to get remote tags: %s", err) 103 return err 104 } 105 106 for tag, id := range tagsList { 107 repoData.ImgList[id] = ®istry.ImgData{ 108 ID: id, 109 Tag: tag, 110 Checksum: "", 111 } 112 } 113 114 layersDownloaded := false 115 for _, imgData := range repoData.ImgList { 116 if isTagged && imgData.Tag != tagged.Tag() { 117 continue 118 } 119 120 err := p.downloadImage(ctx, repoData, imgData, &layersDownloaded) 121 if err != nil { 122 return err 123 } 124 } 125 126 writeStatus(ref.String(), p.config.ProgressOutput, layersDownloaded) 127 return nil 128 } 129 130 func (p *v1Puller) downloadImage(ctx context.Context, repoData *registry.RepositoryData, img *registry.ImgData, layersDownloaded *bool) error { 131 if img.Tag == "" { 132 logrus.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) 133 return nil 134 } 135 136 localNameRef, err := reference.WithTag(p.repoInfo, img.Tag) 137 if err != nil { 138 retErr := fmt.Errorf("Image (id: %s) has invalid tag: %s", img.ID, img.Tag) 139 logrus.Debug(retErr.Error()) 140 return retErr 141 } 142 143 if err := v1.ValidateID(img.ID); err != nil { 144 return err 145 } 146 147 progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, p.repoInfo.FullName()) 148 success := false 149 var lastErr error 150 for _, ep := range p.repoInfo.Index.Mirrors { 151 ep += "v1/" 152 progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), fmt.Sprintf("Pulling image (%s) from %s, mirror: %s", img.Tag, p.repoInfo.FullName(), ep)) 153 if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil { 154 // Don't report errors when pulling from mirrors. 155 logrus.Debugf("Error pulling image (%s) from %s, mirror: %s, %s", img.Tag, p.repoInfo.FullName(), ep, err) 156 continue 157 } 158 success = true 159 break 160 } 161 if !success { 162 for _, ep := range repoData.Endpoints { 163 progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Pulling image (%s) from %s, endpoint: %s", img.Tag, p.repoInfo.FullName(), ep) 164 if err = p.pullImage(ctx, img.ID, ep, localNameRef, layersDownloaded); err != nil { 165 // It's not ideal that only the last error is returned, it would be better to concatenate the errors. 166 // As the error is also given to the output stream the user will see the error. 167 lastErr = err 168 progress.Updatef(p.config.ProgressOutput, stringid.TruncateID(img.ID), "Error pulling image (%s) from %s, endpoint: %s, %s", img.Tag, p.repoInfo.FullName(), ep, err) 169 continue 170 } 171 success = true 172 break 173 } 174 } 175 if !success { 176 err := fmt.Errorf("Error pulling image (%s) from %s, %v", img.Tag, p.repoInfo.FullName(), lastErr) 177 progress.Update(p.config.ProgressOutput, stringid.TruncateID(img.ID), err.Error()) 178 return err 179 } 180 return nil 181 } 182 183 func (p *v1Puller) pullImage(ctx context.Context, v1ID, endpoint string, localNameRef reference.Named, layersDownloaded *bool) (err error) { 184 var history []string 185 history, err = p.session.GetRemoteHistory(v1ID, endpoint) 186 if err != nil { 187 return err 188 } 189 if len(history) < 1 { 190 return fmt.Errorf("empty history for image %s", v1ID) 191 } 192 progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1ID), "Pulling dependent layers") 193 194 var ( 195 descriptors []xfer.DownloadDescriptor 196 newHistory []image.History 197 imgJSON []byte 198 imgSize int64 199 ) 200 201 // Iterate over layers, in order from bottom-most to top-most. Download 202 // config for all layers and create descriptors. 203 for i := len(history) - 1; i >= 0; i-- { 204 v1LayerID := history[i] 205 imgJSON, imgSize, err = p.downloadLayerConfig(v1LayerID, endpoint) 206 if err != nil { 207 return err 208 } 209 210 // Create a new-style config from the legacy configs 211 h, err := v1.HistoryFromConfig(imgJSON, false) 212 if err != nil { 213 return err 214 } 215 newHistory = append(newHistory, h) 216 217 layerDescriptor := &v1LayerDescriptor{ 218 v1LayerID: v1LayerID, 219 indexName: p.repoInfo.Index.Name, 220 endpoint: endpoint, 221 v1IDService: p.v1IDService, 222 layersDownloaded: layersDownloaded, 223 layerSize: imgSize, 224 session: p.session, 225 } 226 227 descriptors = append(descriptors, layerDescriptor) 228 } 229 230 rootFS := image.NewRootFS() 231 resultRootFS, release, err := p.config.DownloadManager.Download(ctx, *rootFS, descriptors, p.config.ProgressOutput) 232 if err != nil { 233 return err 234 } 235 defer release() 236 237 config, err := v1.MakeConfigFromV1Config(imgJSON, &resultRootFS, newHistory) 238 if err != nil { 239 return err 240 } 241 242 imageID, err := p.config.ImageStore.Create(config) 243 if err != nil { 244 return err 245 } 246 247 if err := p.config.ReferenceStore.AddTag(localNameRef, imageID, true); err != nil { 248 return err 249 } 250 251 return nil 252 } 253 254 func (p *v1Puller) downloadLayerConfig(v1LayerID, endpoint string) (imgJSON []byte, imgSize int64, err error) { 255 progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1LayerID), "Pulling metadata") 256 257 retries := 5 258 for j := 1; j <= retries; j++ { 259 imgJSON, imgSize, err := p.session.GetRemoteImageJSON(v1LayerID, endpoint) 260 if err != nil && j == retries { 261 progress.Update(p.config.ProgressOutput, stringid.TruncateID(v1LayerID), "Error pulling layer metadata") 262 return nil, 0, err 263 } else if err != nil { 264 time.Sleep(time.Duration(j) * 500 * time.Millisecond) 265 continue 266 } 267 268 return imgJSON, imgSize, nil 269 } 270 271 // not reached 272 return nil, 0, nil 273 } 274 275 type v1LayerDescriptor struct { 276 v1LayerID string 277 indexName string 278 endpoint string 279 v1IDService *metadata.V1IDService 280 layersDownloaded *bool 281 layerSize int64 282 session *registry.Session 283 tmpFile *os.File 284 } 285 286 func (ld *v1LayerDescriptor) Key() string { 287 return "v1:" + ld.v1LayerID 288 } 289 290 func (ld *v1LayerDescriptor) ID() string { 291 return stringid.TruncateID(ld.v1LayerID) 292 } 293 294 func (ld *v1LayerDescriptor) DiffID() (layer.DiffID, error) { 295 return ld.v1IDService.Get(ld.v1LayerID, ld.indexName) 296 } 297 298 func (ld *v1LayerDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) { 299 progress.Update(progressOutput, ld.ID(), "Pulling fs layer") 300 layerReader, err := ld.session.GetRemoteImageLayer(ld.v1LayerID, ld.endpoint, ld.layerSize) 301 if err != nil { 302 progress.Update(progressOutput, ld.ID(), "Error pulling dependent layers") 303 if uerr, ok := err.(*url.Error); ok { 304 err = uerr.Err 305 } 306 if terr, ok := err.(net.Error); ok && terr.Timeout() { 307 return nil, 0, err 308 } 309 return nil, 0, xfer.DoNotRetry{Err: err} 310 } 311 *ld.layersDownloaded = true 312 313 ld.tmpFile, err = ioutil.TempFile("", "GetImageBlob") 314 if err != nil { 315 layerReader.Close() 316 return nil, 0, err 317 } 318 319 reader := progress.NewProgressReader(ioutils.NewCancelReadCloser(ctx, layerReader), progressOutput, ld.layerSize, ld.ID(), "Downloading") 320 defer reader.Close() 321 322 _, err = io.Copy(ld.tmpFile, reader) 323 if err != nil { 324 ld.Close() 325 return nil, 0, err 326 } 327 328 progress.Update(progressOutput, ld.ID(), "Download complete") 329 330 logrus.Debugf("Downloaded %s to tempfile %s", ld.ID(), ld.tmpFile.Name()) 331 332 ld.tmpFile.Seek(0, 0) 333 334 // hand off the temporary file to the download manager, so it will only 335 // be closed once 336 tmpFile := ld.tmpFile 337 ld.tmpFile = nil 338 339 return ioutils.NewReadCloserWrapper(tmpFile, func() error { 340 tmpFile.Close() 341 err := os.RemoveAll(tmpFile.Name()) 342 if err != nil { 343 logrus.Errorf("Failed to remove temp file: %s", tmpFile.Name()) 344 } 345 return err 346 }), ld.layerSize, nil 347 } 348 349 func (ld *v1LayerDescriptor) Close() { 350 if ld.tmpFile != nil { 351 ld.tmpFile.Close() 352 if err := os.RemoveAll(ld.tmpFile.Name()); err != nil { 353 logrus.Errorf("Failed to remove temp file: %s", ld.tmpFile.Name()) 354 } 355 ld.tmpFile = nil 356 } 357 } 358 359 func (ld *v1LayerDescriptor) Registered(diffID layer.DiffID) { 360 // Cache mapping from this layer's DiffID to the blobsum 361 ld.v1IDService.Set(ld.v1LayerID, ld.indexName, diffID) 362 }