github.com/mheon/docker@v0.11.2-0.20150922122814-44f47903a831/graph/pull_v2.go (about) 1 package graph 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "os" 8 9 "github.com/Sirupsen/logrus" 10 "github.com/docker/distribution" 11 "github.com/docker/distribution/digest" 12 "github.com/docker/distribution/manifest" 13 "github.com/docker/docker/image" 14 "github.com/docker/docker/pkg/progressreader" 15 "github.com/docker/docker/pkg/streamformatter" 16 "github.com/docker/docker/pkg/stringid" 17 "github.com/docker/docker/registry" 18 "github.com/docker/docker/trust" 19 "github.com/docker/docker/utils" 20 "github.com/docker/libtrust" 21 "golang.org/x/net/context" 22 ) 23 24 type v2Puller struct { 25 *TagStore 26 endpoint registry.APIEndpoint 27 config *ImagePullConfig 28 sf *streamformatter.StreamFormatter 29 repoInfo *registry.RepositoryInfo 30 repo distribution.Repository 31 sessionID string 32 } 33 34 func (p *v2Puller) Pull(tag string) (fallback bool, err error) { 35 // TODO(tiborvass): was ReceiveTimeout 36 p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig, "pull") 37 if err != nil { 38 logrus.Debugf("Error getting v2 registry: %v", err) 39 return true, err 40 } 41 42 p.sessionID = stringid.GenerateRandomID() 43 44 if err := p.pullV2Repository(tag); err != nil { 45 if registry.ContinueOnError(err) { 46 logrus.Debugf("Error trying v2 registry: %v", err) 47 return true, err 48 } 49 return false, err 50 } 51 return false, nil 52 } 53 54 func (p *v2Puller) pullV2Repository(tag string) (err error) { 55 var tags []string 56 taggedName := p.repoInfo.LocalName 57 if len(tag) > 0 { 58 tags = []string{tag} 59 taggedName = utils.ImageReference(p.repoInfo.LocalName, tag) 60 } else { 61 var err error 62 63 manSvc, err := p.repo.Manifests(context.Background()) 64 if err != nil { 65 return err 66 } 67 68 tags, err = manSvc.Tags() 69 if err != nil { 70 return err 71 } 72 73 } 74 75 broadcaster, found := p.poolAdd("pull", taggedName) 76 broadcaster.Add(p.config.OutStream) 77 if found { 78 // Another pull of the same repository is already taking place; just wait for it to finish 79 return broadcaster.Wait() 80 } 81 82 // This must use a closure so it captures the value of err when the 83 // function returns, not when the 'defer' is evaluated. 84 defer func() { 85 p.poolRemoveWithError("pull", taggedName, err) 86 }() 87 88 var layersDownloaded bool 89 for _, tag := range tags { 90 // pulledNew is true if either new layers were downloaded OR if existing images were newly tagged 91 // TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus? 92 pulledNew, err := p.pullV2Tag(broadcaster, tag, taggedName) 93 if err != nil { 94 return err 95 } 96 layersDownloaded = layersDownloaded || pulledNew 97 } 98 99 writeStatus(taggedName, broadcaster, p.sf, layersDownloaded) 100 101 return nil 102 } 103 104 // downloadInfo is used to pass information from download to extractor 105 type downloadInfo struct { 106 img *image.Image 107 tmpFile *os.File 108 digest digest.Digest 109 layer distribution.ReadSeekCloser 110 size int64 111 err chan error 112 poolKey string 113 broadcaster *progressreader.Broadcaster 114 } 115 116 type errVerification struct{} 117 118 func (errVerification) Error() string { return "verification failed" } 119 120 func (p *v2Puller) download(di *downloadInfo) { 121 logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID) 122 123 blobs := p.repo.Blobs(context.Background()) 124 125 desc, err := blobs.Stat(context.Background(), di.digest) 126 if err != nil { 127 logrus.Debugf("Error statting layer: %v", err) 128 di.err <- err 129 return 130 } 131 di.size = desc.Size 132 133 layerDownload, err := blobs.Open(context.Background(), di.digest) 134 if err != nil { 135 logrus.Debugf("Error fetching layer: %v", err) 136 di.err <- err 137 return 138 } 139 defer layerDownload.Close() 140 141 verifier, err := digest.NewDigestVerifier(di.digest) 142 if err != nil { 143 di.err <- err 144 return 145 } 146 147 reader := progressreader.New(progressreader.Config{ 148 In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)), 149 Out: di.broadcaster, 150 Formatter: p.sf, 151 Size: di.size, 152 NewLines: false, 153 ID: stringid.TruncateID(di.img.ID), 154 Action: "Downloading", 155 }) 156 io.Copy(di.tmpFile, reader) 157 158 di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil)) 159 160 if !verifier.Verified() { 161 err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest) 162 logrus.Error(err) 163 di.err <- err 164 return 165 } 166 167 di.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) 168 169 logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, di.tmpFile.Name()) 170 di.layer = layerDownload 171 172 di.err <- nil 173 } 174 175 func (p *v2Puller) pullV2Tag(out io.Writer, tag, taggedName string) (verified bool, err error) { 176 logrus.Debugf("Pulling tag from V2 registry: %q", tag) 177 178 manSvc, err := p.repo.Manifests(context.Background()) 179 if err != nil { 180 return false, err 181 } 182 183 manifest, err := manSvc.GetByTag(tag) 184 if err != nil { 185 return false, err 186 } 187 verified, err = p.validateManifest(manifest, tag) 188 if err != nil { 189 return false, err 190 } 191 if verified { 192 logrus.Printf("Image manifest for %s has been verified", taggedName) 193 } 194 195 out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name())) 196 197 var downloads []*downloadInfo 198 199 var layerIDs []string 200 defer func() { 201 p.graph.Release(p.sessionID, layerIDs...) 202 203 for _, d := range downloads { 204 p.poolRemoveWithError("pull", d.poolKey, err) 205 if d.tmpFile != nil { 206 d.tmpFile.Close() 207 if err := os.RemoveAll(d.tmpFile.Name()); err != nil { 208 logrus.Errorf("Failed to remove temp file: %s", d.tmpFile.Name()) 209 } 210 } 211 } 212 }() 213 214 for i := len(manifest.FSLayers) - 1; i >= 0; i-- { 215 img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility)) 216 if err != nil { 217 logrus.Debugf("error getting image v1 json: %v", err) 218 return false, err 219 } 220 p.graph.Retain(p.sessionID, img.ID) 221 layerIDs = append(layerIDs, img.ID) 222 223 // Check if exists 224 if p.graph.Exists(img.ID) { 225 logrus.Debugf("Image already exists: %s", img.ID) 226 out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Already exists", nil)) 227 continue 228 } 229 out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) 230 231 d := &downloadInfo{ 232 img: img, 233 poolKey: "layer:" + img.ID, 234 digest: manifest.FSLayers[i].BlobSum, 235 // TODO: seems like this chan buffer solved hanging problem in go1.5, 236 // this can indicate some deeper problem that somehow we never take 237 // error from channel in loop below 238 err: make(chan error, 1), 239 } 240 241 tmpFile, err := ioutil.TempFile("", "GetImageBlob") 242 if err != nil { 243 return false, err 244 } 245 d.tmpFile = tmpFile 246 247 downloads = append(downloads, d) 248 249 broadcaster, found := p.poolAdd("pull", d.poolKey) 250 broadcaster.Add(out) 251 d.broadcaster = broadcaster 252 if found { 253 d.err <- nil 254 } else { 255 go p.download(d) 256 } 257 } 258 259 var tagUpdated bool 260 for _, d := range downloads { 261 if err := <-d.err; err != nil { 262 return false, err 263 } 264 265 if d.layer == nil { 266 // Wait for a different pull to download and extract 267 // this layer. 268 err = d.broadcaster.Wait() 269 if err != nil { 270 return false, err 271 } 272 continue 273 } 274 275 d.tmpFile.Seek(0, 0) 276 reader := progressreader.New(progressreader.Config{ 277 In: d.tmpFile, 278 Out: d.broadcaster, 279 Formatter: p.sf, 280 Size: d.size, 281 NewLines: false, 282 ID: stringid.TruncateID(d.img.ID), 283 Action: "Extracting", 284 }) 285 286 err = p.graph.Register(d.img, reader) 287 if err != nil { 288 return false, err 289 } 290 291 if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil { 292 return false, err 293 } 294 295 d.broadcaster.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) 296 d.broadcaster.Close() 297 tagUpdated = true 298 } 299 300 manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName) 301 if err != nil { 302 return false, err 303 } 304 305 // Check for new tag if no layers downloaded 306 if !tagUpdated { 307 repo, err := p.Get(p.repoInfo.LocalName) 308 if err != nil { 309 return false, err 310 } 311 if repo != nil { 312 if _, exists := repo[tag]; !exists { 313 tagUpdated = true 314 } 315 } else { 316 tagUpdated = true 317 } 318 } 319 320 if verified && tagUpdated { 321 out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) 322 } 323 324 firstID := layerIDs[len(layerIDs)-1] 325 if utils.DigestReference(tag) { 326 // TODO(stevvooe): Ideally, we should always set the digest so we can 327 // use the digest whether we pull by it or not. Unfortunately, the tag 328 // store treats the digest as a separate tag, meaning there may be an 329 // untagged digest image that would seem to be dangling by a user. 330 if err = p.SetDigest(p.repoInfo.LocalName, tag, firstID); err != nil { 331 return false, err 332 } 333 } else { 334 // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) 335 if err = p.Tag(p.repoInfo.LocalName, tag, firstID, true); err != nil { 336 return false, err 337 } 338 } 339 340 if manifestDigest != "" { 341 out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest)) 342 } 343 344 return tagUpdated, nil 345 } 346 347 // verifyTrustedKeys checks the keys provided against the trust store, 348 // ensuring that the provided keys are trusted for the namespace. The keys 349 // provided from this method must come from the signatures provided as part of 350 // the manifest JWS package, obtained from unpackSignedManifest or libtrust. 351 func (p *v2Puller) verifyTrustedKeys(namespace string, keys []libtrust.PublicKey) (verified bool, err error) { 352 if namespace[0] != '/' { 353 namespace = "/" + namespace 354 } 355 356 for _, key := range keys { 357 b, err := key.MarshalJSON() 358 if err != nil { 359 return false, fmt.Errorf("error marshalling public key: %s", err) 360 } 361 // Check key has read/write permission (0x03) 362 v, err := p.trustService.CheckKey(namespace, b, 0x03) 363 if err != nil { 364 vErr, ok := err.(trust.NotVerifiedError) 365 if !ok { 366 return false, fmt.Errorf("error running key check: %s", err) 367 } 368 logrus.Debugf("Key check result: %v", vErr) 369 } 370 verified = v 371 } 372 373 if verified { 374 logrus.Debug("Key check result: verified") 375 } 376 377 return 378 } 379 380 func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (verified bool, err error) { 381 // If pull by digest, then verify the manifest digest. NOTE: It is 382 // important to do this first, before any other content validation. If the 383 // digest cannot be verified, don't even bother with those other things. 384 if manifestDigest, err := digest.ParseDigest(tag); err == nil { 385 verifier, err := digest.NewDigestVerifier(manifestDigest) 386 if err != nil { 387 return false, err 388 } 389 payload, err := m.Payload() 390 if err != nil { 391 return false, err 392 } 393 if _, err := verifier.Write(payload); err != nil { 394 return false, err 395 } 396 if !verifier.Verified() { 397 err := fmt.Errorf("image verification failed for digest %s", manifestDigest) 398 logrus.Error(err) 399 return false, err 400 } 401 } 402 403 // TODO(tiborvass): what's the usecase for having manifest == nil and err == nil ? Shouldn't be the error be "DoesNotExist" ? 404 if m == nil { 405 return false, fmt.Errorf("image manifest does not exist for tag %q", tag) 406 } 407 if m.SchemaVersion != 1 { 408 return false, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag) 409 } 410 if len(m.FSLayers) != len(m.History) { 411 return false, fmt.Errorf("length of history not equal to number of layers for tag %q", tag) 412 } 413 if len(m.FSLayers) == 0 { 414 return false, fmt.Errorf("no FSLayers in manifest for tag %q", tag) 415 } 416 keys, err := manifest.Verify(m) 417 if err != nil { 418 return false, fmt.Errorf("error verifying manifest for tag %q: %v", tag, err) 419 } 420 verified, err = p.verifyTrustedKeys(m.Name, keys) 421 if err != nil { 422 return false, fmt.Errorf("error verifying manifest keys: %v", err) 423 } 424 return verified, nil 425 }