github.com/csfrancis/docker@v1.8.0-rc2/graph/pull_v2.go (about) 1 package graph 2 3 import ( 4 "fmt" 5 "io" 6 "io/ioutil" 7 "os" 8 9 "github.com/Sirupsen/logrus" 10 "github.com/docker/distribution" 11 "github.com/docker/distribution/digest" 12 "github.com/docker/distribution/manifest" 13 "github.com/docker/docker/image" 14 "github.com/docker/docker/pkg/progressreader" 15 "github.com/docker/docker/pkg/streamformatter" 16 "github.com/docker/docker/pkg/stringid" 17 "github.com/docker/docker/registry" 18 "github.com/docker/docker/trust" 19 "github.com/docker/docker/utils" 20 "github.com/docker/libtrust" 21 "golang.org/x/net/context" 22 ) 23 24 type v2Puller struct { 25 *TagStore 26 endpoint registry.APIEndpoint 27 config *ImagePullConfig 28 sf *streamformatter.StreamFormatter 29 repoInfo *registry.RepositoryInfo 30 repo distribution.Repository 31 sessionID string 32 } 33 34 func (p *v2Puller) Pull(tag string) (fallback bool, err error) { 35 // TODO(tiborvass): was ReceiveTimeout 36 p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig) 37 if err != nil { 38 logrus.Debugf("Error getting v2 registry: %v", err) 39 return true, err 40 } 41 42 p.sessionID = stringid.GenerateRandomID() 43 44 if err := p.pullV2Repository(tag); err != nil { 45 if registry.ContinueOnError(err) { 46 logrus.Debugf("Error trying v2 registry: %v", err) 47 return true, err 48 } 49 return false, err 50 } 51 return false, nil 52 } 53 54 func (p *v2Puller) pullV2Repository(tag string) (err error) { 55 var tags []string 56 taggedName := p.repoInfo.LocalName 57 if len(tag) > 0 { 58 tags = []string{tag} 59 taggedName = utils.ImageReference(p.repoInfo.LocalName, tag) 60 } else { 61 var err error 62 63 manSvc, err := p.repo.Manifests(context.Background()) 64 if err != nil { 65 return err 66 } 67 68 tags, err = manSvc.Tags() 69 if err != nil { 70 return err 71 } 72 73 } 74 75 c, err := p.poolAdd("pull", taggedName) 76 if err != nil { 77 if c != nil { 78 // Another pull of the same repository is already taking place; just wait for it to finish 79 p.sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", p.repoInfo.CanonicalName) 80 <-c 81 return nil 82 } 83 return err 84 } 85 defer p.poolRemove("pull", taggedName) 86 87 var layersDownloaded bool 88 for _, tag := range tags { 89 // pulledNew is true if either new layers were downloaded OR if existing images were newly tagged 90 // TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus? 91 pulledNew, err := p.pullV2Tag(tag, taggedName) 92 if err != nil { 93 return err 94 } 95 layersDownloaded = layersDownloaded || pulledNew 96 } 97 98 WriteStatus(taggedName, p.config.OutStream, p.sf, layersDownloaded) 99 100 return nil 101 } 102 103 // downloadInfo is used to pass information from download to extractor 104 type downloadInfo struct { 105 img *image.Image 106 tmpFile *os.File 107 digest digest.Digest 108 layer distribution.ReadSeekCloser 109 size int64 110 err chan error 111 } 112 113 type errVerification struct{} 114 115 func (errVerification) Error() string { return "verification failed" } 116 117 func (p *v2Puller) download(di *downloadInfo) { 118 logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID) 119 120 out := p.config.OutStream 121 122 if c, err := p.poolAdd("pull", "img:"+di.img.ID); err != nil { 123 if c != nil { 124 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Layer already being pulled by another client. Waiting.", nil)) 125 <-c 126 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) 127 } else { 128 logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", di.img.ID, err) 129 } 130 di.err <- nil 131 return 132 } 133 134 defer p.poolRemove("pull", "img:"+di.img.ID) 135 tmpFile, err := ioutil.TempFile("", "GetImageBlob") 136 if err != nil { 137 di.err <- err 138 return 139 } 140 141 blobs := p.repo.Blobs(nil) 142 143 desc, err := blobs.Stat(nil, di.digest) 144 if err != nil { 145 logrus.Debugf("Error statting layer: %v", err) 146 di.err <- err 147 return 148 } 149 di.size = desc.Size 150 151 layerDownload, err := blobs.Open(nil, di.digest) 152 if err != nil { 153 logrus.Debugf("Error fetching layer: %v", err) 154 di.err <- err 155 return 156 } 157 defer layerDownload.Close() 158 159 verifier, err := digest.NewDigestVerifier(di.digest) 160 if err != nil { 161 di.err <- err 162 return 163 } 164 165 reader := progressreader.New(progressreader.Config{ 166 In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)), 167 Out: out, 168 Formatter: p.sf, 169 Size: int(di.size), 170 NewLines: false, 171 ID: stringid.TruncateID(di.img.ID), 172 Action: "Downloading", 173 }) 174 io.Copy(tmpFile, reader) 175 176 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil)) 177 178 if !verifier.Verified() { 179 err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest) 180 logrus.Error(err) 181 di.err <- err 182 return 183 } 184 185 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) 186 187 logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, tmpFile.Name()) 188 di.tmpFile = tmpFile 189 di.layer = layerDownload 190 191 di.err <- nil 192 } 193 194 func (p *v2Puller) pullV2Tag(tag, taggedName string) (bool, error) { 195 logrus.Debugf("Pulling tag from V2 registry: %q", tag) 196 out := p.config.OutStream 197 198 manSvc, err := p.repo.Manifests(context.Background()) 199 if err != nil { 200 return false, err 201 } 202 203 manifest, err := manSvc.GetByTag(tag) 204 if err != nil { 205 return false, err 206 } 207 verified, err := p.validateManifest(manifest, tag) 208 if err != nil { 209 return false, err 210 } 211 if verified { 212 logrus.Printf("Image manifest for %s has been verified", taggedName) 213 } 214 215 out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name())) 216 217 downloads := make([]downloadInfo, len(manifest.FSLayers)) 218 219 layerIDs := []string{} 220 defer func() { 221 p.graph.Release(p.sessionID, layerIDs...) 222 }() 223 224 for i := len(manifest.FSLayers) - 1; i >= 0; i-- { 225 img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility)) 226 if err != nil { 227 logrus.Debugf("error getting image v1 json: %v", err) 228 return false, err 229 } 230 downloads[i].img = img 231 downloads[i].digest = manifest.FSLayers[i].BlobSum 232 233 p.graph.Retain(p.sessionID, img.ID) 234 layerIDs = append(layerIDs, img.ID) 235 236 // Check if exists 237 if p.graph.Exists(img.ID) { 238 logrus.Debugf("Image already exists: %s", img.ID) 239 continue 240 } 241 242 out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) 243 244 downloads[i].err = make(chan error) 245 go p.download(&downloads[i]) 246 } 247 248 var tagUpdated bool 249 for i := len(downloads) - 1; i >= 0; i-- { 250 d := &downloads[i] 251 if d.err != nil { 252 if err := <-d.err; err != nil { 253 return false, err 254 } 255 } 256 if d.layer != nil { 257 // if tmpFile is empty assume download and extracted elsewhere 258 defer os.Remove(d.tmpFile.Name()) 259 defer d.tmpFile.Close() 260 d.tmpFile.Seek(0, 0) 261 if d.tmpFile != nil { 262 263 reader := progressreader.New(progressreader.Config{ 264 In: d.tmpFile, 265 Out: out, 266 Formatter: p.sf, 267 Size: int(d.size), 268 NewLines: false, 269 ID: stringid.TruncateID(d.img.ID), 270 Action: "Extracting", 271 }) 272 273 err = p.graph.Register(d.img, reader) 274 if err != nil { 275 return false, err 276 } 277 278 if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil { 279 return false, err 280 } 281 282 // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) 283 } 284 out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) 285 tagUpdated = true 286 } else { 287 out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil)) 288 } 289 } 290 291 manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName) 292 if err != nil { 293 return false, err 294 } 295 296 // Check for new tag if no layers downloaded 297 if !tagUpdated { 298 repo, err := p.Get(p.repoInfo.LocalName) 299 if err != nil { 300 return false, err 301 } 302 if repo != nil { 303 if _, exists := repo[tag]; !exists { 304 tagUpdated = true 305 } 306 } else { 307 tagUpdated = true 308 } 309 } 310 311 if verified && tagUpdated { 312 out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) 313 } 314 315 if utils.DigestReference(tag) { 316 // TODO(stevvooe): Ideally, we should always set the digest so we can 317 // use the digest whether we pull by it or not. Unfortunately, the tag 318 // store treats the digest as a separate tag, meaning there may be an 319 // untagged digest image that would seem to be dangling by a user. 320 if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.ID); err != nil { 321 return false, err 322 } 323 } else { 324 // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) 325 if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { 326 return false, err 327 } 328 } 329 330 if manifestDigest != "" { 331 out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest)) 332 } 333 334 return tagUpdated, nil 335 } 336 337 // verifyTrustedKeys checks the keys provided against the trust store, 338 // ensuring that the provided keys are trusted for the namespace. The keys 339 // provided from this method must come from the signatures provided as part of 340 // the manifest JWS package, obtained from unpackSignedManifest or libtrust. 341 func (p *v2Puller) verifyTrustedKeys(namespace string, keys []libtrust.PublicKey) (verified bool, err error) { 342 if namespace[0] != '/' { 343 namespace = "/" + namespace 344 } 345 346 for _, key := range keys { 347 b, err := key.MarshalJSON() 348 if err != nil { 349 return false, fmt.Errorf("error marshalling public key: %s", err) 350 } 351 // Check key has read/write permission (0x03) 352 v, err := p.trustService.CheckKey(namespace, b, 0x03) 353 if err != nil { 354 vErr, ok := err.(trust.NotVerifiedError) 355 if !ok { 356 return false, fmt.Errorf("error running key check: %s", err) 357 } 358 logrus.Debugf("Key check result: %v", vErr) 359 } 360 verified = v 361 } 362 363 if verified { 364 logrus.Debug("Key check result: verified") 365 } 366 367 return 368 } 369 370 func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (verified bool, err error) { 371 // If pull by digest, then verify the manifest digest. NOTE: It is 372 // important to do this first, before any other content validation. If the 373 // digest cannot be verified, don't even bother with those other things. 374 if manifestDigest, err := digest.ParseDigest(tag); err == nil { 375 verifier, err := digest.NewDigestVerifier(manifestDigest) 376 if err != nil { 377 return false, err 378 } 379 payload, err := m.Payload() 380 if err != nil { 381 return false, err 382 } 383 if _, err := verifier.Write(payload); err != nil { 384 return false, err 385 } 386 if !verifier.Verified() { 387 err := fmt.Errorf("image verification failed for digest %s", manifestDigest) 388 logrus.Error(err) 389 return false, err 390 } 391 } 392 393 // TODO(tiborvass): what's the usecase for having manifest == nil and err == nil ? Shouldn't be the error be "DoesNotExist" ? 394 if m == nil { 395 return false, fmt.Errorf("image manifest does not exist for tag %q", tag) 396 } 397 if m.SchemaVersion != 1 { 398 return false, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag) 399 } 400 if len(m.FSLayers) != len(m.History) { 401 return false, fmt.Errorf("length of history not equal to number of layers for tag %q", tag) 402 } 403 if len(m.FSLayers) == 0 { 404 return false, fmt.Errorf("no FSLayers in manifest for tag %q", tag) 405 } 406 keys, err := manifest.Verify(m) 407 if err != nil { 408 return false, fmt.Errorf("error verifying manifest for tag %q: %v", tag, err) 409 } 410 verified, err = p.verifyTrustedKeys(m.Name, keys) 411 if err != nil { 412 return false, fmt.Errorf("error verifying manifest keys: %v", err) 413 } 414 return verified, nil 415 }