github.com/torfuzx/docker@v1.8.1/graph/pull_v2.go (about) 1 package graph 2 3 import ( 4 "errors" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 10 "github.com/Sirupsen/logrus" 11 "github.com/docker/distribution" 12 "github.com/docker/distribution/digest" 13 "github.com/docker/distribution/manifest" 14 "github.com/docker/docker/image" 15 "github.com/docker/docker/pkg/progressreader" 16 "github.com/docker/docker/pkg/streamformatter" 17 "github.com/docker/docker/pkg/stringid" 18 "github.com/docker/docker/registry" 19 "github.com/docker/docker/trust" 20 "github.com/docker/docker/utils" 21 "github.com/docker/libtrust" 22 "golang.org/x/net/context" 23 ) 24 25 type v2Puller struct { 26 *TagStore 27 endpoint registry.APIEndpoint 28 config *ImagePullConfig 29 sf *streamformatter.StreamFormatter 30 repoInfo *registry.RepositoryInfo 31 repo distribution.Repository 32 sessionID string 33 } 34 35 func (p *v2Puller) Pull(tag string) (fallback bool, err error) { 36 // TODO(tiborvass): was ReceiveTimeout 37 p.repo, err = NewV2Repository(p.repoInfo, p.endpoint, p.config.MetaHeaders, p.config.AuthConfig) 38 if err != nil { 39 logrus.Debugf("Error getting v2 registry: %v", err) 40 return true, err 41 } 42 43 p.sessionID = stringid.GenerateRandomID() 44 45 if err := p.pullV2Repository(tag); err != nil { 46 if registry.ContinueOnError(err) { 47 logrus.Debugf("Error trying v2 registry: %v", err) 48 return true, err 49 } 50 return false, err 51 } 52 return false, nil 53 } 54 55 func (p *v2Puller) pullV2Repository(tag string) (err error) { 56 var tags []string 57 taggedName := p.repoInfo.LocalName 58 if len(tag) > 0 { 59 tags = []string{tag} 60 taggedName = utils.ImageReference(p.repoInfo.LocalName, tag) 61 } else { 62 var err error 63 64 manSvc, err := p.repo.Manifests(context.Background()) 65 if err != nil { 66 return err 67 } 68 69 tags, err = manSvc.Tags() 70 if err != nil { 71 return err 72 } 73 74 } 75 76 c, err := p.poolAdd("pull", taggedName) 77 if err != nil { 78 if c != nil { 79 // Another pull of the same repository is already taking place; just wait for it to finish 80 p.sf.FormatStatus("", "Repository %s already being pulled by another client. Waiting.", p.repoInfo.CanonicalName) 81 <-c 82 return nil 83 } 84 return err 85 } 86 defer p.poolRemove("pull", taggedName) 87 88 var layersDownloaded bool 89 for _, tag := range tags { 90 // pulledNew is true if either new layers were downloaded OR if existing images were newly tagged 91 // TODO(tiborvass): should we change the name of `layersDownload`? What about message in WriteStatus? 92 pulledNew, err := p.pullV2Tag(tag, taggedName) 93 if err != nil { 94 return err 95 } 96 layersDownloaded = layersDownloaded || pulledNew 97 } 98 99 WriteStatus(taggedName, p.config.OutStream, p.sf, layersDownloaded) 100 101 return nil 102 } 103 104 // downloadInfo is used to pass information from download to extractor 105 type downloadInfo struct { 106 img *image.Image 107 tmpFile *os.File 108 digest digest.Digest 109 layer distribution.ReadSeekCloser 110 size int64 111 err chan error 112 out io.Writer // Download progress is written here. 113 } 114 115 type errVerification struct{} 116 117 func (errVerification) Error() string { return "verification failed" } 118 119 func (p *v2Puller) download(di *downloadInfo) { 120 logrus.Debugf("pulling blob %q to %s", di.digest, di.img.ID) 121 122 out := di.out 123 124 if c, err := p.poolAdd("pull", "img:"+di.img.ID); err != nil { 125 if c != nil { 126 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Layer already being pulled by another client. Waiting.", nil)) 127 <-c 128 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) 129 } else { 130 logrus.Debugf("Image (id: %s) pull is already running, skipping: %v", di.img.ID, err) 131 } 132 di.err <- nil 133 return 134 } 135 136 defer p.poolRemove("pull", "img:"+di.img.ID) 137 tmpFile, err := ioutil.TempFile("", "GetImageBlob") 138 if err != nil { 139 di.err <- err 140 return 141 } 142 143 blobs := p.repo.Blobs(nil) 144 145 desc, err := blobs.Stat(nil, di.digest) 146 if err != nil { 147 logrus.Debugf("Error statting layer: %v", err) 148 di.err <- err 149 return 150 } 151 di.size = desc.Size 152 153 layerDownload, err := blobs.Open(nil, di.digest) 154 if err != nil { 155 logrus.Debugf("Error fetching layer: %v", err) 156 di.err <- err 157 return 158 } 159 defer layerDownload.Close() 160 161 verifier, err := digest.NewDigestVerifier(di.digest) 162 if err != nil { 163 di.err <- err 164 return 165 } 166 167 reader := progressreader.New(progressreader.Config{ 168 In: ioutil.NopCloser(io.TeeReader(layerDownload, verifier)), 169 Out: out, 170 Formatter: p.sf, 171 Size: int(di.size), 172 NewLines: false, 173 ID: stringid.TruncateID(di.img.ID), 174 Action: "Downloading", 175 }) 176 io.Copy(tmpFile, reader) 177 178 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Verifying Checksum", nil)) 179 180 if !verifier.Verified() { 181 err = fmt.Errorf("filesystem layer verification failed for digest %s", di.digest) 182 logrus.Error(err) 183 di.err <- err 184 return 185 } 186 187 out.Write(p.sf.FormatProgress(stringid.TruncateID(di.img.ID), "Download complete", nil)) 188 189 logrus.Debugf("Downloaded %s to tempfile %s", di.img.ID, tmpFile.Name()) 190 di.tmpFile = tmpFile 191 di.layer = layerDownload 192 193 di.err <- nil 194 } 195 196 func (p *v2Puller) pullV2Tag(tag, taggedName string) (verified bool, err error) { 197 logrus.Debugf("Pulling tag from V2 registry: %q", tag) 198 out := p.config.OutStream 199 200 manSvc, err := p.repo.Manifests(context.Background()) 201 if err != nil { 202 return false, err 203 } 204 205 manifest, err := manSvc.GetByTag(tag) 206 if err != nil { 207 return false, err 208 } 209 verified, err = p.validateManifest(manifest, tag) 210 if err != nil { 211 return false, err 212 } 213 if verified { 214 logrus.Printf("Image manifest for %s has been verified", taggedName) 215 } 216 217 // By using a pipeWriter for each of the downloads to write their progress 218 // to, we can avoid an issue where this function returns an error but 219 // leaves behind running download goroutines. By splitting the writer 220 // with a pipe, we can close the pipe if there is any error, consequently 221 // causing each download to cancel due to an error writing to this pipe. 222 pipeReader, pipeWriter := io.Pipe() 223 go func() { 224 if _, err := io.Copy(out, pipeReader); err != nil { 225 logrus.Errorf("error copying from layer download progress reader: %s", err) 226 } 227 }() 228 defer func() { 229 if err != nil { 230 // All operations on the pipe are synchronous. This call will wait 231 // until all current readers/writers are done using the pipe then 232 // set the error. All successive reads/writes will return with this 233 // error. 234 pipeWriter.CloseWithError(errors.New("download canceled")) 235 } 236 }() 237 238 out.Write(p.sf.FormatStatus(tag, "Pulling from %s", p.repo.Name())) 239 240 downloads := make([]downloadInfo, len(manifest.FSLayers)) 241 242 layerIDs := []string{} 243 defer func() { 244 p.graph.Release(p.sessionID, layerIDs...) 245 }() 246 247 for i := len(manifest.FSLayers) - 1; i >= 0; i-- { 248 img, err := image.NewImgJSON([]byte(manifest.History[i].V1Compatibility)) 249 if err != nil { 250 logrus.Debugf("error getting image v1 json: %v", err) 251 return false, err 252 } 253 downloads[i].img = img 254 downloads[i].digest = manifest.FSLayers[i].BlobSum 255 256 p.graph.Retain(p.sessionID, img.ID) 257 layerIDs = append(layerIDs, img.ID) 258 259 // Check if exists 260 if p.graph.Exists(img.ID) { 261 logrus.Debugf("Image already exists: %s", img.ID) 262 continue 263 } 264 265 out.Write(p.sf.FormatProgress(stringid.TruncateID(img.ID), "Pulling fs layer", nil)) 266 267 downloads[i].err = make(chan error) 268 downloads[i].out = pipeWriter 269 go p.download(&downloads[i]) 270 } 271 272 var tagUpdated bool 273 for i := len(downloads) - 1; i >= 0; i-- { 274 d := &downloads[i] 275 if d.err != nil { 276 if err := <-d.err; err != nil { 277 return false, err 278 } 279 } 280 if d.layer != nil { 281 // if tmpFile is empty assume download and extracted elsewhere 282 defer os.Remove(d.tmpFile.Name()) 283 defer d.tmpFile.Close() 284 d.tmpFile.Seek(0, 0) 285 if d.tmpFile != nil { 286 287 reader := progressreader.New(progressreader.Config{ 288 In: d.tmpFile, 289 Out: out, 290 Formatter: p.sf, 291 Size: int(d.size), 292 NewLines: false, 293 ID: stringid.TruncateID(d.img.ID), 294 Action: "Extracting", 295 }) 296 297 err = p.graph.Register(d.img, reader) 298 if err != nil { 299 return false, err 300 } 301 302 if err := p.graph.SetDigest(d.img.ID, d.digest); err != nil { 303 return false, err 304 } 305 306 // FIXME: Pool release here for parallel tag pull (ensures any downloads block until fully extracted) 307 } 308 out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Pull complete", nil)) 309 tagUpdated = true 310 } else { 311 out.Write(p.sf.FormatProgress(stringid.TruncateID(d.img.ID), "Already exists", nil)) 312 } 313 } 314 315 manifestDigest, _, err := digestFromManifest(manifest, p.repoInfo.LocalName) 316 if err != nil { 317 return false, err 318 } 319 320 // Check for new tag if no layers downloaded 321 if !tagUpdated { 322 repo, err := p.Get(p.repoInfo.LocalName) 323 if err != nil { 324 return false, err 325 } 326 if repo != nil { 327 if _, exists := repo[tag]; !exists { 328 tagUpdated = true 329 } 330 } else { 331 tagUpdated = true 332 } 333 } 334 335 if verified && tagUpdated { 336 out.Write(p.sf.FormatStatus(p.repo.Name()+":"+tag, "The image you are pulling has been verified. Important: image verification is a tech preview feature and should not be relied on to provide security.")) 337 } 338 339 if utils.DigestReference(tag) { 340 // TODO(stevvooe): Ideally, we should always set the digest so we can 341 // use the digest whether we pull by it or not. Unfortunately, the tag 342 // store treats the digest as a separate tag, meaning there may be an 343 // untagged digest image that would seem to be dangling by a user. 344 if err = p.SetDigest(p.repoInfo.LocalName, tag, downloads[0].img.ID); err != nil { 345 return false, err 346 } 347 } else { 348 // only set the repository/tag -> image ID mapping when pulling by tag (i.e. not by digest) 349 if err = p.Tag(p.repoInfo.LocalName, tag, downloads[0].img.ID, true); err != nil { 350 return false, err 351 } 352 } 353 354 if manifestDigest != "" { 355 out.Write(p.sf.FormatStatus("", "Digest: %s", manifestDigest)) 356 } 357 358 return tagUpdated, nil 359 } 360 361 // verifyTrustedKeys checks the keys provided against the trust store, 362 // ensuring that the provided keys are trusted for the namespace. The keys 363 // provided from this method must come from the signatures provided as part of 364 // the manifest JWS package, obtained from unpackSignedManifest or libtrust. 365 func (p *v2Puller) verifyTrustedKeys(namespace string, keys []libtrust.PublicKey) (verified bool, err error) { 366 if namespace[0] != '/' { 367 namespace = "/" + namespace 368 } 369 370 for _, key := range keys { 371 b, err := key.MarshalJSON() 372 if err != nil { 373 return false, fmt.Errorf("error marshalling public key: %s", err) 374 } 375 // Check key has read/write permission (0x03) 376 v, err := p.trustService.CheckKey(namespace, b, 0x03) 377 if err != nil { 378 vErr, ok := err.(trust.NotVerifiedError) 379 if !ok { 380 return false, fmt.Errorf("error running key check: %s", err) 381 } 382 logrus.Debugf("Key check result: %v", vErr) 383 } 384 verified = v 385 } 386 387 if verified { 388 logrus.Debug("Key check result: verified") 389 } 390 391 return 392 } 393 394 func (p *v2Puller) validateManifest(m *manifest.SignedManifest, tag string) (verified bool, err error) { 395 // If pull by digest, then verify the manifest digest. NOTE: It is 396 // important to do this first, before any other content validation. If the 397 // digest cannot be verified, don't even bother with those other things. 398 if manifestDigest, err := digest.ParseDigest(tag); err == nil { 399 verifier, err := digest.NewDigestVerifier(manifestDigest) 400 if err != nil { 401 return false, err 402 } 403 payload, err := m.Payload() 404 if err != nil { 405 return false, err 406 } 407 if _, err := verifier.Write(payload); err != nil { 408 return false, err 409 } 410 if !verifier.Verified() { 411 err := fmt.Errorf("image verification failed for digest %s", manifestDigest) 412 logrus.Error(err) 413 return false, err 414 } 415 } 416 417 // TODO(tiborvass): what's the usecase for having manifest == nil and err == nil ? Shouldn't be the error be "DoesNotExist" ? 418 if m == nil { 419 return false, fmt.Errorf("image manifest does not exist for tag %q", tag) 420 } 421 if m.SchemaVersion != 1 { 422 return false, fmt.Errorf("unsupported schema version %d for tag %q", m.SchemaVersion, tag) 423 } 424 if len(m.FSLayers) != len(m.History) { 425 return false, fmt.Errorf("length of history not equal to number of layers for tag %q", tag) 426 } 427 if len(m.FSLayers) == 0 { 428 return false, fmt.Errorf("no FSLayers in manifest for tag %q", tag) 429 } 430 keys, err := manifest.Verify(m) 431 if err != nil { 432 return false, fmt.Errorf("error verifying manifest for tag %q: %v", tag, err) 433 } 434 verified, err = p.verifyTrustedKeys(m.Name, keys) 435 if err != nil { 436 return false, fmt.Errorf("error verifying manifest keys: %v", err) 437 } 438 return verified, nil 439 }