github.com/containerd/containerd@v22.0.0-20200918172823-438c87b8e050+incompatible/pull.go (about)

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  package containerd
    18  
    19  import (
    20  	"context"
    21  
    22  	"github.com/containerd/containerd/errdefs"
    23  	"github.com/containerd/containerd/images"
    24  	"github.com/containerd/containerd/platforms"
    25  	"github.com/containerd/containerd/remotes"
    26  	"github.com/containerd/containerd/remotes/docker"
    27  	"github.com/containerd/containerd/remotes/docker/schema1"
    28  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    29  	"github.com/pkg/errors"
    30  	"golang.org/x/sync/errgroup"
    31  	"golang.org/x/sync/semaphore"
    32  )
    33  
    34  // Pull downloads the provided content into containerd's content store
    35  // and returns a platform specific image object
    36  func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (_ Image, retErr error) {
    37  	pullCtx := defaultRemoteContext()
    38  	for _, o := range opts {
    39  		if err := o(c, pullCtx); err != nil {
    40  			return nil, err
    41  		}
    42  	}
    43  
    44  	if pullCtx.PlatformMatcher == nil {
    45  		if len(pullCtx.Platforms) > 1 {
    46  			return nil, errors.New("cannot pull multiplatform image locally, try Fetch")
    47  		} else if len(pullCtx.Platforms) == 0 {
    48  			pullCtx.PlatformMatcher = c.platform
    49  		} else {
    50  			p, err := platforms.Parse(pullCtx.Platforms[0])
    51  			if err != nil {
    52  				return nil, errors.Wrapf(err, "invalid platform %s", pullCtx.Platforms[0])
    53  			}
    54  
    55  			pullCtx.PlatformMatcher = platforms.Only(p)
    56  		}
    57  	}
    58  
    59  	ctx, done, err := c.WithLease(ctx)
    60  	if err != nil {
    61  		return nil, err
    62  	}
    63  	defer done(ctx)
    64  
    65  	var unpacks int32
    66  	var unpackEg *errgroup.Group
    67  	var unpackWrapper func(f images.Handler) images.Handler
    68  
    69  	if pullCtx.Unpack {
    70  		// unpacker only supports schema 2 image, for schema 1 this is noop.
    71  		u, err := c.newUnpacker(ctx, pullCtx)
    72  		if err != nil {
    73  			return nil, errors.Wrap(err, "create unpacker")
    74  		}
    75  		unpackWrapper, unpackEg = u.handlerWrapper(ctx, pullCtx, &unpacks)
    76  		defer func() {
    77  			if err := unpackEg.Wait(); err != nil {
    78  				if retErr == nil {
    79  					retErr = errors.Wrap(err, "unpack")
    80  				}
    81  			}
    82  		}()
    83  		wrapper := pullCtx.HandlerWrapper
    84  		pullCtx.HandlerWrapper = func(h images.Handler) images.Handler {
    85  			if wrapper == nil {
    86  				return unpackWrapper(h)
    87  			}
    88  			return unpackWrapper(wrapper(h))
    89  		}
    90  	}
    91  
    92  	img, err := c.fetch(ctx, pullCtx, ref, 1)
    93  	if err != nil {
    94  		return nil, err
    95  	}
    96  
    97  	// NOTE(fuweid): unpacker defers blobs download. before create image
    98  	// record in ImageService, should wait for unpacking(including blobs
    99  	// download).
   100  	if pullCtx.Unpack {
   101  		if unpackEg != nil {
   102  			if err := unpackEg.Wait(); err != nil {
   103  				return nil, err
   104  			}
   105  		}
   106  	}
   107  
   108  	img, err = c.createNewImage(ctx, img)
   109  	if err != nil {
   110  		return nil, err
   111  	}
   112  
   113  	i := NewImageWithPlatform(c, img, pullCtx.PlatformMatcher)
   114  
   115  	if pullCtx.Unpack {
   116  		if unpacks == 0 {
   117  			// Try to unpack is none is done previously.
   118  			// This is at least required for schema 1 image.
   119  			if err := i.Unpack(ctx, pullCtx.Snapshotter, pullCtx.UnpackOpts...); err != nil {
   120  				return nil, errors.Wrapf(err, "failed to unpack image on snapshotter %s", pullCtx.Snapshotter)
   121  			}
   122  		}
   123  	}
   124  
   125  	return i, nil
   126  }
   127  
   128  func (c *Client) fetch(ctx context.Context, rCtx *RemoteContext, ref string, limit int) (images.Image, error) {
   129  	store := c.ContentStore()
   130  	name, desc, err := rCtx.Resolver.Resolve(ctx, ref)
   131  	if err != nil {
   132  		return images.Image{}, errors.Wrapf(err, "failed to resolve reference %q", ref)
   133  	}
   134  
   135  	fetcher, err := rCtx.Resolver.Fetcher(ctx, name)
   136  	if err != nil {
   137  		return images.Image{}, errors.Wrapf(err, "failed to get fetcher for %q", name)
   138  	}
   139  
   140  	var (
   141  		handler images.Handler
   142  
   143  		isConvertible bool
   144  		converterFunc func(context.Context, ocispec.Descriptor) (ocispec.Descriptor, error)
   145  		limiter       *semaphore.Weighted
   146  	)
   147  
   148  	if desc.MediaType == images.MediaTypeDockerSchema1Manifest && rCtx.ConvertSchema1 {
   149  		schema1Converter := schema1.NewConverter(store, fetcher)
   150  
   151  		handler = images.Handlers(append(rCtx.BaseHandlers, schema1Converter)...)
   152  
   153  		isConvertible = true
   154  
   155  		converterFunc = func(ctx context.Context, _ ocispec.Descriptor) (ocispec.Descriptor, error) {
   156  			return schema1Converter.Convert(ctx)
   157  		}
   158  	} else {
   159  		// Get all the children for a descriptor
   160  		childrenHandler := images.ChildrenHandler(store)
   161  		// Set any children labels for that content
   162  		childrenHandler = images.SetChildrenMappedLabels(store, childrenHandler, rCtx.ChildLabelMap)
   163  		if rCtx.AllMetadata {
   164  			// Filter manifests by platforms but allow to handle manifest
   165  			// and configuration for not-target platforms
   166  			childrenHandler = remotes.FilterManifestByPlatformHandler(childrenHandler, rCtx.PlatformMatcher)
   167  		} else {
   168  			// Filter children by platforms if specified.
   169  			childrenHandler = images.FilterPlatforms(childrenHandler, rCtx.PlatformMatcher)
   170  		}
   171  		// Sort and limit manifests if a finite number is needed
   172  		if limit > 0 {
   173  			childrenHandler = images.LimitManifests(childrenHandler, rCtx.PlatformMatcher, limit)
   174  		}
   175  
   176  		// set isConvertible to true if there is application/octet-stream media type
   177  		convertibleHandler := images.HandlerFunc(
   178  			func(_ context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
   179  				if desc.MediaType == docker.LegacyConfigMediaType {
   180  					isConvertible = true
   181  				}
   182  
   183  				return []ocispec.Descriptor{}, nil
   184  			},
   185  		)
   186  
   187  		appendDistSrcLabelHandler, err := docker.AppendDistributionSourceLabel(store, ref)
   188  		if err != nil {
   189  			return images.Image{}, err
   190  		}
   191  
   192  		handlers := append(rCtx.BaseHandlers,
   193  			remotes.FetchHandler(store, fetcher),
   194  			convertibleHandler,
   195  			childrenHandler,
   196  			appendDistSrcLabelHandler,
   197  		)
   198  
   199  		handler = images.Handlers(handlers...)
   200  
   201  		converterFunc = func(ctx context.Context, desc ocispec.Descriptor) (ocispec.Descriptor, error) {
   202  			return docker.ConvertManifest(ctx, store, desc)
   203  		}
   204  	}
   205  
   206  	if rCtx.HandlerWrapper != nil {
   207  		handler = rCtx.HandlerWrapper(handler)
   208  	}
   209  
   210  	if rCtx.MaxConcurrentDownloads > 0 {
   211  		limiter = semaphore.NewWeighted(int64(rCtx.MaxConcurrentDownloads))
   212  	}
   213  
   214  	if err := images.Dispatch(ctx, handler, limiter, desc); err != nil {
   215  		return images.Image{}, err
   216  	}
   217  
   218  	if isConvertible {
   219  		if desc, err = converterFunc(ctx, desc); err != nil {
   220  			return images.Image{}, err
   221  		}
   222  	}
   223  
   224  	return images.Image{
   225  		Name:   name,
   226  		Target: desc,
   227  		Labels: rCtx.Labels,
   228  	}, nil
   229  }
   230  
   231  func (c *Client) createNewImage(ctx context.Context, img images.Image) (images.Image, error) {
   232  	is := c.ImageService()
   233  	for {
   234  		if created, err := is.Create(ctx, img); err != nil {
   235  			if !errdefs.IsAlreadyExists(err) {
   236  				return images.Image{}, err
   237  			}
   238  
   239  			updated, err := is.Update(ctx, img)
   240  			if err != nil {
   241  				// if image was removed, try create again
   242  				if errdefs.IsNotFound(err) {
   243  					continue
   244  				}
   245  				return images.Image{}, err
   246  			}
   247  
   248  			img = updated
   249  		} else {
   250  			img = created
   251  		}
   252  
   253  		return img, nil
   254  	}
   255  }