github.com/containerd/Containerd@v1.4.13/unpacker.go (about)

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  package containerd
    18  
    19  import (
    20  	"context"
    21  	"encoding/base64"
    22  	"encoding/json"
    23  	"fmt"
    24  	"math/rand"
    25  	"sync"
    26  	"sync/atomic"
    27  	"time"
    28  
    29  	"github.com/containerd/containerd/content"
    30  	"github.com/containerd/containerd/errdefs"
    31  	"github.com/containerd/containerd/images"
    32  	"github.com/containerd/containerd/log"
    33  	"github.com/containerd/containerd/mount"
    34  	"github.com/containerd/containerd/snapshots"
    35  	"github.com/opencontainers/go-digest"
    36  	"github.com/opencontainers/image-spec/identity"
    37  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    38  	"github.com/pkg/errors"
    39  	"github.com/sirupsen/logrus"
    40  	"golang.org/x/sync/errgroup"
    41  	"golang.org/x/sync/semaphore"
    42  )
    43  
    44  const (
    45  	labelSnapshotRef = "containerd.io/snapshot.ref"
    46  )
    47  
    48  type unpacker struct {
    49  	updateCh    chan ocispec.Descriptor
    50  	snapshotter string
    51  	config      UnpackConfig
    52  	c           *Client
    53  	limiter     *semaphore.Weighted
    54  }
    55  
    56  func (c *Client) newUnpacker(ctx context.Context, rCtx *RemoteContext) (*unpacker, error) {
    57  	snapshotter, err := c.resolveSnapshotterName(ctx, rCtx.Snapshotter)
    58  	if err != nil {
    59  		return nil, err
    60  	}
    61  	var config UnpackConfig
    62  	for _, o := range rCtx.UnpackOpts {
    63  		if err := o(ctx, &config); err != nil {
    64  			return nil, err
    65  		}
    66  	}
    67  	return &unpacker{
    68  		updateCh:    make(chan ocispec.Descriptor, 128),
    69  		snapshotter: snapshotter,
    70  		config:      config,
    71  		c:           c,
    72  	}, nil
    73  }
    74  
    75  func (u *unpacker) unpack(
    76  	ctx context.Context,
    77  	rCtx *RemoteContext,
    78  	h images.Handler,
    79  	config ocispec.Descriptor,
    80  	layers []ocispec.Descriptor,
    81  ) error {
    82  	p, err := content.ReadBlob(ctx, u.c.ContentStore(), config)
    83  	if err != nil {
    84  		return err
    85  	}
    86  
    87  	var i ocispec.Image
    88  	if err := json.Unmarshal(p, &i); err != nil {
    89  		return errors.Wrap(err, "unmarshal image config")
    90  	}
    91  	diffIDs := i.RootFS.DiffIDs
    92  	if len(layers) != len(diffIDs) {
    93  		return errors.Errorf("number of layers and diffIDs don't match: %d != %d", len(layers), len(diffIDs))
    94  	}
    95  
    96  	var (
    97  		sn = u.c.SnapshotService(u.snapshotter)
    98  		a  = u.c.DiffService()
    99  		cs = u.c.ContentStore()
   100  
   101  		chain []digest.Digest
   102  
   103  		fetchOffset int
   104  		fetchC      []chan struct{}
   105  		fetchErr    chan error
   106  	)
   107  
   108  	// If there is an early return, ensure any ongoing
   109  	// fetches get their context cancelled
   110  	ctx, cancel := context.WithCancel(ctx)
   111  	defer cancel()
   112  
   113  EachLayer:
   114  	for i, desc := range layers {
   115  		parent := identity.ChainID(chain)
   116  		chain = append(chain, diffIDs[i])
   117  
   118  		chainID := identity.ChainID(chain).String()
   119  		if _, err := sn.Stat(ctx, chainID); err == nil {
   120  			// no need to handle
   121  			continue
   122  		} else if !errdefs.IsNotFound(err) {
   123  			return errors.Wrapf(err, "failed to stat snapshot %s", chainID)
   124  		}
   125  
   126  		// inherits annotations which are provided as snapshot labels.
   127  		labels := snapshots.FilterInheritedLabels(desc.Annotations)
   128  		if labels == nil {
   129  			labels = make(map[string]string)
   130  		}
   131  		labels[labelSnapshotRef] = chainID
   132  
   133  		var (
   134  			key    string
   135  			mounts []mount.Mount
   136  			opts   = append(rCtx.SnapshotterOpts, snapshots.WithLabels(labels))
   137  		)
   138  
   139  		for try := 1; try <= 3; try++ {
   140  			// Prepare snapshot with from parent, label as root
   141  			key = fmt.Sprintf("extract-%s %s", uniquePart(), chainID)
   142  			mounts, err = sn.Prepare(ctx, key, parent.String(), opts...)
   143  			if err != nil {
   144  				if errdefs.IsAlreadyExists(err) {
   145  					if _, err := sn.Stat(ctx, chainID); err != nil {
   146  						if !errdefs.IsNotFound(err) {
   147  							return errors.Wrapf(err, "failed to stat snapshot %s", chainID)
   148  						}
   149  						// Try again, this should be rare, log it
   150  						log.G(ctx).WithField("key", key).WithField("chainid", chainID).Debug("extraction snapshot already exists, chain id not found")
   151  					} else {
   152  						// no need to handle, snapshot now found with chain id
   153  						continue EachLayer
   154  					}
   155  				} else {
   156  					return errors.Wrapf(err, "failed to prepare extraction snapshot %q", key)
   157  				}
   158  			} else {
   159  				break
   160  			}
   161  		}
   162  		if err != nil {
   163  			return errors.Wrap(err, "unable to prepare extraction snapshot")
   164  		}
   165  
   166  		// Abort the snapshot if commit does not happen
   167  		abort := func() {
   168  			if err := sn.Remove(ctx, key); err != nil {
   169  				log.G(ctx).WithError(err).Errorf("failed to cleanup %q", key)
   170  			}
   171  		}
   172  
   173  		if fetchErr == nil {
   174  			fetchErr = make(chan error, 1)
   175  			fetchOffset = i
   176  			fetchC = make([]chan struct{}, len(layers)-fetchOffset)
   177  			for i := range fetchC {
   178  				fetchC[i] = make(chan struct{})
   179  			}
   180  
   181  			go func(i int) {
   182  				err := u.fetch(ctx, h, layers[i:], fetchC)
   183  				if err != nil {
   184  					fetchErr <- err
   185  				}
   186  				close(fetchErr)
   187  			}(i)
   188  		}
   189  
   190  		select {
   191  		case <-ctx.Done():
   192  			return ctx.Err()
   193  		case err := <-fetchErr:
   194  			if err != nil {
   195  				return err
   196  			}
   197  		case <-fetchC[i-fetchOffset]:
   198  		}
   199  
   200  		diff, err := a.Apply(ctx, desc, mounts, u.config.ApplyOpts...)
   201  		if err != nil {
   202  			abort()
   203  			return errors.Wrapf(err, "failed to extract layer %s", diffIDs[i])
   204  		}
   205  		if diff.Digest != diffIDs[i] {
   206  			abort()
   207  			return errors.Errorf("wrong diff id calculated on extraction %q", diffIDs[i])
   208  		}
   209  
   210  		if err = sn.Commit(ctx, chainID, key, opts...); err != nil {
   211  			abort()
   212  			if errdefs.IsAlreadyExists(err) {
   213  				continue
   214  			}
   215  			return errors.Wrapf(err, "failed to commit snapshot %s", key)
   216  		}
   217  
   218  		// Set the uncompressed label after the uncompressed
   219  		// digest has been verified through apply.
   220  		cinfo := content.Info{
   221  			Digest: desc.Digest,
   222  			Labels: map[string]string{
   223  				"containerd.io/uncompressed": diff.Digest.String(),
   224  			},
   225  		}
   226  		if _, err := cs.Update(ctx, cinfo, "labels.containerd.io/uncompressed"); err != nil {
   227  			return err
   228  		}
   229  
   230  	}
   231  
   232  	chainID := identity.ChainID(chain).String()
   233  	cinfo := content.Info{
   234  		Digest: config.Digest,
   235  		Labels: map[string]string{
   236  			fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", u.snapshotter): chainID,
   237  		},
   238  	}
   239  	_, err = cs.Update(ctx, cinfo, fmt.Sprintf("labels.containerd.io/gc.ref.snapshot.%s", u.snapshotter))
   240  	if err != nil {
   241  		return err
   242  	}
   243  	log.G(ctx).WithFields(logrus.Fields{
   244  		"config":  config.Digest,
   245  		"chainID": chainID,
   246  	}).Debug("image unpacked")
   247  
   248  	return nil
   249  }
   250  
   251  func (u *unpacker) fetch(ctx context.Context, h images.Handler, layers []ocispec.Descriptor, done []chan struct{}) error {
   252  	eg, ctx2 := errgroup.WithContext(ctx)
   253  	for i, desc := range layers {
   254  		desc := desc
   255  		i := i
   256  
   257  		if u.limiter != nil {
   258  			if err := u.limiter.Acquire(ctx, 1); err != nil {
   259  				return err
   260  			}
   261  		}
   262  
   263  		eg.Go(func() error {
   264  			_, err := h.Handle(ctx2, desc)
   265  			if u.limiter != nil {
   266  				u.limiter.Release(1)
   267  			}
   268  			if err != nil && !errors.Is(err, images.ErrSkipDesc) {
   269  				return err
   270  			}
   271  			close(done[i])
   272  
   273  			return nil
   274  		})
   275  	}
   276  
   277  	return eg.Wait()
   278  }
   279  
   280  func (u *unpacker) handlerWrapper(
   281  	uctx context.Context,
   282  	rCtx *RemoteContext,
   283  	unpacks *int32,
   284  ) (func(images.Handler) images.Handler, *errgroup.Group) {
   285  	eg, uctx := errgroup.WithContext(uctx)
   286  	return func(f images.Handler) images.Handler {
   287  		var (
   288  			lock   sync.Mutex
   289  			layers = map[digest.Digest][]ocispec.Descriptor{}
   290  		)
   291  		return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) {
   292  			children, err := f.Handle(ctx, desc)
   293  			if err != nil {
   294  				return children, err
   295  			}
   296  
   297  			switch desc.MediaType {
   298  			case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest:
   299  				var nonLayers []ocispec.Descriptor
   300  				var manifestLayers []ocispec.Descriptor
   301  
   302  				// Split layers from non-layers, layers will be handled after
   303  				// the config
   304  				for _, child := range children {
   305  					if images.IsLayerType(child.MediaType) {
   306  						manifestLayers = append(manifestLayers, child)
   307  					} else {
   308  						nonLayers = append(nonLayers, child)
   309  					}
   310  				}
   311  
   312  				lock.Lock()
   313  				for _, nl := range nonLayers {
   314  					layers[nl.Digest] = manifestLayers
   315  				}
   316  				lock.Unlock()
   317  
   318  				children = nonLayers
   319  			case images.MediaTypeDockerSchema2Config, ocispec.MediaTypeImageConfig:
   320  				lock.Lock()
   321  				l := layers[desc.Digest]
   322  				lock.Unlock()
   323  				if len(l) > 0 {
   324  					atomic.AddInt32(unpacks, 1)
   325  					eg.Go(func() error {
   326  						return u.unpack(uctx, rCtx, f, desc, l)
   327  					})
   328  				}
   329  			}
   330  			return children, nil
   331  		})
   332  	}, eg
   333  }
   334  
   335  func uniquePart() string {
   336  	t := time.Now()
   337  	var b [3]byte
   338  	// Ignore read failures, just decreases uniqueness
   339  	rand.Read(b[:])
   340  	return fmt.Sprintf("%d-%s", t.Nanosecond(), base64.URLEncoding.EncodeToString(b[:]))
   341  }