github.com/containerd/containerd@v22.0.0-20200918172823-438c87b8e050+incompatible/images/archive/importer.go (about)

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  // Package archive provides a Docker and OCI compatible importer
    18  package archive
    19  
    20  import (
    21  	"archive/tar"
    22  	"bytes"
    23  	"context"
    24  	"encoding/json"
    25  	"fmt"
    26  	"io"
    27  	"io/ioutil"
    28  	"path"
    29  
    30  	"github.com/containerd/containerd/archive/compression"
    31  	"github.com/containerd/containerd/content"
    32  	"github.com/containerd/containerd/errdefs"
    33  	"github.com/containerd/containerd/images"
    34  	"github.com/containerd/containerd/log"
    35  	digest "github.com/opencontainers/go-digest"
    36  	specs "github.com/opencontainers/image-spec/specs-go"
    37  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    38  	"github.com/pkg/errors"
    39  )
    40  
    41  type importOpts struct {
    42  	compress bool
    43  }
    44  
    45  // ImportOpt is an option for importing an OCI index
    46  type ImportOpt func(*importOpts) error
    47  
    48  // WithImportCompression compresses uncompressed layers on import.
    49  // This is used for import formats which do not include the manifest.
    50  func WithImportCompression() ImportOpt {
    51  	return func(io *importOpts) error {
    52  		io.compress = true
    53  		return nil
    54  	}
    55  }
    56  
    57  // ImportIndex imports an index from a tar archive image bundle
    58  // - implements Docker v1.1, v1.2 and OCI v1.
    59  // - prefers OCI v1 when provided
    60  // - creates OCI index for Docker formats
    61  // - normalizes Docker references and adds as OCI ref name
    62  //      e.g. alpine:latest -> docker.io/library/alpine:latest
    63  // - existing OCI reference names are untouched
    64  func ImportIndex(ctx context.Context, store content.Store, reader io.Reader, opts ...ImportOpt) (ocispec.Descriptor, error) {
    65  	var (
    66  		tr = tar.NewReader(reader)
    67  
    68  		ociLayout ocispec.ImageLayout
    69  		mfsts     []struct {
    70  			Config   string
    71  			RepoTags []string
    72  			Layers   []string
    73  		}
    74  		symlinks = make(map[string]string)
    75  		blobs    = make(map[string]ocispec.Descriptor)
    76  		iopts    importOpts
    77  	)
    78  
    79  	for _, o := range opts {
    80  		if err := o(&iopts); err != nil {
    81  			return ocispec.Descriptor{}, err
    82  		}
    83  	}
    84  
    85  	for {
    86  		hdr, err := tr.Next()
    87  		if err == io.EOF {
    88  			break
    89  		}
    90  		if err != nil {
    91  			return ocispec.Descriptor{}, err
    92  		}
    93  		if hdr.Typeflag == tar.TypeSymlink {
    94  			symlinks[hdr.Name] = path.Join(path.Dir(hdr.Name), hdr.Linkname)
    95  		}
    96  
    97  		if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA {
    98  			if hdr.Typeflag != tar.TypeDir {
    99  				log.G(ctx).WithField("file", hdr.Name).Debug("file type ignored")
   100  			}
   101  			continue
   102  		}
   103  
   104  		hdrName := path.Clean(hdr.Name)
   105  		if hdrName == ocispec.ImageLayoutFile {
   106  			if err = onUntarJSON(tr, &ociLayout); err != nil {
   107  				return ocispec.Descriptor{}, errors.Wrapf(err, "untar oci layout %q", hdr.Name)
   108  			}
   109  		} else if hdrName == "manifest.json" {
   110  			if err = onUntarJSON(tr, &mfsts); err != nil {
   111  				return ocispec.Descriptor{}, errors.Wrapf(err, "untar manifest %q", hdr.Name)
   112  			}
   113  		} else {
   114  			dgst, err := onUntarBlob(ctx, tr, store, hdr.Size, "tar-"+hdrName)
   115  			if err != nil {
   116  				return ocispec.Descriptor{}, errors.Wrapf(err, "failed to ingest %q", hdr.Name)
   117  			}
   118  
   119  			blobs[hdrName] = ocispec.Descriptor{
   120  				Digest: dgst,
   121  				Size:   hdr.Size,
   122  			}
   123  		}
   124  	}
   125  
   126  	// If OCI layout was given, interpret the tar as an OCI layout.
   127  	// When not provided, the layout of the tar will be interpreted
   128  	// as Docker v1.1 or v1.2.
   129  	if ociLayout.Version != "" {
   130  		if ociLayout.Version != ocispec.ImageLayoutVersion {
   131  			return ocispec.Descriptor{}, errors.Errorf("unsupported OCI version %s", ociLayout.Version)
   132  		}
   133  
   134  		idx, ok := blobs["index.json"]
   135  		if !ok {
   136  			return ocispec.Descriptor{}, errors.Errorf("missing index.json in OCI layout %s", ocispec.ImageLayoutVersion)
   137  		}
   138  
   139  		idx.MediaType = ocispec.MediaTypeImageIndex
   140  		return idx, nil
   141  	}
   142  
   143  	if mfsts == nil {
   144  		return ocispec.Descriptor{}, errors.Errorf("unrecognized image format")
   145  	}
   146  
   147  	for name, linkname := range symlinks {
   148  		desc, ok := blobs[linkname]
   149  		if !ok {
   150  			return ocispec.Descriptor{}, errors.Errorf("no target for symlink layer from %q to %q", name, linkname)
   151  		}
   152  		blobs[name] = desc
   153  	}
   154  
   155  	idx := ocispec.Index{
   156  		Versioned: specs.Versioned{
   157  			SchemaVersion: 2,
   158  		},
   159  	}
   160  	for _, mfst := range mfsts {
   161  		config, ok := blobs[mfst.Config]
   162  		if !ok {
   163  			return ocispec.Descriptor{}, errors.Errorf("image config %q not found", mfst.Config)
   164  		}
   165  		config.MediaType = images.MediaTypeDockerSchema2Config
   166  
   167  		layers, err := resolveLayers(ctx, store, mfst.Layers, blobs, iopts.compress)
   168  		if err != nil {
   169  			return ocispec.Descriptor{}, errors.Wrap(err, "failed to resolve layers")
   170  		}
   171  
   172  		manifest := struct {
   173  			SchemaVersion int                  `json:"schemaVersion"`
   174  			MediaType     string               `json:"mediaType"`
   175  			Config        ocispec.Descriptor   `json:"config"`
   176  			Layers        []ocispec.Descriptor `json:"layers"`
   177  		}{
   178  			SchemaVersion: 2,
   179  			MediaType:     images.MediaTypeDockerSchema2Manifest,
   180  			Config:        config,
   181  			Layers:        layers,
   182  		}
   183  
   184  		desc, err := writeManifest(ctx, store, manifest, manifest.MediaType)
   185  		if err != nil {
   186  			return ocispec.Descriptor{}, errors.Wrap(err, "write docker manifest")
   187  		}
   188  
   189  		platforms, err := images.Platforms(ctx, store, desc)
   190  		if err != nil {
   191  			return ocispec.Descriptor{}, errors.Wrap(err, "unable to resolve platform")
   192  		}
   193  		if len(platforms) > 0 {
   194  			// Only one platform can be resolved from non-index manifest,
   195  			// The platform can only come from the config included above,
   196  			// if the config has no platform it can be safely omitted.
   197  			desc.Platform = &platforms[0]
   198  		}
   199  
   200  		if len(mfst.RepoTags) == 0 {
   201  			idx.Manifests = append(idx.Manifests, desc)
   202  		} else {
   203  			// Add descriptor per tag
   204  			for _, ref := range mfst.RepoTags {
   205  				mfstdesc := desc
   206  
   207  				normalized, err := normalizeReference(ref)
   208  				if err != nil {
   209  					return ocispec.Descriptor{}, err
   210  				}
   211  
   212  				mfstdesc.Annotations = map[string]string{
   213  					images.AnnotationImageName: normalized,
   214  					ocispec.AnnotationRefName:  ociReferenceName(normalized),
   215  				}
   216  
   217  				idx.Manifests = append(idx.Manifests, mfstdesc)
   218  			}
   219  		}
   220  	}
   221  
   222  	return writeManifest(ctx, store, idx, ocispec.MediaTypeImageIndex)
   223  }
   224  
   225  func onUntarJSON(r io.Reader, j interface{}) error {
   226  	b, err := ioutil.ReadAll(r)
   227  	if err != nil {
   228  		return err
   229  	}
   230  	return json.Unmarshal(b, j)
   231  }
   232  
   233  func onUntarBlob(ctx context.Context, r io.Reader, store content.Ingester, size int64, ref string) (digest.Digest, error) {
   234  	dgstr := digest.Canonical.Digester()
   235  
   236  	if err := content.WriteBlob(ctx, store, ref, io.TeeReader(r, dgstr.Hash()), ocispec.Descriptor{Size: size}); err != nil {
   237  		return "", err
   238  	}
   239  
   240  	return dgstr.Digest(), nil
   241  }
   242  
   243  func resolveLayers(ctx context.Context, store content.Store, layerFiles []string, blobs map[string]ocispec.Descriptor, compress bool) ([]ocispec.Descriptor, error) {
   244  	layers := make([]ocispec.Descriptor, len(layerFiles))
   245  	descs := map[digest.Digest]*ocispec.Descriptor{}
   246  	filters := []string{}
   247  	for i, f := range layerFiles {
   248  		desc, ok := blobs[f]
   249  		if !ok {
   250  			return nil, errors.Errorf("layer %q not found", f)
   251  		}
   252  		layers[i] = desc
   253  		descs[desc.Digest] = &layers[i]
   254  		filters = append(filters, "labels.\"containerd.io/uncompressed\"=="+desc.Digest.String())
   255  	}
   256  
   257  	err := store.Walk(ctx, func(info content.Info) error {
   258  		dgst, ok := info.Labels["containerd.io/uncompressed"]
   259  		if ok {
   260  			desc := descs[digest.Digest(dgst)]
   261  			if desc != nil {
   262  				desc.MediaType = images.MediaTypeDockerSchema2LayerGzip
   263  				desc.Digest = info.Digest
   264  				desc.Size = info.Size
   265  			}
   266  		}
   267  		return nil
   268  	}, filters...)
   269  	if err != nil {
   270  		return nil, errors.Wrap(err, "failure checking for compressed blobs")
   271  	}
   272  
   273  	for i, desc := range layers {
   274  		if desc.MediaType != "" {
   275  			continue
   276  		}
   277  		// Open blob, resolve media type
   278  		ra, err := store.ReaderAt(ctx, desc)
   279  		if err != nil {
   280  			return nil, errors.Wrapf(err, "failed to open %q (%s)", layerFiles[i], desc.Digest)
   281  		}
   282  		s, err := compression.DecompressStream(content.NewReader(ra))
   283  		if err != nil {
   284  			return nil, errors.Wrapf(err, "failed to detect compression for %q", layerFiles[i])
   285  		}
   286  		if s.GetCompression() == compression.Uncompressed {
   287  			if compress {
   288  				ref := fmt.Sprintf("compress-blob-%s-%s", desc.Digest.Algorithm().String(), desc.Digest.Encoded())
   289  				labels := map[string]string{
   290  					"containerd.io/uncompressed": desc.Digest.String(),
   291  				}
   292  				layers[i], err = compressBlob(ctx, store, s, ref, content.WithLabels(labels))
   293  				if err != nil {
   294  					s.Close()
   295  					return nil, err
   296  				}
   297  				layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
   298  			} else {
   299  				layers[i].MediaType = images.MediaTypeDockerSchema2Layer
   300  			}
   301  		} else {
   302  			layers[i].MediaType = images.MediaTypeDockerSchema2LayerGzip
   303  		}
   304  		s.Close()
   305  
   306  	}
   307  	return layers, nil
   308  }
   309  
   310  func compressBlob(ctx context.Context, cs content.Store, r io.Reader, ref string, opts ...content.Opt) (desc ocispec.Descriptor, err error) {
   311  	w, err := content.OpenWriter(ctx, cs, content.WithRef(ref))
   312  	if err != nil {
   313  		return ocispec.Descriptor{}, errors.Wrap(err, "failed to open writer")
   314  	}
   315  
   316  	defer func() {
   317  		w.Close()
   318  		if err != nil {
   319  			cs.Abort(ctx, ref)
   320  		}
   321  	}()
   322  	if err := w.Truncate(0); err != nil {
   323  		return ocispec.Descriptor{}, errors.Wrap(err, "failed to truncate writer")
   324  	}
   325  
   326  	cw, err := compression.CompressStream(w, compression.Gzip)
   327  	if err != nil {
   328  		return ocispec.Descriptor{}, err
   329  	}
   330  
   331  	if _, err := io.Copy(cw, r); err != nil {
   332  		return ocispec.Descriptor{}, err
   333  	}
   334  	if err := cw.Close(); err != nil {
   335  		return ocispec.Descriptor{}, err
   336  	}
   337  
   338  	cst, err := w.Status()
   339  	if err != nil {
   340  		return ocispec.Descriptor{}, errors.Wrap(err, "failed to get writer status")
   341  	}
   342  
   343  	desc.Digest = w.Digest()
   344  	desc.Size = cst.Offset
   345  
   346  	if err := w.Commit(ctx, desc.Size, desc.Digest, opts...); err != nil {
   347  		if !errdefs.IsAlreadyExists(err) {
   348  			return ocispec.Descriptor{}, errors.Wrap(err, "failed to commit")
   349  		}
   350  	}
   351  
   352  	return desc, nil
   353  }
   354  
   355  func writeManifest(ctx context.Context, cs content.Ingester, manifest interface{}, mediaType string) (ocispec.Descriptor, error) {
   356  	manifestBytes, err := json.Marshal(manifest)
   357  	if err != nil {
   358  		return ocispec.Descriptor{}, err
   359  	}
   360  
   361  	desc := ocispec.Descriptor{
   362  		MediaType: mediaType,
   363  		Digest:    digest.FromBytes(manifestBytes),
   364  		Size:      int64(len(manifestBytes)),
   365  	}
   366  	if err := content.WriteBlob(ctx, cs, "manifest-"+desc.Digest.String(), bytes.NewReader(manifestBytes), desc); err != nil {
   367  		return ocispec.Descriptor{}, err
   368  	}
   369  
   370  	return desc, nil
   371  }