github.com/containerd/nerdctl/v2@v2.0.0-beta.5.0.20240520001846-b5758f54fa28/pkg/cmd/image/convert.go (about)

     1  /*
     2     Copyright The containerd Authors.
     3  
     4     Licensed under the Apache License, Version 2.0 (the "License");
     5     you may not use this file except in compliance with the License.
     6     You may obtain a copy of the License at
     7  
     8         http://www.apache.org/licenses/LICENSE-2.0
     9  
    10     Unless required by applicable law or agreed to in writing, software
    11     distributed under the License is distributed on an "AS IS" BASIS,
    12     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13     See the License for the specific language governing permissions and
    14     limitations under the License.
    15  */
    16  
    17  package image
    18  
    19  import (
    20  	"context"
    21  	"encoding/json"
    22  	"errors"
    23  	"fmt"
    24  	"io"
    25  	"os"
    26  	"strings"
    27  
    28  	overlaybdconvert "github.com/containerd/accelerated-container-image/pkg/convertor"
    29  	"github.com/containerd/containerd"
    30  	"github.com/containerd/containerd/content"
    31  	"github.com/containerd/containerd/images"
    32  	"github.com/containerd/containerd/images/converter"
    33  	"github.com/containerd/containerd/images/converter/uncompress"
    34  	"github.com/containerd/log"
    35  	"github.com/containerd/nerdctl/v2/pkg/api/types"
    36  	"github.com/containerd/nerdctl/v2/pkg/clientutil"
    37  	converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter"
    38  	"github.com/containerd/nerdctl/v2/pkg/platformutil"
    39  	"github.com/containerd/nerdctl/v2/pkg/referenceutil"
    40  	nydusconvert "github.com/containerd/nydus-snapshotter/pkg/converter"
    41  	"github.com/containerd/stargz-snapshotter/estargz"
    42  	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
    43  	estargzexternaltocconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz/externaltoc"
    44  	zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked"
    45  	"github.com/containerd/stargz-snapshotter/recorder"
    46  	"github.com/klauspost/compress/zstd"
    47  	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
    48  )
    49  
    50  func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error {
    51  	var (
    52  		convertOpts = []converter.Opt{}
    53  	)
    54  	if srcRawRef == "" || targetRawRef == "" {
    55  		return errors.New("src and target image need to be specified")
    56  	}
    57  
    58  	srcNamed, err := referenceutil.ParseAny(srcRawRef)
    59  	if err != nil {
    60  		return err
    61  	}
    62  	srcRef := srcNamed.String()
    63  
    64  	targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
    65  	if err != nil {
    66  		return err
    67  	}
    68  	targetRef := targetNamed.String()
    69  
    70  	platMC, err := platformutil.NewMatchComparer(options.AllPlatforms, options.Platforms)
    71  	if err != nil {
    72  		return err
    73  	}
    74  	convertOpts = append(convertOpts, converter.WithPlatform(platMC))
    75  
    76  	estargz := options.Estargz
    77  	zstd := options.Zstd
    78  	zstdchunked := options.ZstdChunked
    79  	overlaybd := options.Overlaybd
    80  	nydus := options.Nydus
    81  	var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error)
    82  	if estargz || zstd || zstdchunked || overlaybd || nydus {
    83  		convertCount := 0
    84  		if estargz {
    85  			convertCount++
    86  		}
    87  		if zstd {
    88  			convertCount++
    89  		}
    90  		if zstdchunked {
    91  			convertCount++
    92  		}
    93  		if overlaybd {
    94  			convertCount++
    95  		}
    96  		if nydus {
    97  			convertCount++
    98  		}
    99  
   100  		if convertCount > 1 {
   101  			return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used")
   102  		}
   103  
   104  		var convertFunc converter.ConvertFunc
   105  		var convertType string
   106  		switch {
   107  		case estargz:
   108  			convertFunc, finalize, err = getESGZConverter(options)
   109  			if err != nil {
   110  				return err
   111  			}
   112  			convertType = "estargz"
   113  		case zstd:
   114  			convertFunc, err = getZstdConverter(options)
   115  			if err != nil {
   116  				return err
   117  			}
   118  			convertType = "zstd"
   119  		case zstdchunked:
   120  			convertFunc, err = getZstdchunkedConverter(options)
   121  			if err != nil {
   122  				return err
   123  			}
   124  			convertType = "zstdchunked"
   125  		case overlaybd:
   126  			obdOpts, err := getOBDConvertOpts(options)
   127  			if err != nil {
   128  				return err
   129  			}
   130  			obdOpts = append(obdOpts, overlaybdconvert.WithClient(client))
   131  			obdOpts = append(obdOpts, overlaybdconvert.WithImageRef(srcRef))
   132  			convertFunc = overlaybdconvert.IndexConvertFunc(obdOpts...)
   133  			convertOpts = append(convertOpts, converter.WithIndexConvertFunc(convertFunc))
   134  			convertType = "overlaybd"
   135  		case nydus:
   136  			nydusOpts, err := getNydusConvertOpts(options)
   137  			if err != nil {
   138  				return err
   139  			}
   140  			convertHooks := converter.ConvertHooks{
   141  				PostConvertHook: nydusconvert.ConvertHookFunc(nydusconvert.MergeOption{
   142  					WorkDir:          nydusOpts.WorkDir,
   143  					BuilderPath:      nydusOpts.BuilderPath,
   144  					FsVersion:        nydusOpts.FsVersion,
   145  					ChunkDictPath:    nydusOpts.ChunkDictPath,
   146  					PrefetchPatterns: nydusOpts.PrefetchPatterns,
   147  					OCI:              true,
   148  				}),
   149  			}
   150  			convertOpts = append(convertOpts, converter.WithIndexConvertFunc(
   151  				converter.IndexConvertFuncWithHook(
   152  					nydusconvert.LayerConvertFunc(*nydusOpts),
   153  					true,
   154  					platMC,
   155  					convertHooks,
   156  				)),
   157  			)
   158  			convertType = "nydus"
   159  		}
   160  
   161  		if convertType != "overlaybd" {
   162  			convertOpts = append(convertOpts, converter.WithLayerConvertFunc(convertFunc))
   163  		}
   164  		if !options.Oci {
   165  			if nydus || overlaybd {
   166  				log.G(ctx).Warnf("option --%s should be used in conjunction with --oci, forcibly enabling on oci mediatype for %s conversion", convertType, convertType)
   167  			} else {
   168  				log.G(ctx).Warnf("option --%s should be used in conjunction with --oci", convertType)
   169  			}
   170  		}
   171  		if options.Uncompress {
   172  			return fmt.Errorf("option --%s conflicts with --uncompress", convertType)
   173  		}
   174  	}
   175  
   176  	if options.Uncompress {
   177  		convertOpts = append(convertOpts, converter.WithLayerConvertFunc(uncompress.LayerConvertFunc))
   178  	}
   179  
   180  	if options.Oci {
   181  		convertOpts = append(convertOpts, converter.WithDockerToOCI(true))
   182  	}
   183  
   184  	// converter.Convert() gains the lease by itself
   185  	newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...)
   186  	if err != nil {
   187  		return err
   188  	}
   189  	res := converterutil.ConvertedImageInfo{
   190  		Image: newImg.Name + "@" + newImg.Target.Digest.String(),
   191  	}
   192  	if finalize != nil {
   193  		ctx, done, err := client.WithLease(ctx)
   194  		if err != nil {
   195  			return err
   196  		}
   197  		defer done(ctx)
   198  		newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target)
   199  		if err != nil {
   200  			return err
   201  		}
   202  		is := client.ImageService()
   203  		_ = is.Delete(ctx, newI.Name)
   204  		finimg, err := is.Create(ctx, *newI)
   205  		if err != nil {
   206  			return err
   207  		}
   208  		res.ExtraImages = append(res.ExtraImages, finimg.Name+"@"+finimg.Target.Digest.String())
   209  	}
   210  	return printConvertedImage(options.Stdout, options, res)
   211  }
   212  
   213  func getESGZConverter(options types.ImageConvertOptions) (convertFunc converter.ConvertFunc, finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error), _ error) {
   214  	if options.EstargzExternalToc && !options.GOptions.Experimental {
   215  		return nil, nil, fmt.Errorf("estargz-external-toc requires experimental mode to be enabled")
   216  	}
   217  	if options.EstargzKeepDiffID && !options.GOptions.Experimental {
   218  		return nil, nil, fmt.Errorf("option --estargz-keep-diff-id must be specified with --estargz-external-toc")
   219  	}
   220  	if options.EstargzExternalToc {
   221  		if !options.EstargzKeepDiffID {
   222  			esgzOpts, err := getESGZConvertOpts(options)
   223  			if err != nil {
   224  				return nil, nil, err
   225  			}
   226  			convertFunc, finalize = estargzexternaltocconvert.LayerConvertFunc(esgzOpts, options.EstargzCompressionLevel)
   227  		} else {
   228  			convertFunc, finalize = estargzexternaltocconvert.LayerConvertLossLessFunc(estargzexternaltocconvert.LayerConvertLossLessConfig{
   229  				CompressionLevel: options.EstargzCompressionLevel,
   230  				ChunkSize:        options.EstargzChunkSize,
   231  				MinChunkSize:     options.EstargzMinChunkSize,
   232  			})
   233  		}
   234  	} else {
   235  		esgzOpts, err := getESGZConvertOpts(options)
   236  		if err != nil {
   237  			return nil, nil, err
   238  		}
   239  		convertFunc = estargzconvert.LayerConvertFunc(esgzOpts...)
   240  	}
   241  	return convertFunc, finalize, nil
   242  }
   243  
   244  func getESGZConvertOpts(options types.ImageConvertOptions) ([]estargz.Option, error) {
   245  
   246  	esgzOpts := []estargz.Option{
   247  		estargz.WithCompressionLevel(options.EstargzCompressionLevel),
   248  		estargz.WithChunkSize(options.EstargzChunkSize),
   249  		estargz.WithMinChunkSize(options.EstargzMinChunkSize),
   250  	}
   251  
   252  	if options.EstargzRecordIn != "" {
   253  		if !options.GOptions.Experimental {
   254  			return nil, fmt.Errorf("estargz-record-in requires experimental mode to be enabled")
   255  		}
   256  
   257  		log.L.Warn("--estargz-record-in flag is experimental and subject to change")
   258  		paths, err := readPathsFromRecordFile(options.EstargzRecordIn)
   259  		if err != nil {
   260  			return nil, err
   261  		}
   262  		esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths))
   263  		var ignored []string
   264  		esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored))
   265  	}
   266  	return esgzOpts, nil
   267  }
   268  
   269  func getZstdConverter(options types.ImageConvertOptions) (converter.ConvertFunc, error) {
   270  	return converterutil.ZstdLayerConvertFunc(options)
   271  }
   272  
   273  func getZstdchunkedConverter(options types.ImageConvertOptions) (converter.ConvertFunc, error) {
   274  
   275  	esgzOpts := []estargz.Option{
   276  		estargz.WithChunkSize(options.ZstdChunkedChunkSize),
   277  	}
   278  
   279  	if options.ZstdChunkedRecordIn != "" {
   280  		if !options.GOptions.Experimental {
   281  			return nil, fmt.Errorf("zstdchunked-record-in requires experimental mode to be enabled")
   282  		}
   283  
   284  		log.L.Warn("--zstdchunked-record-in flag is experimental and subject to change")
   285  		paths, err := readPathsFromRecordFile(options.ZstdChunkedRecordIn)
   286  		if err != nil {
   287  			return nil, err
   288  		}
   289  		esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths))
   290  		var ignored []string
   291  		esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored))
   292  	}
   293  	return zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(zstd.EncoderLevelFromZstd(options.ZstdChunkedCompressionLevel), esgzOpts...), nil
   294  }
   295  
   296  func getNydusConvertOpts(options types.ImageConvertOptions) (*nydusconvert.PackOption, error) {
   297  	workDir := options.NydusWorkDir
   298  	if workDir == "" {
   299  		var err error
   300  		workDir, err = clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address)
   301  		if err != nil {
   302  			return nil, err
   303  		}
   304  	}
   305  	return &nydusconvert.PackOption{
   306  		BuilderPath: options.NydusBuilderPath,
   307  		// the path will finally be used is <NERDCTL_DATA_ROOT>/nydus-converter-<hash>,
   308  		// for example: /var/lib/nerdctl/1935db59/nydus-converter-3269662176/,
   309  		// and it will be deleted after the conversion
   310  		WorkDir:          workDir,
   311  		PrefetchPatterns: options.NydusPrefetchPatterns,
   312  		Compressor:       options.NydusCompressor,
   313  		FsVersion:        "6",
   314  	}, nil
   315  }
   316  
   317  func getOBDConvertOpts(options types.ImageConvertOptions) ([]overlaybdconvert.Option, error) {
   318  	obdOpts := []overlaybdconvert.Option{
   319  		overlaybdconvert.WithFsType(options.OverlayFsType),
   320  		overlaybdconvert.WithDbstr(options.OverlaydbDBStr),
   321  	}
   322  	return obdOpts, nil
   323  }
   324  
   325  func readPathsFromRecordFile(filename string) ([]string, error) {
   326  	r, err := os.Open(filename)
   327  	if err != nil {
   328  		return nil, err
   329  	}
   330  	defer r.Close()
   331  	dec := json.NewDecoder(r)
   332  	var paths []string
   333  	added := make(map[string]struct{})
   334  	for dec.More() {
   335  		var e recorder.Entry
   336  		if err := dec.Decode(&e); err != nil {
   337  			return nil, err
   338  		}
   339  		if _, ok := added[e.Path]; !ok {
   340  			paths = append(paths, e.Path)
   341  			added[e.Path] = struct{}{}
   342  		}
   343  	}
   344  	return paths, nil
   345  }
   346  
   347  func printConvertedImage(stdout io.Writer, options types.ImageConvertOptions, img converterutil.ConvertedImageInfo) error {
   348  	switch options.Format {
   349  	case "json":
   350  		b, err := json.MarshalIndent(img, "", "    ")
   351  		if err != nil {
   352  			return err
   353  		}
   354  		fmt.Fprintln(stdout, string(b))
   355  	default:
   356  		for i, e := range img.ExtraImages {
   357  			elems := strings.SplitN(e, "@", 2)
   358  			if len(elems) < 2 {
   359  				log.L.Errorf("extra reference %q doesn't contain digest", e)
   360  			} else {
   361  				log.L.Infof("Extra image(%d) %s", i, elems[0])
   362  			}
   363  		}
   364  		elems := strings.SplitN(img.Image, "@", 2)
   365  		if len(elems) < 2 {
   366  			log.L.Errorf("reference %q doesn't contain digest", img.Image)
   367  		} else {
   368  			fmt.Fprintln(stdout, elems[1])
   369  		}
   370  	}
   371  	return nil
   372  }