github.com/containerd/nerdctl/v2@v2.0.0-beta.5.0.20240520001846-b5758f54fa28/pkg/cmd/image/convert.go (about) 1 /* 2 Copyright The containerd Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package image 18 19 import ( 20 "context" 21 "encoding/json" 22 "errors" 23 "fmt" 24 "io" 25 "os" 26 "strings" 27 28 overlaybdconvert "github.com/containerd/accelerated-container-image/pkg/convertor" 29 "github.com/containerd/containerd" 30 "github.com/containerd/containerd/content" 31 "github.com/containerd/containerd/images" 32 "github.com/containerd/containerd/images/converter" 33 "github.com/containerd/containerd/images/converter/uncompress" 34 "github.com/containerd/log" 35 "github.com/containerd/nerdctl/v2/pkg/api/types" 36 "github.com/containerd/nerdctl/v2/pkg/clientutil" 37 converterutil "github.com/containerd/nerdctl/v2/pkg/imgutil/converter" 38 "github.com/containerd/nerdctl/v2/pkg/platformutil" 39 "github.com/containerd/nerdctl/v2/pkg/referenceutil" 40 nydusconvert "github.com/containerd/nydus-snapshotter/pkg/converter" 41 "github.com/containerd/stargz-snapshotter/estargz" 42 estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz" 43 estargzexternaltocconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz/externaltoc" 44 zstdchunkedconvert "github.com/containerd/stargz-snapshotter/nativeconverter/zstdchunked" 45 "github.com/containerd/stargz-snapshotter/recorder" 46 "github.com/klauspost/compress/zstd" 47 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 48 ) 49 50 func Convert(ctx context.Context, client *containerd.Client, srcRawRef, targetRawRef string, options types.ImageConvertOptions) error { 51 var ( 52 convertOpts = []converter.Opt{} 53 ) 54 if srcRawRef == "" || targetRawRef == "" { 55 return errors.New("src and target image need to be specified") 56 } 57 58 srcNamed, err := referenceutil.ParseAny(srcRawRef) 59 if err != nil { 60 return err 61 } 62 srcRef := srcNamed.String() 63 64 targetNamed, err := referenceutil.ParseDockerRef(targetRawRef) 65 if err != nil { 66 return err 67 } 68 targetRef := targetNamed.String() 69 70 platMC, err := platformutil.NewMatchComparer(options.AllPlatforms, options.Platforms) 71 if err != nil { 72 return err 73 } 74 convertOpts = append(convertOpts, converter.WithPlatform(platMC)) 75 76 estargz := options.Estargz 77 zstd := options.Zstd 78 zstdchunked := options.ZstdChunked 79 overlaybd := options.Overlaybd 80 nydus := options.Nydus 81 var finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error) 82 if estargz || zstd || zstdchunked || overlaybd || nydus { 83 convertCount := 0 84 if estargz { 85 convertCount++ 86 } 87 if zstd { 88 convertCount++ 89 } 90 if zstdchunked { 91 convertCount++ 92 } 93 if overlaybd { 94 convertCount++ 95 } 96 if nydus { 97 convertCount++ 98 } 99 100 if convertCount > 1 { 101 return errors.New("options --estargz, --zstdchunked, --overlaybd and --nydus lead to conflict, only one of them can be used") 102 } 103 104 var convertFunc converter.ConvertFunc 105 var convertType string 106 switch { 107 case estargz: 108 convertFunc, finalize, err = getESGZConverter(options) 109 if err != nil { 110 return err 111 } 112 convertType = "estargz" 113 case zstd: 114 convertFunc, err = getZstdConverter(options) 115 if err != nil { 116 return err 117 } 118 convertType = "zstd" 119 case zstdchunked: 120 convertFunc, err = getZstdchunkedConverter(options) 121 if err != nil { 122 return err 123 } 124 convertType = "zstdchunked" 125 case overlaybd: 126 obdOpts, err := getOBDConvertOpts(options) 127 if err != nil { 128 return err 129 } 130 obdOpts = append(obdOpts, overlaybdconvert.WithClient(client)) 131 obdOpts = append(obdOpts, overlaybdconvert.WithImageRef(srcRef)) 132 convertFunc = overlaybdconvert.IndexConvertFunc(obdOpts...) 133 convertOpts = append(convertOpts, converter.WithIndexConvertFunc(convertFunc)) 134 convertType = "overlaybd" 135 case nydus: 136 nydusOpts, err := getNydusConvertOpts(options) 137 if err != nil { 138 return err 139 } 140 convertHooks := converter.ConvertHooks{ 141 PostConvertHook: nydusconvert.ConvertHookFunc(nydusconvert.MergeOption{ 142 WorkDir: nydusOpts.WorkDir, 143 BuilderPath: nydusOpts.BuilderPath, 144 FsVersion: nydusOpts.FsVersion, 145 ChunkDictPath: nydusOpts.ChunkDictPath, 146 PrefetchPatterns: nydusOpts.PrefetchPatterns, 147 OCI: true, 148 }), 149 } 150 convertOpts = append(convertOpts, converter.WithIndexConvertFunc( 151 converter.IndexConvertFuncWithHook( 152 nydusconvert.LayerConvertFunc(*nydusOpts), 153 true, 154 platMC, 155 convertHooks, 156 )), 157 ) 158 convertType = "nydus" 159 } 160 161 if convertType != "overlaybd" { 162 convertOpts = append(convertOpts, converter.WithLayerConvertFunc(convertFunc)) 163 } 164 if !options.Oci { 165 if nydus || overlaybd { 166 log.G(ctx).Warnf("option --%s should be used in conjunction with --oci, forcibly enabling on oci mediatype for %s conversion", convertType, convertType) 167 } else { 168 log.G(ctx).Warnf("option --%s should be used in conjunction with --oci", convertType) 169 } 170 } 171 if options.Uncompress { 172 return fmt.Errorf("option --%s conflicts with --uncompress", convertType) 173 } 174 } 175 176 if options.Uncompress { 177 convertOpts = append(convertOpts, converter.WithLayerConvertFunc(uncompress.LayerConvertFunc)) 178 } 179 180 if options.Oci { 181 convertOpts = append(convertOpts, converter.WithDockerToOCI(true)) 182 } 183 184 // converter.Convert() gains the lease by itself 185 newImg, err := converter.Convert(ctx, client, targetRef, srcRef, convertOpts...) 186 if err != nil { 187 return err 188 } 189 res := converterutil.ConvertedImageInfo{ 190 Image: newImg.Name + "@" + newImg.Target.Digest.String(), 191 } 192 if finalize != nil { 193 ctx, done, err := client.WithLease(ctx) 194 if err != nil { 195 return err 196 } 197 defer done(ctx) 198 newI, err := finalize(ctx, client.ContentStore(), targetRef, &newImg.Target) 199 if err != nil { 200 return err 201 } 202 is := client.ImageService() 203 _ = is.Delete(ctx, newI.Name) 204 finimg, err := is.Create(ctx, *newI) 205 if err != nil { 206 return err 207 } 208 res.ExtraImages = append(res.ExtraImages, finimg.Name+"@"+finimg.Target.Digest.String()) 209 } 210 return printConvertedImage(options.Stdout, options, res) 211 } 212 213 func getESGZConverter(options types.ImageConvertOptions) (convertFunc converter.ConvertFunc, finalize func(ctx context.Context, cs content.Store, ref string, desc *ocispec.Descriptor) (*images.Image, error), _ error) { 214 if options.EstargzExternalToc && !options.GOptions.Experimental { 215 return nil, nil, fmt.Errorf("estargz-external-toc requires experimental mode to be enabled") 216 } 217 if options.EstargzKeepDiffID && !options.GOptions.Experimental { 218 return nil, nil, fmt.Errorf("option --estargz-keep-diff-id must be specified with --estargz-external-toc") 219 } 220 if options.EstargzExternalToc { 221 if !options.EstargzKeepDiffID { 222 esgzOpts, err := getESGZConvertOpts(options) 223 if err != nil { 224 return nil, nil, err 225 } 226 convertFunc, finalize = estargzexternaltocconvert.LayerConvertFunc(esgzOpts, options.EstargzCompressionLevel) 227 } else { 228 convertFunc, finalize = estargzexternaltocconvert.LayerConvertLossLessFunc(estargzexternaltocconvert.LayerConvertLossLessConfig{ 229 CompressionLevel: options.EstargzCompressionLevel, 230 ChunkSize: options.EstargzChunkSize, 231 MinChunkSize: options.EstargzMinChunkSize, 232 }) 233 } 234 } else { 235 esgzOpts, err := getESGZConvertOpts(options) 236 if err != nil { 237 return nil, nil, err 238 } 239 convertFunc = estargzconvert.LayerConvertFunc(esgzOpts...) 240 } 241 return convertFunc, finalize, nil 242 } 243 244 func getESGZConvertOpts(options types.ImageConvertOptions) ([]estargz.Option, error) { 245 246 esgzOpts := []estargz.Option{ 247 estargz.WithCompressionLevel(options.EstargzCompressionLevel), 248 estargz.WithChunkSize(options.EstargzChunkSize), 249 estargz.WithMinChunkSize(options.EstargzMinChunkSize), 250 } 251 252 if options.EstargzRecordIn != "" { 253 if !options.GOptions.Experimental { 254 return nil, fmt.Errorf("estargz-record-in requires experimental mode to be enabled") 255 } 256 257 log.L.Warn("--estargz-record-in flag is experimental and subject to change") 258 paths, err := readPathsFromRecordFile(options.EstargzRecordIn) 259 if err != nil { 260 return nil, err 261 } 262 esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths)) 263 var ignored []string 264 esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored)) 265 } 266 return esgzOpts, nil 267 } 268 269 func getZstdConverter(options types.ImageConvertOptions) (converter.ConvertFunc, error) { 270 return converterutil.ZstdLayerConvertFunc(options) 271 } 272 273 func getZstdchunkedConverter(options types.ImageConvertOptions) (converter.ConvertFunc, error) { 274 275 esgzOpts := []estargz.Option{ 276 estargz.WithChunkSize(options.ZstdChunkedChunkSize), 277 } 278 279 if options.ZstdChunkedRecordIn != "" { 280 if !options.GOptions.Experimental { 281 return nil, fmt.Errorf("zstdchunked-record-in requires experimental mode to be enabled") 282 } 283 284 log.L.Warn("--zstdchunked-record-in flag is experimental and subject to change") 285 paths, err := readPathsFromRecordFile(options.ZstdChunkedRecordIn) 286 if err != nil { 287 return nil, err 288 } 289 esgzOpts = append(esgzOpts, estargz.WithPrioritizedFiles(paths)) 290 var ignored []string 291 esgzOpts = append(esgzOpts, estargz.WithAllowPrioritizeNotFound(&ignored)) 292 } 293 return zstdchunkedconvert.LayerConvertFuncWithCompressionLevel(zstd.EncoderLevelFromZstd(options.ZstdChunkedCompressionLevel), esgzOpts...), nil 294 } 295 296 func getNydusConvertOpts(options types.ImageConvertOptions) (*nydusconvert.PackOption, error) { 297 workDir := options.NydusWorkDir 298 if workDir == "" { 299 var err error 300 workDir, err = clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address) 301 if err != nil { 302 return nil, err 303 } 304 } 305 return &nydusconvert.PackOption{ 306 BuilderPath: options.NydusBuilderPath, 307 // the path will finally be used is <NERDCTL_DATA_ROOT>/nydus-converter-<hash>, 308 // for example: /var/lib/nerdctl/1935db59/nydus-converter-3269662176/, 309 // and it will be deleted after the conversion 310 WorkDir: workDir, 311 PrefetchPatterns: options.NydusPrefetchPatterns, 312 Compressor: options.NydusCompressor, 313 FsVersion: "6", 314 }, nil 315 } 316 317 func getOBDConvertOpts(options types.ImageConvertOptions) ([]overlaybdconvert.Option, error) { 318 obdOpts := []overlaybdconvert.Option{ 319 overlaybdconvert.WithFsType(options.OverlayFsType), 320 overlaybdconvert.WithDbstr(options.OverlaydbDBStr), 321 } 322 return obdOpts, nil 323 } 324 325 func readPathsFromRecordFile(filename string) ([]string, error) { 326 r, err := os.Open(filename) 327 if err != nil { 328 return nil, err 329 } 330 defer r.Close() 331 dec := json.NewDecoder(r) 332 var paths []string 333 added := make(map[string]struct{}) 334 for dec.More() { 335 var e recorder.Entry 336 if err := dec.Decode(&e); err != nil { 337 return nil, err 338 } 339 if _, ok := added[e.Path]; !ok { 340 paths = append(paths, e.Path) 341 added[e.Path] = struct{}{} 342 } 343 } 344 return paths, nil 345 } 346 347 func printConvertedImage(stdout io.Writer, options types.ImageConvertOptions, img converterutil.ConvertedImageInfo) error { 348 switch options.Format { 349 case "json": 350 b, err := json.MarshalIndent(img, "", " ") 351 if err != nil { 352 return err 353 } 354 fmt.Fprintln(stdout, string(b)) 355 default: 356 for i, e := range img.ExtraImages { 357 elems := strings.SplitN(e, "@", 2) 358 if len(elems) < 2 { 359 log.L.Errorf("extra reference %q doesn't contain digest", e) 360 } else { 361 log.L.Infof("Extra image(%d) %s", i, elems[0]) 362 } 363 } 364 elems := strings.SplitN(img.Image, "@", 2) 365 if len(elems) < 2 { 366 log.L.Errorf("reference %q doesn't contain digest", img.Image) 367 } else { 368 fmt.Fprintln(stdout, elems[1]) 369 } 370 } 371 return nil 372 }