github.com/rawahars/moby@v24.0.4+incompatible/daemon/containerd/image_import.go (about) 1 package containerd 2 3 import ( 4 "bufio" 5 "bytes" 6 "context" 7 "encoding/json" 8 "io" 9 "time" 10 11 "github.com/containerd/containerd" 12 "github.com/containerd/containerd/content" 13 cerrdefs "github.com/containerd/containerd/errdefs" 14 "github.com/containerd/containerd/images" 15 "github.com/containerd/containerd/platforms" 16 "github.com/docker/distribution/reference" 17 "github.com/docker/docker/api/types/container" 18 "github.com/docker/docker/builder/dockerfile" 19 "github.com/docker/docker/errdefs" 20 "github.com/docker/docker/image" 21 "github.com/docker/docker/pkg/archive" 22 "github.com/docker/docker/pkg/pools" 23 "github.com/google/uuid" 24 "github.com/opencontainers/go-digest" 25 "github.com/opencontainers/image-spec/specs-go" 26 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 27 "github.com/pkg/errors" 28 "github.com/sirupsen/logrus" 29 ) 30 31 // ImportImage imports an image, getting the archived layer data from layerReader. 32 // Layer archive is imported as-is if the compression is gzip or zstd. 33 // Uncompressed, xz and bzip2 archives are recompressed into gzip. 34 // The image is tagged with the given reference. 35 // If the platform is nil, the default host platform is used. 36 // The message is used as the history comment. 37 // Image configuration is derived from the dockerfile instructions in changes. 38 func (i *ImageService) ImportImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) { 39 refString := "" 40 if ref != nil { 41 refString = ref.String() 42 } 43 logger := logrus.WithField("ref", refString) 44 45 ctx, release, err := i.client.WithLease(ctx) 46 if err != nil { 47 return "", errdefs.System(err) 48 } 49 defer release(ctx) 50 51 if platform == nil { 52 def := platforms.DefaultSpec() 53 platform = &def 54 } 55 56 imageConfig, err := dockerfile.BuildFromConfig(ctx, &container.Config{}, changes, platform.OS) 57 if err != nil { 58 logger.WithError(err).Debug("failed to process changes") 59 return "", errdefs.InvalidParameter(err) 60 } 61 62 cs := i.client.ContentStore() 63 64 compressedDigest, uncompressedDigest, mt, err := saveArchive(ctx, cs, layerReader) 65 if err != nil { 66 logger.WithError(err).Debug("failed to write layer blob") 67 return "", err 68 } 69 logger = logger.WithFields(logrus.Fields{ 70 "compressedDigest": compressedDigest, 71 "uncompressedDigest": uncompressedDigest, 72 }) 73 74 size, err := fillUncompressedLabel(ctx, cs, compressedDigest, uncompressedDigest) 75 if err != nil { 76 logger.WithError(err).Debug("failed to set uncompressed label on the compressed blob") 77 return "", err 78 } 79 80 compressedRootfsDesc := ocispec.Descriptor{ 81 MediaType: mt, 82 Digest: compressedDigest, 83 Size: size, 84 } 85 86 ociCfg := containerConfigToOciImageConfig(imageConfig) 87 createdAt := time.Now() 88 config := ocispec.Image{ 89 Platform: *platform, 90 Created: &createdAt, 91 Author: "", 92 Config: ociCfg, 93 RootFS: ocispec.RootFS{ 94 Type: "layers", 95 DiffIDs: []digest.Digest{uncompressedDigest}, 96 }, 97 History: []ocispec.History{ 98 { 99 Created: &createdAt, 100 CreatedBy: "", 101 Author: "", 102 Comment: msg, 103 EmptyLayer: false, 104 }, 105 }, 106 } 107 configDesc, err := storeJson(ctx, cs, ocispec.MediaTypeImageConfig, config, nil) 108 if err != nil { 109 return "", err 110 } 111 112 manifest := ocispec.Manifest{ 113 MediaType: ocispec.MediaTypeImageManifest, 114 Versioned: specs.Versioned{ 115 SchemaVersion: 2, 116 }, 117 Config: configDesc, 118 Layers: []ocispec.Descriptor{ 119 compressedRootfsDesc, 120 }, 121 } 122 manifestDesc, err := storeJson(ctx, cs, ocispec.MediaTypeImageManifest, manifest, map[string]string{ 123 "containerd.io/gc.ref.content.config": configDesc.Digest.String(), 124 "containerd.io/gc.ref.content.l.0": compressedDigest.String(), 125 }) 126 if err != nil { 127 return "", err 128 } 129 130 id := image.ID(manifestDesc.Digest.String()) 131 img := images.Image{ 132 Name: refString, 133 Target: manifestDesc, 134 CreatedAt: createdAt, 135 } 136 if img.Name == "" { 137 img.Name = danglingImageName(manifestDesc.Digest) 138 } 139 140 err = i.saveImage(ctx, img) 141 if err != nil { 142 logger.WithError(err).Debug("failed to save image") 143 return "", err 144 } 145 err = i.unpackImage(ctx, img, *platform) 146 if err != nil { 147 logger.WithError(err).Debug("failed to unpack image") 148 } else { 149 i.LogImageEvent(id.String(), id.String(), "import") 150 } 151 152 return id, err 153 } 154 155 // saveArchive saves the archive from bufRd to the content store, compressing it if necessary. 156 // Returns compressed blob digest, digest of the uncompressed data and media type of the stored blob. 157 func saveArchive(ctx context.Context, cs content.Store, layerReader io.Reader) (digest.Digest, digest.Digest, string, error) { 158 // Wrap the reader in buffered reader to allow peeks. 159 p := pools.BufioReader32KPool 160 bufRd := p.Get(layerReader) 161 defer p.Put(bufRd) 162 163 compression, err := detectCompression(bufRd) 164 if err != nil { 165 return "", "", "", err 166 } 167 168 var uncompressedReader io.Reader = bufRd 169 switch compression { 170 case archive.Gzip, archive.Zstd: 171 // If the input is already a compressed layer, just save it as is. 172 mediaType := ocispec.MediaTypeImageLayerGzip 173 if compression == archive.Zstd { 174 mediaType = ocispec.MediaTypeImageLayerZstd 175 } 176 177 compressedDigest, uncompressedDigest, err := writeCompressedBlob(ctx, cs, mediaType, bufRd) 178 if err != nil { 179 return "", "", "", err 180 } 181 182 return compressedDigest, uncompressedDigest, mediaType, nil 183 case archive.Bzip2, archive.Xz: 184 r, err := archive.DecompressStream(bufRd) 185 if err != nil { 186 return "", "", "", errdefs.InvalidParameter(err) 187 } 188 defer r.Close() 189 uncompressedReader = r 190 fallthrough 191 case archive.Uncompressed: 192 mediaType := ocispec.MediaTypeImageLayerGzip 193 compression := archive.Gzip 194 195 compressedDigest, uncompressedDigest, err := compressAndWriteBlob(ctx, cs, compression, mediaType, uncompressedReader) 196 if err != nil { 197 return "", "", "", err 198 } 199 200 return compressedDigest, uncompressedDigest, mediaType, nil 201 } 202 203 return "", "", "", errdefs.InvalidParameter(errors.New("unsupported archive compression")) 204 } 205 206 // writeCompressedBlob writes the blob and simultaneously computes the digest of the uncompressed data. 207 func writeCompressedBlob(ctx context.Context, cs content.Store, mediaType string, bufRd *bufio.Reader) (digest.Digest, digest.Digest, error) { 208 pr, pw := io.Pipe() 209 defer pw.Close() 210 defer pr.Close() 211 212 c := make(chan digest.Digest) 213 // Start copying the blob to the content store from the pipe and tee it to the pipe. 214 go func() { 215 compressedDigest, err := writeBlobAndReturnDigest(ctx, cs, mediaType, io.TeeReader(bufRd, pw)) 216 pw.CloseWithError(err) 217 c <- compressedDigest 218 }() 219 220 digester := digest.Canonical.Digester() 221 222 // Decompress the piped blob. 223 decompressedStream, err := archive.DecompressStream(pr) 224 if err == nil { 225 // Feed the digester with decompressed data. 226 _, err = io.Copy(digester.Hash(), decompressedStream) 227 decompressedStream.Close() 228 } 229 pr.CloseWithError(err) 230 231 compressedDigest := <-c 232 if err != nil { 233 if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 234 return "", "", errdefs.Cancelled(err) 235 } 236 return "", "", errdefs.System(err) 237 } 238 239 uncompressedDigest := digester.Digest() 240 return compressedDigest, uncompressedDigest, nil 241 } 242 243 // compressAndWriteBlob compresses the uncompressedReader and stores it in the content store. 244 func compressAndWriteBlob(ctx context.Context, cs content.Store, compression archive.Compression, mediaType string, uncompressedLayerReader io.Reader) (digest.Digest, digest.Digest, error) { 245 pr, pw := io.Pipe() 246 defer pr.Close() 247 defer pw.Close() 248 249 compressor, err := archive.CompressStream(pw, compression) 250 if err != nil { 251 return "", "", errdefs.InvalidParameter(err) 252 } 253 defer compressor.Close() 254 255 writeChan := make(chan digest.Digest) 256 // Start copying the blob to the content store from the pipe. 257 go func() { 258 digest, err := writeBlobAndReturnDigest(ctx, cs, mediaType, pr) 259 pr.CloseWithError(err) 260 writeChan <- digest 261 }() 262 263 // Copy archive to the pipe and tee it to a digester. 264 // This will feed the pipe the above goroutine is reading from. 265 uncompressedDigester := digest.Canonical.Digester() 266 readFromInputAndDigest := io.TeeReader(uncompressedLayerReader, uncompressedDigester.Hash()) 267 _, err = io.Copy(compressor, readFromInputAndDigest) 268 compressor.Close() 269 pw.CloseWithError(err) 270 271 compressedDigest := <-writeChan 272 if err != nil { 273 if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 274 return "", "", errdefs.Cancelled(err) 275 } 276 return "", "", errdefs.System(err) 277 } 278 279 return compressedDigest, uncompressedDigester.Digest(), err 280 } 281 282 // writeBlobAndReturnDigest writes a blob to the content store and returns the digest. 283 func writeBlobAndReturnDigest(ctx context.Context, cs content.Store, mt string, reader io.Reader) (digest.Digest, error) { 284 digester := digest.Canonical.Digester() 285 if err := content.WriteBlob(ctx, cs, uuid.New().String(), io.TeeReader(reader, digester.Hash()), ocispec.Descriptor{MediaType: mt}); err != nil { 286 return "", errdefs.System(err) 287 } 288 return digester.Digest(), nil 289 } 290 291 // saveImage creates an image in the ImageService or updates it if it exists. 292 func (i *ImageService) saveImage(ctx context.Context, img images.Image) error { 293 is := i.client.ImageService() 294 295 if _, err := is.Update(ctx, img); err != nil { 296 if cerrdefs.IsNotFound(err) { 297 if _, err := is.Create(ctx, img); err != nil { 298 return errdefs.Unknown(err) 299 } 300 } else { 301 return errdefs.Unknown(err) 302 } 303 } 304 305 return nil 306 } 307 308 // unpackImage unpacks the image into the snapshotter. 309 func (i *ImageService) unpackImage(ctx context.Context, img images.Image, platform ocispec.Platform) error { 310 c8dImg := containerd.NewImageWithPlatform(i.client, img, platforms.Only(platform)) 311 unpacked, err := c8dImg.IsUnpacked(ctx, i.snapshotter) 312 if err != nil { 313 return err 314 } 315 if !unpacked { 316 err = c8dImg.Unpack(ctx, i.snapshotter) 317 } 318 319 return err 320 } 321 322 // detectCompression dectects the reader compression type. 323 func detectCompression(bufRd *bufio.Reader) (archive.Compression, error) { 324 bs, err := bufRd.Peek(10) 325 if err != nil && err != io.EOF { 326 // Note: we'll ignore any io.EOF error because there are some odd 327 // cases where the layer.tar file will be empty (zero bytes) and 328 // that results in an io.EOF from the Peek() call. So, in those 329 // cases we'll just treat it as a non-compressed stream and 330 // that means just create an empty layer. 331 // See Issue 18170 332 return archive.Uncompressed, errdefs.Unknown(err) 333 } 334 335 return archive.DetectCompression(bs), nil 336 } 337 338 // fillUncompressedLabel sets the uncompressed digest label on the compressed blob metadata 339 // and returns the compressed blob size. 340 func fillUncompressedLabel(ctx context.Context, cs content.Store, compressedDigest digest.Digest, uncompressedDigest digest.Digest) (int64, error) { 341 info, err := cs.Info(ctx, compressedDigest) 342 if err != nil { 343 return 0, errdefs.Unknown(errors.Wrapf(err, "couldn't open previously written blob")) 344 } 345 size := info.Size 346 info.Labels = map[string]string{"containerd.io/uncompressed": uncompressedDigest.String()} 347 348 _, err = cs.Update(ctx, info, "labels.*") 349 if err != nil { 350 return 0, errdefs.System(errors.Wrapf(err, "couldn't set uncompressed label")) 351 } 352 return size, nil 353 } 354 355 // storeJson marshals the provided object as json and stores it. 356 func storeJson(ctx context.Context, cs content.Ingester, mt string, obj interface{}, labels map[string]string) (ocispec.Descriptor, error) { 357 configData, err := json.Marshal(obj) 358 if err != nil { 359 return ocispec.Descriptor{}, errdefs.InvalidParameter(err) 360 } 361 configDigest := digest.FromBytes(configData) 362 if err != nil { 363 return ocispec.Descriptor{}, errdefs.InvalidParameter(err) 364 } 365 desc := ocispec.Descriptor{ 366 MediaType: mt, 367 Digest: configDigest, 368 Size: int64(len(configData)), 369 } 370 371 var opts []content.Opt 372 if labels != nil { 373 opts = append(opts, content.WithLabels(labels)) 374 } 375 376 err = content.WriteBlob(ctx, cs, configDigest.String(), bytes.NewReader(configData), desc, opts...) 377 if err != nil { 378 return ocispec.Descriptor{}, errdefs.System(err) 379 } 380 return desc, nil 381 } 382 383 func containerConfigToOciImageConfig(cfg *container.Config) ocispec.ImageConfig { 384 ociCfg := ocispec.ImageConfig{ 385 User: cfg.User, 386 Env: cfg.Env, 387 Entrypoint: cfg.Entrypoint, 388 Cmd: cfg.Cmd, 389 Volumes: cfg.Volumes, 390 WorkingDir: cfg.WorkingDir, 391 Labels: cfg.Labels, 392 StopSignal: cfg.StopSignal, 393 ArgsEscaped: cfg.ArgsEscaped, 394 } 395 for k, v := range cfg.ExposedPorts { 396 ociCfg.ExposedPorts[string(k)] = v 397 } 398 399 return ociCfg 400 }