github.com/rish1988/moby@v25.0.2+incompatible/daemon/containerd/image_import.go (about) 1 package containerd 2 3 import ( 4 "bufio" 5 "bytes" 6 "context" 7 "encoding/json" 8 "fmt" 9 "io" 10 "time" 11 12 "github.com/containerd/containerd/content" 13 cerrdefs "github.com/containerd/containerd/errdefs" 14 "github.com/containerd/containerd/images" 15 "github.com/containerd/containerd/platforms" 16 "github.com/containerd/log" 17 "github.com/distribution/reference" 18 "github.com/docker/docker/api/types/container" 19 "github.com/docker/docker/api/types/events" 20 "github.com/docker/docker/builder/dockerfile" 21 "github.com/docker/docker/errdefs" 22 "github.com/docker/docker/image" 23 imagespec "github.com/docker/docker/image/spec/specs-go/v1" 24 "github.com/docker/docker/internal/compatcontext" 25 "github.com/docker/docker/pkg/archive" 26 "github.com/docker/docker/pkg/pools" 27 "github.com/google/uuid" 28 "github.com/opencontainers/go-digest" 29 "github.com/opencontainers/image-spec/specs-go" 30 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 31 "github.com/pkg/errors" 32 ) 33 34 // ImportImage imports an image, getting the archived layer data from layerReader. 35 // Layer archive is imported as-is if the compression is gzip or zstd. 36 // Uncompressed, xz and bzip2 archives are recompressed into gzip. 37 // The image is tagged with the given reference. 38 // If the platform is nil, the default host platform is used. 39 // The message is used as the history comment. 40 // Image configuration is derived from the dockerfile instructions in changes. 41 func (i *ImageService) ImportImage(ctx context.Context, ref reference.Named, platform *ocispec.Platform, msg string, layerReader io.Reader, changes []string) (image.ID, error) { 42 refString := "" 43 if ref != nil { 44 refString = ref.String() 45 } 46 logger := log.G(ctx).WithField("ref", refString) 47 48 ctx, release, err := i.client.WithLease(ctx) 49 if err != nil { 50 return "", errdefs.System(err) 51 } 52 defer func() { 53 if err := release(compatcontext.WithoutCancel(ctx)); err != nil { 54 logger.WithError(err).Warn("failed to release lease created for import") 55 } 56 }() 57 58 if platform == nil { 59 def := platforms.DefaultSpec() 60 platform = &def 61 } 62 63 imageConfig, err := dockerfile.BuildFromConfig(ctx, &container.Config{}, changes, platform.OS) 64 if err != nil { 65 logger.WithError(err).Debug("failed to process changes") 66 return "", errdefs.InvalidParameter(err) 67 } 68 69 cs := i.client.ContentStore() 70 71 compressedDigest, uncompressedDigest, mt, err := saveArchive(ctx, cs, layerReader) 72 if err != nil { 73 logger.WithError(err).Debug("failed to write layer blob") 74 return "", err 75 } 76 logger = logger.WithFields(log.Fields{ 77 "compressedDigest": compressedDigest, 78 "uncompressedDigest": uncompressedDigest, 79 }) 80 81 size, err := fillUncompressedLabel(ctx, cs, compressedDigest, uncompressedDigest) 82 if err != nil { 83 logger.WithError(err).Debug("failed to set uncompressed label on the compressed blob") 84 return "", err 85 } 86 87 compressedRootfsDesc := ocispec.Descriptor{ 88 MediaType: mt, 89 Digest: compressedDigest, 90 Size: size, 91 } 92 93 dockerCfg := containerConfigToDockerOCIImageConfig(imageConfig) 94 createdAt := time.Now() 95 config := imagespec.DockerOCIImage{ 96 Image: ocispec.Image{ 97 Platform: *platform, 98 Created: &createdAt, 99 Author: "", 100 RootFS: ocispec.RootFS{ 101 Type: "layers", 102 DiffIDs: []digest.Digest{uncompressedDigest}, 103 }, 104 History: []ocispec.History{ 105 { 106 Created: &createdAt, 107 CreatedBy: "", 108 Author: "", 109 Comment: msg, 110 EmptyLayer: false, 111 }, 112 }, 113 }, 114 Config: dockerCfg, 115 } 116 configDesc, err := storeJson(ctx, cs, ocispec.MediaTypeImageConfig, config, nil) 117 if err != nil { 118 return "", err 119 } 120 121 manifest := ocispec.Manifest{ 122 MediaType: ocispec.MediaTypeImageManifest, 123 Versioned: specs.Versioned{ 124 SchemaVersion: 2, 125 }, 126 Config: configDesc, 127 Layers: []ocispec.Descriptor{ 128 compressedRootfsDesc, 129 }, 130 } 131 manifestDesc, err := storeJson(ctx, cs, ocispec.MediaTypeImageManifest, manifest, map[string]string{ 132 "containerd.io/gc.ref.content.config": configDesc.Digest.String(), 133 "containerd.io/gc.ref.content.l.0": compressedDigest.String(), 134 }) 135 if err != nil { 136 return "", err 137 } 138 139 id := image.ID(manifestDesc.Digest.String()) 140 img := images.Image{ 141 Name: refString, 142 Target: manifestDesc, 143 CreatedAt: createdAt, 144 } 145 if img.Name == "" { 146 img.Name = danglingImageName(manifestDesc.Digest) 147 } 148 149 err = i.saveImage(ctx, img) 150 if err != nil { 151 logger.WithError(err).Debug("failed to save image") 152 return "", err 153 } 154 155 err = i.unpackImage(ctx, i.StorageDriver(), img, manifestDesc) 156 if err != nil { 157 logger.WithError(err).Debug("failed to unpack image") 158 } else { 159 i.LogImageEvent(id.String(), id.String(), events.ActionImport) 160 } 161 162 return id, err 163 } 164 165 // saveArchive saves the archive from bufRd to the content store, compressing it if necessary. 166 // Returns compressed blob digest, digest of the uncompressed data and media type of the stored blob. 167 func saveArchive(ctx context.Context, cs content.Store, layerReader io.Reader) (digest.Digest, digest.Digest, string, error) { 168 // Wrap the reader in buffered reader to allow peeks. 169 p := pools.BufioReader32KPool 170 bufRd := p.Get(layerReader) 171 defer p.Put(bufRd) 172 173 compression, err := detectCompression(bufRd) 174 if err != nil { 175 return "", "", "", err 176 } 177 178 var uncompressedReader io.Reader = bufRd 179 switch compression { 180 case archive.Gzip, archive.Zstd: 181 // If the input is already a compressed layer, just save it as is. 182 mediaType := ocispec.MediaTypeImageLayerGzip 183 if compression == archive.Zstd { 184 mediaType = ocispec.MediaTypeImageLayerZstd 185 } 186 187 compressedDigest, uncompressedDigest, err := writeCompressedBlob(ctx, cs, mediaType, bufRd) 188 if err != nil { 189 return "", "", "", err 190 } 191 192 return compressedDigest, uncompressedDigest, mediaType, nil 193 case archive.Bzip2, archive.Xz: 194 r, err := archive.DecompressStream(bufRd) 195 if err != nil { 196 return "", "", "", errdefs.InvalidParameter(err) 197 } 198 defer r.Close() 199 uncompressedReader = r 200 fallthrough 201 case archive.Uncompressed: 202 mediaType := ocispec.MediaTypeImageLayerGzip 203 compression := archive.Gzip 204 205 compressedDigest, uncompressedDigest, err := compressAndWriteBlob(ctx, cs, compression, mediaType, uncompressedReader) 206 if err != nil { 207 return "", "", "", err 208 } 209 210 return compressedDigest, uncompressedDigest, mediaType, nil 211 } 212 213 return "", "", "", errdefs.InvalidParameter(errors.New("unsupported archive compression")) 214 } 215 216 // writeCompressedBlob writes the blob and simultaneously computes the digest of the uncompressed data. 217 func writeCompressedBlob(ctx context.Context, cs content.Store, mediaType string, bufRd *bufio.Reader) (digest.Digest, digest.Digest, error) { 218 pr, pw := io.Pipe() 219 defer pw.Close() 220 defer pr.Close() 221 222 c := make(chan digest.Digest) 223 // Start copying the blob to the content store from the pipe and tee it to the pipe. 224 go func() { 225 compressedDigest, err := writeBlobAndReturnDigest(ctx, cs, mediaType, io.TeeReader(bufRd, pw)) 226 pw.CloseWithError(err) 227 c <- compressedDigest 228 }() 229 230 digester := digest.Canonical.Digester() 231 232 // Decompress the piped blob. 233 decompressedStream, err := archive.DecompressStream(pr) 234 if err == nil { 235 // Feed the digester with decompressed data. 236 _, err = io.Copy(digester.Hash(), decompressedStream) 237 decompressedStream.Close() 238 } 239 pr.CloseWithError(err) 240 241 compressedDigest := <-c 242 if err != nil { 243 if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 244 return "", "", errdefs.Cancelled(err) 245 } 246 return "", "", errdefs.System(err) 247 } 248 249 uncompressedDigest := digester.Digest() 250 return compressedDigest, uncompressedDigest, nil 251 } 252 253 // compressAndWriteBlob compresses the uncompressedReader and stores it in the content store. 254 func compressAndWriteBlob(ctx context.Context, cs content.Store, compression archive.Compression, mediaType string, uncompressedLayerReader io.Reader) (digest.Digest, digest.Digest, error) { 255 pr, pw := io.Pipe() 256 defer pr.Close() 257 defer pw.Close() 258 259 compressor, err := archive.CompressStream(pw, compression) 260 if err != nil { 261 return "", "", errdefs.InvalidParameter(err) 262 } 263 264 writeChan := make(chan digest.Digest) 265 // Start copying the blob to the content store from the pipe. 266 go func() { 267 dgst, err := writeBlobAndReturnDigest(ctx, cs, mediaType, pr) 268 pr.CloseWithError(err) 269 writeChan <- dgst 270 }() 271 272 // Copy archive to the pipe and tee it to a digester. 273 // This will feed the pipe the above goroutine is reading from. 274 uncompressedDigester := digest.Canonical.Digester() 275 readFromInputAndDigest := io.TeeReader(uncompressedLayerReader, uncompressedDigester.Hash()) 276 _, err = io.Copy(compressor, readFromInputAndDigest) 277 compressor.Close() 278 pw.CloseWithError(err) 279 280 compressedDigest := <-writeChan 281 if err != nil { 282 if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 283 return "", "", errdefs.Cancelled(err) 284 } 285 return "", "", errdefs.System(err) 286 } 287 288 return compressedDigest, uncompressedDigester.Digest(), err 289 } 290 291 // writeBlobAndReturnDigest writes a blob to the content store and returns the digest. 292 func writeBlobAndReturnDigest(ctx context.Context, cs content.Store, mt string, reader io.Reader) (digest.Digest, error) { 293 digester := digest.Canonical.Digester() 294 if err := content.WriteBlob(ctx, cs, uuid.New().String(), io.TeeReader(reader, digester.Hash()), ocispec.Descriptor{MediaType: mt}); err != nil { 295 return "", errdefs.System(err) 296 } 297 return digester.Digest(), nil 298 } 299 300 // saveImage creates an image in the ImageService or updates it if it exists. 301 func (i *ImageService) saveImage(ctx context.Context, img images.Image) error { 302 is := i.client.ImageService() 303 304 if _, err := is.Update(ctx, img); err != nil { 305 if cerrdefs.IsNotFound(err) { 306 if _, err := is.Create(ctx, img); err != nil { 307 return errdefs.Unknown(err) 308 } 309 } else { 310 return errdefs.Unknown(err) 311 } 312 } 313 314 return nil 315 } 316 317 // unpackImage unpacks the platform-specific manifest of a image into the snapshotter. 318 func (i *ImageService) unpackImage(ctx context.Context, snapshotter string, img images.Image, manifestDesc ocispec.Descriptor) error { 319 c8dImg, err := i.NewImageManifest(ctx, img, manifestDesc) 320 if err != nil { 321 return err 322 } 323 324 if err := c8dImg.Unpack(ctx, snapshotter); err != nil { 325 if !cerrdefs.IsAlreadyExists(err) { 326 return errdefs.System(fmt.Errorf("failed to unpack image: %w", err)) 327 } 328 } 329 330 return nil 331 } 332 333 // detectCompression dectects the reader compression type. 334 func detectCompression(bufRd *bufio.Reader) (archive.Compression, error) { 335 bs, err := bufRd.Peek(10) 336 if err != nil && err != io.EOF { 337 // Note: we'll ignore any io.EOF error because there are some odd 338 // cases where the layer.tar file will be empty (zero bytes) and 339 // that results in an io.EOF from the Peek() call. So, in those 340 // cases we'll just treat it as a non-compressed stream and 341 // that means just create an empty layer. 342 // See Issue 18170 343 return archive.Uncompressed, errdefs.Unknown(err) 344 } 345 346 return archive.DetectCompression(bs), nil 347 } 348 349 // fillUncompressedLabel sets the uncompressed digest label on the compressed blob metadata 350 // and returns the compressed blob size. 351 func fillUncompressedLabel(ctx context.Context, cs content.Store, compressedDigest digest.Digest, uncompressedDigest digest.Digest) (int64, error) { 352 info, err := cs.Info(ctx, compressedDigest) 353 if err != nil { 354 return 0, errdefs.Unknown(errors.Wrapf(err, "couldn't open previously written blob")) 355 } 356 size := info.Size 357 info.Labels = map[string]string{"containerd.io/uncompressed": uncompressedDigest.String()} 358 359 _, err = cs.Update(ctx, info, "labels.*") 360 if err != nil { 361 return 0, errdefs.System(errors.Wrapf(err, "couldn't set uncompressed label")) 362 } 363 return size, nil 364 } 365 366 // storeJson marshals the provided object as json and stores it. 367 func storeJson(ctx context.Context, cs content.Ingester, mt string, obj interface{}, labels map[string]string) (ocispec.Descriptor, error) { 368 configData, err := json.Marshal(obj) 369 if err != nil { 370 return ocispec.Descriptor{}, errdefs.InvalidParameter(err) 371 } 372 configDigest := digest.FromBytes(configData) 373 if err != nil { 374 return ocispec.Descriptor{}, errdefs.InvalidParameter(err) 375 } 376 desc := ocispec.Descriptor{ 377 MediaType: mt, 378 Digest: configDigest, 379 Size: int64(len(configData)), 380 } 381 382 var opts []content.Opt 383 if labels != nil { 384 opts = append(opts, content.WithLabels(labels)) 385 } 386 387 err = content.WriteBlob(ctx, cs, configDigest.String(), bytes.NewReader(configData), desc, opts...) 388 if err != nil { 389 return ocispec.Descriptor{}, errdefs.System(err) 390 } 391 return desc, nil 392 }