github.com/demonoid81/containerd@v1.3.4/unpacker.go (about) 1 /* 2 Copyright The containerd Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package containerd 18 19 import ( 20 "context" 21 "encoding/json" 22 "fmt" 23 "sync" 24 "sync/atomic" 25 26 "github.com/containerd/containerd/content" 27 "github.com/containerd/containerd/images" 28 "github.com/containerd/containerd/log" 29 "github.com/containerd/containerd/rootfs" 30 "github.com/opencontainers/go-digest" 31 "github.com/opencontainers/image-spec/identity" 32 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 33 "github.com/pkg/errors" 34 "github.com/sirupsen/logrus" 35 "golang.org/x/sync/errgroup" 36 ) 37 38 type layerState struct { 39 layer rootfs.Layer 40 downloaded bool 41 unpacked bool 42 } 43 44 type unpacker struct { 45 updateCh chan ocispec.Descriptor 46 snapshotter string 47 config UnpackConfig 48 c *Client 49 } 50 51 func (c *Client) newUnpacker(ctx context.Context, rCtx *RemoteContext) (*unpacker, error) { 52 snapshotter, err := c.resolveSnapshotterName(ctx, rCtx.Snapshotter) 53 if err != nil { 54 return nil, err 55 } 56 var config UnpackConfig 57 for _, o := range rCtx.UnpackOpts { 58 if err := o(ctx, &config); err != nil { 59 return nil, err 60 } 61 } 62 return &unpacker{ 63 updateCh: make(chan ocispec.Descriptor, 128), 64 snapshotter: snapshotter, 65 config: config, 66 c: c, 67 }, nil 68 } 69 70 func (u *unpacker) unpack(ctx context.Context, config ocispec.Descriptor, layers []ocispec.Descriptor) error { 71 p, err := content.ReadBlob(ctx, u.c.ContentStore(), config) 72 if err != nil { 73 return err 74 } 75 76 var i ocispec.Image 77 if err := json.Unmarshal(p, &i); err != nil { 78 return errors.Wrap(err, "unmarshal image config") 79 } 80 diffIDs := i.RootFS.DiffIDs 81 if len(layers) != len(diffIDs) { 82 return errors.Errorf("number of layers and diffIDs don't match: %d != %d", len(layers), len(diffIDs)) 83 } 84 85 var ( 86 sn = u.c.SnapshotService(u.snapshotter) 87 a = u.c.DiffService() 88 cs = u.c.ContentStore() 89 90 states []layerState 91 chain []digest.Digest 92 ) 93 for i, desc := range layers { 94 states = append(states, layerState{ 95 layer: rootfs.Layer{ 96 Blob: desc, 97 Diff: ocispec.Descriptor{ 98 MediaType: ocispec.MediaTypeImageLayer, 99 Digest: diffIDs[i], 100 }, 101 }, 102 }) 103 } 104 for { 105 var layer ocispec.Descriptor 106 select { 107 case layer = <-u.updateCh: 108 case <-ctx.Done(): 109 return ctx.Err() 110 } 111 log.G(ctx).WithField("desc", layer).Debug("layer downloaded") 112 for i := range states { 113 if states[i].layer.Blob.Digest != layer.Digest { 114 continue 115 } 116 // Different layers may have the same digest. When that 117 // happens, we should continue marking the next layer 118 // as downloaded. 119 if states[i].downloaded { 120 continue 121 } 122 states[i].downloaded = true 123 break 124 } 125 for i := range states { 126 if !states[i].downloaded { 127 break 128 } 129 if states[i].unpacked { 130 continue 131 } 132 133 log.G(ctx).WithFields(logrus.Fields{ 134 "desc": states[i].layer.Blob, 135 "diff": states[i].layer.Diff, 136 }).Debug("unpack layer") 137 138 unpacked, err := rootfs.ApplyLayerWithOpts(ctx, states[i].layer, chain, sn, a, 139 u.config.SnapshotOpts, u.config.ApplyOpts) 140 if err != nil { 141 return err 142 } 143 144 if unpacked { 145 // Set the uncompressed label after the uncompressed 146 // digest has been verified through apply. 147 cinfo := content.Info{ 148 Digest: states[i].layer.Blob.Digest, 149 Labels: map[string]string{ 150 "containerd.io/uncompressed": states[i].layer.Diff.Digest.String(), 151 }, 152 } 153 if _, err := cs.Update(ctx, cinfo, "labels.containerd.io/uncompressed"); err != nil { 154 return err 155 } 156 } 157 158 chain = append(chain, states[i].layer.Diff.Digest) 159 states[i].unpacked = true 160 log.G(ctx).WithFields(logrus.Fields{ 161 "desc": states[i].layer.Blob, 162 "diff": states[i].layer.Diff, 163 }).Debug("layer unpacked") 164 } 165 // Check whether all layers are unpacked. 166 if states[len(states)-1].unpacked { 167 break 168 } 169 } 170 171 chainID := identity.ChainID(chain).String() 172 cinfo := content.Info{ 173 Digest: config.Digest, 174 Labels: map[string]string{ 175 fmt.Sprintf("containerd.io/gc.ref.snapshot.%s", u.snapshotter): chainID, 176 }, 177 } 178 _, err = cs.Update(ctx, cinfo, fmt.Sprintf("labels.containerd.io/gc.ref.snapshot.%s", u.snapshotter)) 179 if err != nil { 180 return err 181 } 182 log.G(ctx).WithFields(logrus.Fields{ 183 "config": config.Digest, 184 "chainID": chainID, 185 }).Debug("image unpacked") 186 return nil 187 } 188 189 type errGroup struct { 190 *errgroup.Group 191 cancel context.CancelFunc 192 } 193 194 func newErrGroup(ctx context.Context) (*errGroup, context.Context) { 195 ctx, cancel := context.WithCancel(ctx) 196 eg, ctx := errgroup.WithContext(ctx) 197 return &errGroup{ 198 Group: eg, 199 cancel: cancel, 200 }, ctx 201 } 202 203 func (e *errGroup) Cancel() { 204 e.cancel() 205 } 206 207 func (e *errGroup) Wait() error { 208 err := e.Group.Wait() 209 e.cancel() 210 return err 211 } 212 213 func (u *unpacker) handlerWrapper(uctx context.Context, unpacks *int32) (func(images.Handler) images.Handler, *errGroup) { 214 eg, uctx := newErrGroup(uctx) 215 return func(f images.Handler) images.Handler { 216 var ( 217 lock sync.Mutex 218 layers []ocispec.Descriptor 219 schema1 bool 220 ) 221 return images.HandlerFunc(func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { 222 children, err := f.Handle(ctx, desc) 223 if err != nil { 224 return children, err 225 } 226 227 // `Pull` only supports one platform, so there is only 228 // one manifest to handle, and manifest list can be 229 // safely skipped. 230 // TODO: support multi-platform unpack. 231 switch mt := desc.MediaType; { 232 case mt == images.MediaTypeDockerSchema1Manifest: 233 lock.Lock() 234 schema1 = true 235 lock.Unlock() 236 case mt == images.MediaTypeDockerSchema2Manifest || mt == ocispec.MediaTypeImageManifest: 237 lock.Lock() 238 for _, child := range children { 239 if child.MediaType == images.MediaTypeDockerSchema2Config || 240 child.MediaType == ocispec.MediaTypeImageConfig { 241 continue 242 } 243 layers = append(layers, child) 244 } 245 lock.Unlock() 246 case mt == images.MediaTypeDockerSchema2Config || mt == ocispec.MediaTypeImageConfig: 247 lock.Lock() 248 l := append([]ocispec.Descriptor{}, layers...) 249 lock.Unlock() 250 if len(l) > 0 { 251 atomic.AddInt32(unpacks, 1) 252 eg.Go(func() error { 253 return u.unpack(uctx, desc, l) 254 }) 255 } 256 case images.IsLayerType(mt): 257 lock.Lock() 258 update := !schema1 259 lock.Unlock() 260 if update { 261 select { 262 case <-uctx.Done(): 263 // Do not send update if unpacker is not running. 264 default: 265 select { 266 case u.updateCh <- desc: 267 case <-uctx.Done(): 268 // Do not send update if unpacker is not running. 269 } 270 } 271 // Checking ctx.Done() prevents the case that unpacker 272 // exits unexpectedly, but update continues to be generated, 273 // and eventually fills up updateCh and blocks forever. 274 } 275 } 276 return children, nil 277 }) 278 }, eg 279 }