github.com/Prakhar-Agarwal-byte/moby@v0.0.0-20231027092010-a14e3e8ab87e/distribution/manifest.go (about) 1 package distribution 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "io" 8 "strings" 9 10 "github.com/containerd/containerd/content" 11 cerrdefs "github.com/containerd/containerd/errdefs" 12 "github.com/containerd/containerd/remotes" 13 "github.com/containerd/log" 14 "github.com/distribution/reference" 15 "github.com/docker/distribution" 16 "github.com/docker/distribution/manifest/manifestlist" 17 "github.com/docker/distribution/manifest/schema1" 18 "github.com/docker/distribution/manifest/schema2" 19 "github.com/Prakhar-Agarwal-byte/moby/registry" 20 "github.com/opencontainers/go-digest" 21 ocispec "github.com/opencontainers/image-spec/specs-go/v1" 22 "github.com/pkg/errors" 23 ) 24 25 // labelDistributionSource describes the source blob comes from. 26 const labelDistributionSource = "containerd.io/distribution.source" 27 28 // This is used by manifestStore to pare down the requirements to implement a 29 // full distribution.ManifestService, since `Get` is all we use here. 30 type manifestGetter interface { 31 Get(ctx context.Context, dgst digest.Digest, options ...distribution.ManifestServiceOption) (distribution.Manifest, error) 32 Exists(ctx context.Context, dgst digest.Digest) (bool, error) 33 } 34 35 type manifestStore struct { 36 local ContentStore 37 remote manifestGetter 38 } 39 40 // ContentStore is the interface used to persist registry blobs 41 // 42 // Currently this is only used to persist manifests and manifest lists. 43 // It is exported because `distribution.Pull` takes one as an argument. 44 type ContentStore interface { 45 content.Ingester 46 content.Provider 47 Info(ctx context.Context, dgst digest.Digest) (content.Info, error) 48 Abort(ctx context.Context, ref string) error 49 Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) 50 } 51 52 func makeDistributionSourceLabel(ref reference.Named) (string, string) { 53 domain := reference.Domain(ref) 54 if domain == "" { 55 domain = registry.DefaultNamespace 56 } 57 repo := reference.Path(ref) 58 59 return fmt.Sprintf("%s.%s", labelDistributionSource, domain), repo 60 } 61 62 // Taken from https://github.com/containerd/containerd/blob/e079e4a155c86f07bbd602fe6753ecacc78198c2/remotes/docker/handler.go#L84-L108 63 func appendDistributionSourceLabel(originLabel, repo string) string { 64 repos := []string{} 65 if originLabel != "" { 66 repos = strings.Split(originLabel, ",") 67 } 68 repos = append(repos, repo) 69 70 // use empty string to present duplicate items 71 for i := 1; i < len(repos); i++ { 72 tmp, j := repos[i], i-1 73 for ; j >= 0 && repos[j] >= tmp; j-- { 74 if repos[j] == tmp { 75 tmp = "" 76 } 77 repos[j+1] = repos[j] 78 } 79 repos[j+1] = tmp 80 } 81 82 i := 0 83 for ; i < len(repos) && repos[i] == ""; i++ { 84 } 85 86 return strings.Join(repos[i:], ",") 87 } 88 89 func hasDistributionSource(label, repo string) bool { 90 sources := strings.Split(label, ",") 91 for _, s := range sources { 92 if s == repo { 93 return true 94 } 95 } 96 return false 97 } 98 99 func (m *manifestStore) getLocal(ctx context.Context, desc ocispec.Descriptor, ref reference.Named) (distribution.Manifest, error) { 100 ra, err := m.local.ReaderAt(ctx, desc) 101 if err != nil { 102 return nil, errors.Wrap(err, "error getting content store reader") 103 } 104 defer ra.Close() 105 106 distKey, distRepo := makeDistributionSourceLabel(ref) 107 info, err := m.local.Info(ctx, desc.Digest) 108 if err != nil { 109 return nil, errors.Wrap(err, "error getting content info") 110 } 111 112 if _, ok := ref.(reference.Canonical); ok { 113 // Since this is specified by digest... 114 // We know we have the content locally, we need to check if we've seen this content at the specified repository before. 115 // If we have, we can just return the manifest from the local content store. 116 // If we haven't, we need to check the remote repository to see if it has the content, otherwise we can end up returning 117 // a manifest that has never even existed in the remote before. 118 if !hasDistributionSource(info.Labels[distKey], distRepo) { 119 log.G(ctx).WithField("ref", ref).Debug("found manifest but no mataching source repo is listed, checking with remote") 120 exists, err := m.remote.Exists(ctx, desc.Digest) 121 if err != nil { 122 return nil, errors.Wrap(err, "error checking if remote exists") 123 } 124 125 if !exists { 126 return nil, errors.Wrapf(cerrdefs.ErrNotFound, "manifest %v not found", desc.Digest) 127 } 128 129 } 130 } 131 132 // Update the distribution sources since we now know the content exists in the remote. 133 if info.Labels == nil { 134 info.Labels = map[string]string{} 135 } 136 info.Labels[distKey] = appendDistributionSourceLabel(info.Labels[distKey], distRepo) 137 if _, err := m.local.Update(ctx, info, "labels."+distKey); err != nil { 138 log.G(ctx).WithError(err).WithField("ref", ref).Warn("Could not update content distribution source") 139 } 140 141 r := io.NewSectionReader(ra, 0, ra.Size()) 142 data, err := io.ReadAll(r) 143 if err != nil { 144 return nil, errors.Wrap(err, "error reading manifest from content store") 145 } 146 147 manifest, _, err := distribution.UnmarshalManifest(desc.MediaType, data) 148 if err != nil { 149 return nil, errors.Wrap(err, "error unmarshaling manifest from content store") 150 } 151 152 return manifest, nil 153 } 154 155 func (m *manifestStore) getMediaType(ctx context.Context, desc ocispec.Descriptor) (string, error) { 156 ra, err := m.local.ReaderAt(ctx, desc) 157 if err != nil { 158 return "", errors.Wrap(err, "error getting reader to detect media type") 159 } 160 defer ra.Close() 161 162 mt, err := detectManifestMediaType(ra) 163 if err != nil { 164 return "", errors.Wrap(err, "error detecting media type") 165 } 166 return mt, nil 167 } 168 169 func (m *manifestStore) Get(ctx context.Context, desc ocispec.Descriptor, ref reference.Named) (distribution.Manifest, error) { 170 l := log.G(ctx) 171 172 if desc.MediaType == "" { 173 // When pulling by digest we will not have the media type on the 174 // descriptor since we have not made a request to the registry yet 175 // 176 // We already have the digest, so we only lookup locally... by digest. 177 // 178 // Let's try to detect the media type so we can have a good ref key 179 // here. We may not even have the content locally, and this is fine, but 180 // if we do we should determine that. 181 mt, err := m.getMediaType(ctx, desc) 182 if err != nil && !cerrdefs.IsNotFound(err) { 183 l.WithError(err).Warn("Error looking up media type of content") 184 } 185 desc.MediaType = mt 186 } 187 188 key := remotes.MakeRefKey(ctx, desc) 189 190 // Here we open a writer to the requested content. This both gives us a 191 // reference to write to if indeed we need to persist it and increments the 192 // ref count on the content. 193 w, err := m.local.Writer(ctx, content.WithDescriptor(desc), content.WithRef(key)) 194 if err != nil { 195 if cerrdefs.IsAlreadyExists(err) { 196 var manifest distribution.Manifest 197 if manifest, err = m.getLocal(ctx, desc, ref); err == nil { 198 return manifest, nil 199 } 200 } 201 // always fallback to the remote if there is an error with the local store 202 } 203 if w != nil { 204 defer w.Close() 205 } 206 207 l.WithError(err).Debug("Fetching manifest from remote") 208 209 manifest, err := m.remote.Get(ctx, desc.Digest) 210 if err != nil { 211 if err := m.local.Abort(ctx, key); err != nil { 212 l.WithError(err).Warn("Error while attempting to abort content ingest") 213 } 214 return nil, err 215 } 216 217 if w != nil { 218 // if `w` is nil here, something happened with the content store, so don't bother trying to persist. 219 if err := m.Put(ctx, manifest, desc, w, ref); err != nil { 220 if err := m.local.Abort(ctx, key); err != nil { 221 l.WithError(err).Warn("error aborting content ingest") 222 } 223 l.WithError(err).Warn("Error persisting manifest") 224 } 225 } 226 return manifest, nil 227 } 228 229 func (m *manifestStore) Put(ctx context.Context, manifest distribution.Manifest, desc ocispec.Descriptor, w content.Writer, ref reference.Named) error { 230 mt, payload, err := manifest.Payload() 231 if err != nil { 232 return err 233 } 234 desc.Size = int64(len(payload)) 235 desc.MediaType = mt 236 237 if _, err = w.Write(payload); err != nil { 238 return errors.Wrap(err, "error writing manifest to content store") 239 } 240 241 distKey, distSource := makeDistributionSourceLabel(ref) 242 if err := w.Commit(ctx, desc.Size, desc.Digest, content.WithLabels(map[string]string{ 243 distKey: distSource, 244 })); err != nil { 245 return errors.Wrap(err, "error committing manifest to content store") 246 } 247 return nil 248 } 249 250 func detectManifestMediaType(ra content.ReaderAt) (string, error) { 251 dt := make([]byte, ra.Size()) 252 if _, err := ra.ReadAt(dt, 0); err != nil { 253 return "", err 254 } 255 256 return detectManifestBlobMediaType(dt) 257 } 258 259 // This is used when the manifest store does not know the media type of a sha it 260 // was told to get. This would currently only happen when pulling by digest. 261 // The media type is needed so the blob can be unmarshalled properly. 262 func detectManifestBlobMediaType(dt []byte) (string, error) { 263 var mfst struct { 264 MediaType string `json:"mediaType"` 265 Manifests json.RawMessage `json:"manifests"` // oci index, manifest list 266 Config json.RawMessage `json:"config"` // schema2 Manifest 267 Layers json.RawMessage `json:"layers"` // schema2 Manifest 268 FSLayers json.RawMessage `json:"fsLayers"` // schema1 Manifest 269 } 270 271 if err := json.Unmarshal(dt, &mfst); err != nil { 272 return "", err 273 } 274 275 // We may have a media type specified in the json, in which case that should be used. 276 // Docker types should generally have a media type set. 277 // OCI (golang) types do not have a `mediaType` defined, and it is optional in the spec. 278 // 279 // `distribution.UnmarshalManifest`, which is used to unmarshal this for real, checks these media type values. 280 // If the specified media type does not match it will error, and in some cases (docker media types) it is required. 281 // So pretty much if we don't have a media type we can fall back to OCI. 282 // This does have a special fallback for schema1 manifests just because it is easy to detect. 283 switch mfst.MediaType { 284 case schema2.MediaTypeManifest, ocispec.MediaTypeImageManifest: 285 if mfst.Manifests != nil || mfst.FSLayers != nil { 286 return "", fmt.Errorf(`media-type: %q should not have "manifests" or "fsLayers"`, mfst.MediaType) 287 } 288 return mfst.MediaType, nil 289 case manifestlist.MediaTypeManifestList, ocispec.MediaTypeImageIndex: 290 if mfst.Config != nil || mfst.Layers != nil || mfst.FSLayers != nil { 291 return "", fmt.Errorf(`media-type: %q should not have "config", "layers", or "fsLayers"`, mfst.MediaType) 292 } 293 return mfst.MediaType, nil 294 case schema1.MediaTypeManifest: 295 if mfst.Manifests != nil || mfst.Layers != nil { 296 return "", fmt.Errorf(`media-type: %q should not have "manifests" or "layers"`, mfst.MediaType) 297 } 298 return mfst.MediaType, nil 299 default: 300 if mfst.MediaType != "" { 301 return mfst.MediaType, nil 302 } 303 } 304 switch { 305 case mfst.FSLayers != nil && mfst.Manifests == nil && mfst.Layers == nil && mfst.Config == nil: 306 return schema1.MediaTypeManifest, nil 307 case mfst.Config != nil && mfst.Manifests == nil && mfst.FSLayers == nil, 308 mfst.Layers != nil && mfst.Manifests == nil && mfst.FSLayers == nil: 309 return ocispec.MediaTypeImageManifest, nil 310 case mfst.Config == nil && mfst.Layers == nil && mfst.FSLayers == nil: 311 // fallback to index 312 return ocispec.MediaTypeImageIndex, nil 313 } 314 return "", errors.New("media-type: cannot determine") 315 }