zotregistry.dev/zot@v1.4.4-0.20240314164342-eec277e14d20/pkg/extensions/search/convert/metadb.go (about) 1 package convert 2 3 import ( 4 "context" 5 "sort" 6 "strconv" 7 "time" 8 9 "github.com/99designs/gqlgen/graphql" 10 ispec "github.com/opencontainers/image-spec/specs-go/v1" 11 "github.com/vektah/gqlparser/v2/gqlerror" 12 13 zerr "zotregistry.dev/zot/errors" 14 zcommon "zotregistry.dev/zot/pkg/common" 15 cveinfo "zotregistry.dev/zot/pkg/extensions/search/cve" 16 "zotregistry.dev/zot/pkg/extensions/search/gql_generated" 17 "zotregistry.dev/zot/pkg/extensions/search/pagination" 18 "zotregistry.dev/zot/pkg/log" 19 mTypes "zotregistry.dev/zot/pkg/meta/types" 20 reqCtx "zotregistry.dev/zot/pkg/requestcontext" 21 ) 22 23 type SkipQGLField struct { 24 Vulnerabilities bool 25 } 26 27 func UpdateLastUpdatedTimestamp(repoLastUpdatedTimestamp *time.Time, 28 lastUpdatedImageSummary *gql_generated.ImageSummary, imageSummary *gql_generated.ImageSummary, 29 ) *gql_generated.ImageSummary { 30 newLastUpdatedImageSummary := lastUpdatedImageSummary 31 32 if repoLastUpdatedTimestamp.Equal(time.Time{}) { 33 // initialize with first time value 34 *repoLastUpdatedTimestamp = *imageSummary.LastUpdated 35 newLastUpdatedImageSummary = imageSummary 36 } else if repoLastUpdatedTimestamp.Before(*imageSummary.LastUpdated) { 37 *repoLastUpdatedTimestamp = *imageSummary.LastUpdated 38 newLastUpdatedImageSummary = imageSummary 39 } 40 41 return newLastUpdatedImageSummary 42 } 43 44 func getReferrers(referrersInfo []mTypes.ReferrerInfo) []*gql_generated.Referrer { 45 referrers := make([]*gql_generated.Referrer, 0, len(referrersInfo)) 46 47 for _, referrerInfo := range referrersInfo { 48 referrerInfo := referrerInfo 49 50 referrers = append(referrers, &gql_generated.Referrer{ 51 MediaType: &referrerInfo.MediaType, 52 ArtifactType: &referrerInfo.ArtifactType, 53 Size: &referrerInfo.Size, 54 Digest: &referrerInfo.Digest, 55 Annotations: getAnnotationsFromMap(referrerInfo.Annotations), 56 }) 57 } 58 59 return referrers 60 } 61 62 func getAnnotationsFromMap(annotationsMap map[string]string) []*gql_generated.Annotation { 63 annotations := make([]*gql_generated.Annotation, 0, len(annotationsMap)) 64 65 for key, value := range annotationsMap { 66 key := key 67 value := value 68 69 annotations = append(annotations, &gql_generated.Annotation{ 70 Key: &key, 71 Value: &value, 72 }) 73 } 74 75 return annotations 76 } 77 78 func getImageBlobsInfo(manifestDigest string, manifestSize int64, configDigest string, configSize int64, 79 layers []ispec.Descriptor, 80 ) (int64, map[string]int64) { 81 imageBlobsMap := map[string]int64{} 82 imageSize := int64(0) 83 84 // add config size 85 imageSize += configSize 86 imageBlobsMap[configDigest] = configSize 87 88 // add manifest size 89 imageSize += manifestSize 90 imageBlobsMap[manifestDigest] = manifestSize 91 92 // add layers size 93 for _, layer := range layers { 94 imageBlobsMap[layer.Digest.String()] = layer.Size 95 imageSize += layer.Size 96 } 97 98 return imageSize, imageBlobsMap 99 } 100 101 func RepoMeta2ImageSummaries(ctx context.Context, repoMeta mTypes.RepoMeta, 102 imageMeta map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo, 103 ) []*gql_generated.ImageSummary { 104 imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) 105 106 // Make sure the tags are sorted 107 // We need to implement a proper fix for this taking into account 108 // the sorting criteria used in the requested page 109 tags := make([]string, 0, len(repoMeta.Tags)) 110 for tag := range repoMeta.Tags { 111 tags = append(tags, tag) 112 } 113 114 // Sorting ascending by tag name should do for now 115 sort.Strings(tags) 116 117 for _, tag := range tags { 118 descriptor := repoMeta.Tags[tag] 119 120 imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta[descriptor.Digest])) 121 if err != nil { 122 continue 123 } 124 125 // CVE scanning is expensive, only scan for final slice of results 126 updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) 127 128 imageSummaries = append(imageSummaries, imageSummary) 129 } 130 131 return imageSummaries 132 } 133 134 func RepoMeta2ExpandedRepoInfo(ctx context.Context, repoMeta mTypes.RepoMeta, 135 imageMetaMap map[string]mTypes.ImageMeta, skip SkipQGLField, cveInfo cveinfo.CveInfo, log log.Logger, 136 ) (*gql_generated.RepoSummary, []*gql_generated.ImageSummary) { 137 repoName := repoMeta.Name 138 imageSummaries := make([]*gql_generated.ImageSummary, 0, len(repoMeta.Tags)) 139 140 userCanDeleteTag, _ := reqCtx.CanDelete(ctx, repoName) 141 142 for tag, descriptor := range repoMeta.Tags { 143 imageMeta := imageMetaMap[descriptor.Digest] 144 145 imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(tag, repoMeta, imageMeta)) 146 if err != nil { 147 log.Error().Str("repository", repoName).Str("reference", tag).Str("component", "metadb"). 148 Msg("error while converting descriptor for image") 149 150 continue 151 } 152 153 imageSummary.IsDeletable = &userCanDeleteTag 154 155 updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) 156 157 imageSummaries = append(imageSummaries, imageSummary) 158 } 159 160 repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap) 161 162 updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) 163 164 return repoSummary, imageSummaries 165 } 166 167 func GetFullImageMeta(tag string, repoMeta mTypes.RepoMeta, imageMeta mTypes.ImageMeta, 168 ) mTypes.FullImageMeta { 169 return mTypes.FullImageMeta{ 170 Repo: repoMeta.Name, 171 Tag: tag, 172 MediaType: imageMeta.MediaType, 173 Digest: imageMeta.Digest, 174 Size: imageMeta.Size, 175 Index: imageMeta.Index, 176 Manifests: GetFullManifestMeta(repoMeta, imageMeta.Manifests), 177 Referrers: repoMeta.Referrers[imageMeta.Digest.String()], 178 Statistics: repoMeta.Statistics[imageMeta.Digest.String()], 179 Signatures: repoMeta.Signatures[imageMeta.Digest.String()], 180 } 181 } 182 183 func GetFullManifestMeta(repoMeta mTypes.RepoMeta, manifests []mTypes.ManifestMeta) []mTypes.FullManifestMeta { 184 results := make([]mTypes.FullManifestMeta, 0, len(manifests)) 185 186 for i := range manifests { 187 results = append(results, mTypes.FullManifestMeta{ 188 ManifestMeta: manifests[i], 189 Referrers: repoMeta.Referrers[manifests[i].Digest.String()], 190 Statistics: repoMeta.Statistics[manifests[i].Digest.String()], 191 Signatures: repoMeta.Signatures[manifests[i].Digest.String()], 192 }) 193 } 194 195 return results 196 } 197 198 func StringMap2Annotations(strMap map[string]string) []*gql_generated.Annotation { 199 annotations := make([]*gql_generated.Annotation, 0, len(strMap)) 200 201 for key, value := range strMap { 202 key := key 203 value := value 204 205 annotations = append(annotations, &gql_generated.Annotation{ 206 Key: &key, 207 Value: &value, 208 }) 209 } 210 211 return annotations 212 } 213 214 func GetPreloads(ctx context.Context) map[string]bool { 215 if !graphql.HasOperationContext(ctx) { 216 return map[string]bool{} 217 } 218 219 nestedPreloads := GetNestedPreloads( 220 graphql.GetOperationContext(ctx), 221 graphql.CollectFieldsCtx(ctx, nil), 222 "", 223 ) 224 225 preloads := map[string]bool{} 226 227 for _, str := range nestedPreloads { 228 preloads[str] = true 229 } 230 231 return preloads 232 } 233 234 func GetNestedPreloads(ctx *graphql.OperationContext, fields []graphql.CollectedField, prefix string, 235 ) []string { 236 preloads := []string{} 237 238 for _, column := range fields { 239 prefixColumn := GetPreloadString(prefix, column.Name) 240 preloads = append(preloads, prefixColumn) 241 preloads = append(preloads, 242 GetNestedPreloads(ctx, graphql.CollectFields(ctx, column.Selections, nil), prefixColumn)..., 243 ) 244 } 245 246 return preloads 247 } 248 249 func GetPreloadString(prefix, name string) string { 250 if len(prefix) > 0 { 251 return prefix + "." + name 252 } 253 254 return name 255 } 256 257 func GetSignaturesInfo(isSigned bool, signatures mTypes.ManifestSignatures) []*gql_generated.SignatureSummary { 258 signaturesInfo := []*gql_generated.SignatureSummary{} 259 260 if !isSigned { 261 return signaturesInfo 262 } 263 264 for sigType, signatures := range signatures { 265 for _, sig := range signatures { 266 for _, layer := range sig.LayersInfo { 267 var ( 268 isTrusted bool 269 author string 270 tool string 271 ) 272 273 if layer.Signer != "" { 274 author = layer.Signer 275 276 if !layer.Date.IsZero() && time.Now().After(layer.Date) { 277 isTrusted = false 278 } else { 279 isTrusted = true 280 } 281 } else { 282 isTrusted = false 283 author = "" 284 } 285 286 tool = sigType 287 288 signaturesInfo = append(signaturesInfo, 289 &gql_generated.SignatureSummary{Tool: &tool, IsTrusted: &isTrusted, Author: &author}) 290 } 291 } 292 } 293 294 return signaturesInfo 295 } 296 297 func PaginatedRepoMeta2RepoSummaries(ctx context.Context, repoMetaList []mTypes.RepoMeta, 298 imageMetaMap map[string]mTypes.ImageMeta, filter mTypes.Filter, pageInput pagination.PageInput, 299 cveInfo cveinfo.CveInfo, skip SkipQGLField, 300 ) ([]*gql_generated.RepoSummary, zcommon.PageInfo, error) { 301 reposPageFinder, err := pagination.NewRepoSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) 302 if err != nil { 303 return []*gql_generated.RepoSummary{}, zcommon.PageInfo{}, err 304 } 305 306 for _, repoMeta := range repoMetaList { 307 repoSummary := RepoMeta2RepoSummary(ctx, repoMeta, imageMetaMap) 308 309 if RepoSumAcceptedByFilter(repoSummary, filter) { 310 reposPageFinder.Add(repoSummary) 311 } 312 } 313 314 page, pageInfo := reposPageFinder.Page() 315 316 // CVE scanning is expensive, only scan for the current page 317 for _, repoSummary := range page { 318 updateRepoSummaryVulnerabilities(ctx, repoSummary, skip, cveInfo) 319 } 320 321 return page, pageInfo, nil 322 } 323 324 func RepoMeta2RepoSummary(ctx context.Context, repoMeta mTypes.RepoMeta, 325 imageMetaMap map[string]mTypes.ImageMeta, 326 ) *gql_generated.RepoSummary { 327 var ( 328 repoName = repoMeta.Name 329 lastUpdatedImage = deref(repoMeta.LastUpdatedImage, mTypes.LastUpdatedImage{}) 330 lastUpdatedImageMeta = imageMetaMap[lastUpdatedImage.Digest] 331 lastUpdatedTag = lastUpdatedImage.Tag 332 repoLastUpdatedTimestamp = lastUpdatedImage.LastUpdated 333 repoPlatforms = repoMeta.Platforms 334 repoVendors = repoMeta.Vendors 335 repoDownloadCount = repoMeta.DownloadCount 336 repoStarCount = repoMeta.StarCount 337 repoIsUserStarred = repoMeta.IsStarred // value specific to the current user 338 repoIsUserBookMarked = repoMeta.IsBookmarked // value specific to the current user 339 repoSize = repoMeta.Size 340 ) 341 342 if repoLastUpdatedTimestamp == nil { 343 repoLastUpdatedTimestamp = &time.Time{} 344 } 345 346 imageSummary, _, err := FullImageMeta2ImageSummary(ctx, GetFullImageMeta(lastUpdatedTag, repoMeta, 347 lastUpdatedImageMeta)) 348 _ = err 349 350 return &gql_generated.RepoSummary{ 351 Name: &repoName, 352 LastUpdated: repoLastUpdatedTimestamp, 353 Size: ref(strconv.FormatInt(repoSize, 10)), 354 Platforms: getGqlPlatforms(repoPlatforms), 355 Vendors: getGqlVendors(repoVendors), 356 NewestImage: imageSummary, 357 DownloadCount: &repoDownloadCount, 358 StarCount: &repoStarCount, 359 IsBookmarked: &repoIsUserBookMarked, 360 IsStarred: &repoIsUserStarred, 361 Rank: ref(repoMeta.Rank), 362 } 363 } 364 365 func getGqlVendors(repoVendors []string) []*string { 366 result := make([]*string, 0, len(repoVendors)) 367 368 for i := range repoVendors { 369 result = append(result, &repoVendors[i]) 370 } 371 372 return result 373 } 374 375 func getGqlPlatforms(repoPlatforms []ispec.Platform) []*gql_generated.Platform { 376 result := make([]*gql_generated.Platform, 0, len(repoPlatforms)) 377 378 for i := range repoPlatforms { 379 result = append(result, &gql_generated.Platform{ 380 Os: ref(repoPlatforms[i].OS), 381 Arch: ref(getArch(repoPlatforms[i].Architecture, repoPlatforms[i].Variant)), 382 }) 383 } 384 385 return result 386 } 387 388 type ( 389 ManifestDigest = string 390 BlobDigest = string 391 ) 392 393 func FullImageMeta2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta, 394 ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { 395 switch fullImageMeta.MediaType { 396 case ispec.MediaTypeImageManifest: 397 return ImageManifest2ImageSummary(ctx, fullImageMeta) 398 case ispec.MediaTypeImageIndex: 399 return ImageIndex2ImageSummary(ctx, fullImageMeta) 400 default: 401 return nil, nil, zerr.ErrMediaTypeNotSupported 402 } 403 } 404 405 func ImageIndex2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta, 406 ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { 407 var ( 408 repo = fullImageMeta.Repo 409 tag = fullImageMeta.Tag 410 indexLastUpdated time.Time 411 isSigned = isImageSigned(fullImageMeta.Signatures) 412 indexSize = int64(0) 413 manifestAnnotations *ImageAnnotations 414 manifestSummaries = make([]*gql_generated.ManifestSummary, 0, len(fullImageMeta.Manifests)) 415 indexBlobs = map[string]int64{} 416 417 indexDigestStr = fullImageMeta.Digest.String() 418 indexMediaType = ispec.MediaTypeImageIndex 419 ) 420 421 for _, imageManifest := range fullImageMeta.Manifests { 422 imageManifestSummary, manifestBlobs, err := ImageManifest2ImageSummary(ctx, mTypes.FullImageMeta{ 423 Repo: fullImageMeta.Repo, 424 Tag: fullImageMeta.Tag, 425 MediaType: ispec.MediaTypeImageManifest, 426 Digest: imageManifest.Digest, 427 Size: imageManifest.Size, 428 Manifests: []mTypes.FullManifestMeta{imageManifest}, 429 Referrers: imageManifest.Referrers, 430 Statistics: imageManifest.Statistics, 431 Signatures: imageManifest.Signatures, 432 }) 433 if err != nil { 434 return &gql_generated.ImageSummary{}, map[string]int64{}, err 435 } 436 437 manifestSize := int64(0) 438 439 for digest, size := range manifestBlobs { 440 indexBlobs[digest] = size 441 manifestSize += size 442 } 443 444 if indexLastUpdated.Before(*imageManifestSummary.LastUpdated) { 445 indexLastUpdated = *imageManifestSummary.LastUpdated 446 } 447 448 annotations := GetAnnotations(imageManifest.Manifest.Annotations, imageManifest.Config.Config.Labels) 449 if manifestAnnotations == nil { 450 manifestAnnotations = &annotations 451 } 452 453 indexSize += manifestSize 454 455 manifestSummaries = append(manifestSummaries, imageManifestSummary.Manifests[0]) 456 } 457 458 signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures) 459 460 if manifestAnnotations == nil { 461 manifestAnnotations = &ImageAnnotations{} 462 } 463 464 annotations := GetIndexAnnotations(fullImageMeta.Index.Annotations, manifestAnnotations) 465 466 imageLastUpdated := annotations.Created 467 if imageLastUpdated == nil { 468 imageLastUpdated = &indexLastUpdated 469 } 470 471 indexSummary := gql_generated.ImageSummary{ 472 RepoName: &repo, 473 Tag: &tag, 474 Digest: &indexDigestStr, 475 MediaType: &indexMediaType, 476 Manifests: manifestSummaries, 477 LastUpdated: imageLastUpdated, 478 IsSigned: &isSigned, 479 SignatureInfo: signaturesInfo, 480 Size: ref(strconv.FormatInt(indexSize, 10)), 481 DownloadCount: ref(fullImageMeta.Statistics.DownloadCount), 482 Description: &annotations.Description, 483 Title: &annotations.Title, 484 Documentation: &annotations.Documentation, 485 Licenses: &annotations.Licenses, 486 Labels: &annotations.Labels, 487 Source: &annotations.Source, 488 Vendor: &annotations.Vendor, 489 Authors: &annotations.Authors, 490 Referrers: getReferrers(fullImageMeta.Referrers), 491 } 492 493 return &indexSummary, indexBlobs, nil 494 } 495 496 func ImageManifest2ImageSummary(ctx context.Context, fullImageMeta mTypes.FullImageMeta, 497 ) (*gql_generated.ImageSummary, map[BlobDigest]int64, error) { 498 manifest := fullImageMeta.Manifests[0] 499 500 var ( 501 repoName = fullImageMeta.Repo 502 tag = fullImageMeta.Tag 503 configDigest = manifest.Manifest.Config.Digest.String() 504 configSize = manifest.Manifest.Config.Size 505 manifestDigest = manifest.Digest.String() 506 manifestSize = manifest.Size 507 mediaType = manifest.Manifest.MediaType 508 artifactType = zcommon.GetManifestArtifactType(fullImageMeta.Manifests[0].Manifest) 509 platform = getPlatform(manifest.Config.Platform) 510 downloadCount = fullImageMeta.Statistics.DownloadCount 511 isSigned = isImageSigned(fullImageMeta.Signatures) 512 ) 513 514 imageSize, imageBlobsMap := getImageBlobsInfo(manifestDigest, manifestSize, configDigest, configSize, 515 manifest.Manifest.Layers) 516 imageSizeStr := strconv.FormatInt(imageSize, 10) 517 annotations := GetAnnotations(manifest.Manifest.Annotations, manifest.Config.Config.Labels) 518 519 authors := annotations.Authors 520 if authors == "" { 521 authors = manifest.Config.Author 522 } 523 524 imageLastUpdated := annotations.Created 525 if imageLastUpdated == nil { 526 configCreated := zcommon.GetImageLastUpdated(manifest.Config) 527 imageLastUpdated = &configCreated 528 } 529 530 historyEntries, err := getAllHistory(manifest.Manifest, manifest.Config) 531 if err != nil { 532 graphql.AddError(ctx, gqlerror.Errorf("error generating history on tag %s in repo %s: "+ 533 "manifest digest: %s, error: %s", tag, repoName, manifest.Digest, err.Error())) 534 } 535 536 signaturesInfo := GetSignaturesInfo(isSigned, fullImageMeta.Signatures) 537 538 manifestSummary := gql_generated.ManifestSummary{ 539 Digest: &manifestDigest, 540 ConfigDigest: &configDigest, 541 LastUpdated: imageLastUpdated, 542 Size: &imageSizeStr, 543 IsSigned: &isSigned, 544 SignatureInfo: signaturesInfo, 545 Platform: &platform, 546 DownloadCount: &downloadCount, 547 Layers: getLayersSummaries(manifest.Manifest), 548 History: historyEntries, 549 Referrers: getReferrers(fullImageMeta.Referrers), 550 ArtifactType: &artifactType, 551 } 552 553 imageSummary := gql_generated.ImageSummary{ 554 RepoName: &repoName, 555 Tag: &tag, 556 Digest: &manifestDigest, 557 MediaType: &mediaType, 558 Manifests: []*gql_generated.ManifestSummary{&manifestSummary}, 559 LastUpdated: imageLastUpdated, 560 IsSigned: &isSigned, 561 SignatureInfo: signaturesInfo, 562 Size: &imageSizeStr, 563 DownloadCount: &downloadCount, 564 Description: &annotations.Description, 565 Title: &annotations.Title, 566 Documentation: &annotations.Documentation, 567 Licenses: &annotations.Licenses, 568 Labels: &annotations.Labels, 569 Source: &annotations.Source, 570 Vendor: &annotations.Vendor, 571 Authors: &authors, 572 Referrers: manifestSummary.Referrers, 573 } 574 575 return &imageSummary, imageBlobsMap, nil 576 } 577 578 func isImageSigned(manifestSignatures mTypes.ManifestSignatures) bool { 579 for _, signatures := range manifestSignatures { 580 if len(signatures) > 0 { 581 return true 582 } 583 } 584 585 return false 586 } 587 588 func getPlatform(platform ispec.Platform) gql_generated.Platform { 589 return gql_generated.Platform{ 590 Os: ref(platform.OS), 591 Arch: ref(getArch(platform.Architecture, platform.Variant)), 592 } 593 } 594 595 func getArch(arch string, variant string) string { 596 if variant != "" { 597 arch = arch + "/" + variant 598 } 599 600 return arch 601 } 602 603 func ref[T any](val T) *T { 604 ref := val 605 606 return &ref 607 } 608 609 func deref[T any](pointer *T, defaultVal T) T { 610 if pointer != nil { 611 return *pointer 612 } 613 614 return defaultVal 615 } 616 617 func PaginatedFullImageMeta2ImageSummaries(ctx context.Context, imageMetaList []mTypes.FullImageMeta, skip SkipQGLField, 618 cveInfo cveinfo.CveInfo, filter mTypes.Filter, pageInput pagination.PageInput, 619 ) ([]*gql_generated.ImageSummary, zcommon.PageInfo, error) { 620 imagePageFinder, err := pagination.NewImgSumPageFinder(pageInput.Limit, pageInput.Offset, pageInput.SortBy) 621 if err != nil { 622 return []*gql_generated.ImageSummary{}, zcommon.PageInfo{}, err 623 } 624 625 for _, imageMeta := range imageMetaList { 626 imageSummary, _, err := FullImageMeta2ImageSummary(ctx, imageMeta) 627 if err != nil { 628 continue 629 } 630 631 if ImgSumAcceptedByFilter(imageSummary, filter) { 632 imagePageFinder.Add(imageSummary) 633 } 634 } 635 636 page, pageInfo := imagePageFinder.Page() 637 638 for _, imageSummary := range page { 639 // CVE scanning is expensive, only scan for this page 640 updateImageSummaryVulnerabilities(ctx, imageSummary, skip, cveInfo) 641 } 642 643 return page, pageInfo, nil 644 }