zotregistry.io/zot@v1.4.4-0.20231124084042-02a8ed785457/pkg/meta/boltdb/boltdb.go (about) 1 package boltdb 2 3 import ( 4 "context" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "strings" 9 "time" 10 11 godigest "github.com/opencontainers/go-digest" 12 ispec "github.com/opencontainers/image-spec/specs-go/v1" 13 "go.etcd.io/bbolt" 14 "google.golang.org/protobuf/proto" 15 "google.golang.org/protobuf/types/known/timestamppb" 16 17 zerr "zotregistry.io/zot/errors" 18 "zotregistry.io/zot/pkg/api/constants" 19 zcommon "zotregistry.io/zot/pkg/common" 20 "zotregistry.io/zot/pkg/log" 21 "zotregistry.io/zot/pkg/meta/common" 22 mConvert "zotregistry.io/zot/pkg/meta/convert" 23 proto_go "zotregistry.io/zot/pkg/meta/proto/gen" 24 mTypes "zotregistry.io/zot/pkg/meta/types" 25 "zotregistry.io/zot/pkg/meta/version" 26 reqCtx "zotregistry.io/zot/pkg/requestcontext" 27 ) 28 29 type BoltDB struct { 30 DB *bbolt.DB 31 Patches []func(DB *bbolt.DB) error 32 imgTrustStore mTypes.ImageTrustStore 33 Log log.Logger 34 } 35 36 func New(boltDB *bbolt.DB, log log.Logger) (*BoltDB, error) { 37 err := boltDB.Update(func(transaction *bbolt.Tx) error { 38 versionBuck, err := transaction.CreateBucketIfNotExists([]byte(VersionBucket)) 39 if err != nil { 40 return err 41 } 42 43 err = versionBuck.Put([]byte(version.DBVersionKey), []byte(version.CurrentVersion)) 44 if err != nil { 45 return err 46 } 47 48 _, err = transaction.CreateBucketIfNotExists([]byte(UserDataBucket)) 49 if err != nil { 50 return err 51 } 52 53 _, err = transaction.CreateBucketIfNotExists([]byte(UserAPIKeysBucket)) 54 if err != nil { 55 return err 56 } 57 58 _, err = transaction.CreateBucketIfNotExists([]byte(ImageMetaBuck)) 59 if err != nil { 60 return err 61 } 62 63 _, err = transaction.CreateBucketIfNotExists([]byte(RepoMetaBuck)) 64 if err != nil { 65 return err 66 } 67 68 repoBlobsBuck, err := transaction.CreateBucketIfNotExists([]byte(RepoBlobsBuck)) 69 if err != nil { 70 return err 71 } 72 73 _, err = repoBlobsBuck.CreateBucketIfNotExists([]byte(RepoLastUpdatedBuck)) 74 if err != nil { 75 return err 76 } 77 78 return nil 79 }) 80 if err != nil { 81 return nil, err 82 } 83 84 return &BoltDB{ 85 DB: boltDB, 86 Patches: version.GetBoltDBPatches(), 87 imgTrustStore: nil, 88 Log: log, 89 }, nil 90 } 91 92 func (bdw *BoltDB) GetAllRepoNames() ([]string, error) { 93 repoNames := []string{} 94 95 err := bdw.DB.View(func(tx *bbolt.Tx) error { 96 repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) 97 98 return repoMetaBuck.ForEach(func(repo, _ []byte) error { 99 repoNames = append(repoNames, string(repo)) 100 101 return nil 102 }) 103 }) 104 105 return repoNames, err 106 } 107 108 func (bdw *BoltDB) GetRepoLastUpdated(repo string) time.Time { 109 lastUpdated := time.Time{} 110 111 err := bdw.DB.View(func(tx *bbolt.Tx) error { 112 repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) 113 114 lastUpdatedBuck := repoBlobsBuck.Bucket([]byte(RepoLastUpdatedBuck)) 115 116 lastUpdatedBlob := lastUpdatedBuck.Get([]byte(repo)) 117 if len(lastUpdatedBlob) == 0 { 118 return zerr.ErrRepoMetaNotFound 119 } 120 121 protoTime := ×tamppb.Timestamp{} 122 123 err := proto.Unmarshal(lastUpdatedBlob, protoTime) 124 if err != nil { 125 return err 126 } 127 128 lastUpdated = *mConvert.GetTime(protoTime) 129 130 return nil 131 }) 132 if err != nil { 133 return time.Time{} 134 } 135 136 return lastUpdated 137 } 138 139 func (bdw *BoltDB) SetImageMeta(digest godigest.Digest, imageMeta mTypes.ImageMeta) error { 140 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 141 buck := tx.Bucket([]byte(ImageMetaBuck)) 142 143 protoImageMeta := &proto_go.ImageMeta{} 144 145 switch imageMeta.MediaType { 146 case ispec.MediaTypeImageManifest: 147 manifest := imageMeta.Manifests[0] 148 149 protoImageMeta = mConvert.GetProtoImageManifestData(manifest.Manifest, manifest.Config, 150 manifest.Size, manifest.Digest.String()) 151 case ispec.MediaTypeImageIndex: 152 protoImageMeta = mConvert.GetProtoImageIndexMeta(*imageMeta.Index, imageMeta.Size, imageMeta.Digest.String()) 153 } 154 155 pImageMetaBlob, err := proto.Marshal(protoImageMeta) 156 if err != nil { 157 return fmt.Errorf("metadb: error while calculating blob for manifest with digest %s %w", digest, err) 158 } 159 160 err = buck.Put([]byte(digest), pImageMetaBlob) 161 if err != nil { 162 return fmt.Errorf("metadb: error while setting manifest data with for digest %s %w", digest, err) 163 } 164 165 return nil 166 }) 167 168 return err 169 } 170 171 func (bdw *BoltDB) SetRepoReference(ctx context.Context, repo string, reference string, imageMeta mTypes.ImageMeta, 172 ) error { 173 if err := common.ValidateRepoReferenceInput(repo, reference, imageMeta.Digest); err != nil { 174 return err 175 } 176 177 var userid string 178 179 userAc, err := reqCtx.UserAcFromContext(ctx) 180 if err == nil { 181 userid = userAc.GetUsername() 182 } 183 184 err = bdw.DB.Update(func(tx *bbolt.Tx) error { 185 repoBuck := tx.Bucket([]byte(RepoMetaBuck)) 186 repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) 187 repoLastUpdatedBuck := repoBlobsBuck.Bucket([]byte(RepoLastUpdatedBuck)) 188 imageBuck := tx.Bucket([]byte(ImageMetaBuck)) 189 190 // 1. Add image data to db if needed 191 192 protoImageMeta := mConvert.GetProtoImageMeta(imageMeta) 193 194 imageMetaBlob, err := proto.Marshal(protoImageMeta) 195 if err != nil { 196 return err 197 } 198 199 err = imageBuck.Put([]byte(imageMeta.Digest), imageMetaBlob) 200 if err != nil { 201 return err 202 } 203 204 protoRepoMeta, err := getProtoRepoMeta(repo, repoBuck) 205 if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { 206 return err 207 } 208 209 // 2. Referrers 210 if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { 211 refInfo := &proto_go.ReferrersInfo{} 212 if protoRepoMeta.Referrers[subject.Digest.String()] != nil { 213 refInfo = protoRepoMeta.Referrers[subject.Digest.String()] 214 } 215 216 foundReferrer := false 217 218 for i := range refInfo.List { 219 if refInfo.List[i].Digest == mConvert.GetImageDigestStr(protoImageMeta) { 220 foundReferrer = true 221 refInfo.List[i].Count += 1 222 223 break 224 } 225 } 226 227 if !foundReferrer { 228 refInfo.List = append(refInfo.List, &proto_go.ReferrerInfo{ 229 Count: 1, 230 MediaType: protoImageMeta.MediaType, 231 Digest: mConvert.GetImageDigestStr(protoImageMeta), 232 ArtifactType: mConvert.GetImageArtifactType(protoImageMeta), 233 Size: mConvert.GetImageManifestSize(protoImageMeta), 234 Annotations: mConvert.GetImageAnnotations(protoImageMeta), 235 }) 236 } 237 238 protoRepoMeta.Referrers[subject.Digest.String()] = refInfo 239 } 240 241 // 3. Update tag 242 if !common.ReferenceIsDigest(reference) { 243 protoRepoMeta.Tags[reference] = &proto_go.TagDescriptor{ 244 Digest: imageMeta.Digest.String(), 245 MediaType: imageMeta.MediaType, 246 } 247 } 248 249 if _, ok := protoRepoMeta.Statistics[imageMeta.Digest.String()]; !ok { 250 protoRepoMeta.Statistics[imageMeta.Digest.String()] = &proto_go.DescriptorStatistics{ 251 DownloadCount: 0, 252 LastPullTimestamp: ×tamppb.Timestamp{}, 253 PushTimestamp: timestamppb.Now(), 254 PushedBy: userid, 255 } 256 } else if protoRepoMeta.Statistics[imageMeta.Digest.String()].PushTimestamp.AsTime().IsZero() { 257 protoRepoMeta.Statistics[imageMeta.Digest.String()].PushTimestamp = timestamppb.Now() 258 } 259 260 if _, ok := protoRepoMeta.Signatures[imageMeta.Digest.String()]; !ok { 261 protoRepoMeta.Signatures[imageMeta.Digest.String()] = &proto_go.ManifestSignatures{ 262 Map: map[string]*proto_go.SignaturesInfo{"": {}}, 263 } 264 } 265 266 if _, ok := protoRepoMeta.Referrers[imageMeta.Digest.String()]; !ok { 267 protoRepoMeta.Referrers[imageMeta.Digest.String()] = &proto_go.ReferrersInfo{ 268 List: []*proto_go.ReferrerInfo{}, 269 } 270 } 271 272 // 4. Blobs 273 repoBlobsBytes := repoBlobsBuck.Get([]byte(repo)) 274 275 repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes) 276 if err != nil { 277 return err 278 } 279 280 protoRepoMeta, repoBlobs = common.AddImageMetaToRepoMeta(protoRepoMeta, repoBlobs, reference, imageMeta) 281 282 err = setRepoLastUpdated(repo, time.Now(), repoLastUpdatedBuck) 283 if err != nil { 284 return err 285 } 286 287 err = setProtoRepoBlobs(repoBlobs, repoBlobsBuck) 288 if err != nil { 289 return err 290 } 291 292 return setProtoRepoMeta(protoRepoMeta, repoBuck) 293 }) 294 295 return err 296 } 297 298 func setRepoLastUpdated(repo string, lastUpdated time.Time, repoLastUpdatedBuck *bbolt.Bucket) error { 299 protoTime := timestamppb.New(lastUpdated) 300 301 protoTimeBlob, err := proto.Marshal(protoTime) 302 if err != nil { 303 return err 304 } 305 306 return repoLastUpdatedBuck.Put([]byte(repo), protoTimeBlob) 307 } 308 309 func unmarshalProtoRepoBlobs(repo string, repoBlobsBytes []byte) (*proto_go.RepoBlobs, error) { 310 repoBlobs := &proto_go.RepoBlobs{ 311 Name: repo, 312 } 313 314 if len(repoBlobsBytes) > 0 { 315 err := proto.Unmarshal(repoBlobsBytes, repoBlobs) 316 if err != nil { 317 return nil, err 318 } 319 } 320 321 if repoBlobs.Blobs == nil { 322 repoBlobs.Blobs = map[string]*proto_go.BlobInfo{"": {}} 323 } 324 325 return repoBlobs, nil 326 } 327 328 func setProtoRepoBlobs(repoBlobs *proto_go.RepoBlobs, repoBlobsBuck *bbolt.Bucket) error { 329 repoBlobsBytes, err := proto.Marshal(repoBlobs) 330 if err != nil { 331 return err 332 } 333 334 return repoBlobsBuck.Put([]byte(repoBlobs.Name), repoBlobsBytes) 335 } 336 337 func getProtoRepoMeta(repo string, repoMetaBuck *bbolt.Bucket) (*proto_go.RepoMeta, error) { 338 repoMetaBlob := repoMetaBuck.Get([]byte(repo)) 339 340 return unmarshalProtoRepoMeta(repo, repoMetaBlob) 341 } 342 343 // unmarshalProtoRepoMeta will unmarshal the repoMeta blob and initialize nil maps. If the blob is empty 344 // an empty initialized object is returned. 345 func unmarshalProtoRepoMeta(repo string, repoMetaBlob []byte) (*proto_go.RepoMeta, error) { 346 protoRepoMeta := &proto_go.RepoMeta{ 347 Name: repo, 348 } 349 350 if len(repoMetaBlob) > 0 { 351 err := proto.Unmarshal(repoMetaBlob, protoRepoMeta) 352 if err != nil { 353 return protoRepoMeta, err 354 } 355 } 356 357 if protoRepoMeta.Tags == nil { 358 protoRepoMeta.Tags = map[string]*proto_go.TagDescriptor{"": {}} 359 } 360 361 if protoRepoMeta.Statistics == nil { 362 protoRepoMeta.Statistics = map[string]*proto_go.DescriptorStatistics{"": {}} 363 } 364 365 if protoRepoMeta.Signatures == nil { 366 protoRepoMeta.Signatures = map[string]*proto_go.ManifestSignatures{"": {}} 367 } 368 369 if protoRepoMeta.Referrers == nil { 370 protoRepoMeta.Referrers = map[string]*proto_go.ReferrersInfo{"": {}} 371 } 372 373 if len(repoMetaBlob) == 0 { 374 return protoRepoMeta, zerr.ErrRepoMetaNotFound 375 } 376 377 return protoRepoMeta, nil 378 } 379 380 func setProtoRepoMeta(repoMeta *proto_go.RepoMeta, repoBuck *bbolt.Bucket) error { 381 repoMetaBlob, err := proto.Marshal(repoMeta) 382 if err != nil { 383 return err 384 } 385 386 return repoBuck.Put([]byte(repoMeta.Name), repoMetaBlob) 387 } 388 389 func (bdw *BoltDB) FilterImageMeta(ctx context.Context, digests []string, 390 ) (map[string]mTypes.ImageMeta, error) { 391 imageMetaMap := map[string]mTypes.ImageMeta{} 392 393 err := bdw.DB.View(func(transaction *bbolt.Tx) error { 394 imageBuck := transaction.Bucket([]byte(ImageMetaBuck)) 395 396 for _, digest := range digests { 397 protoImageMeta, err := getProtoImageMeta(imageBuck, digest) 398 if err != nil { 399 return err 400 } 401 402 if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { 403 manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) 404 405 for _, manifest := range protoImageMeta.Index.Index.Manifests { 406 imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) 407 if err != nil { 408 return err 409 } 410 411 manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) 412 } 413 414 protoImageMeta.Manifests = manifestDataList 415 } 416 417 imageMetaMap[digest] = mConvert.GetImageMeta(protoImageMeta) 418 } 419 420 return nil 421 }) 422 423 return imageMetaMap, err 424 } 425 426 func (bdw *BoltDB) SearchRepos(ctx context.Context, searchText string, 427 ) ([]mTypes.RepoMeta, error) { 428 repos := []mTypes.RepoMeta{} 429 430 err := bdw.DB.View(func(transaction *bbolt.Tx) error { 431 var ( 432 repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) 433 userBookmarks = getUserBookmarks(ctx, transaction) 434 userStars = getUserStars(ctx, transaction) 435 ) 436 437 cursor := repoBuck.Cursor() 438 439 for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { 440 if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { 441 continue 442 } 443 444 rank := common.RankRepoName(searchText, string(repoName)) 445 if rank == -1 { 446 continue 447 } 448 449 protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) 450 if err != nil { 451 return err 452 } 453 454 delete(protoRepoMeta.Tags, "") 455 456 if len(protoRepoMeta.Tags) == 0 { 457 continue 458 } 459 460 protoRepoMeta.Rank = int32(rank) 461 protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) 462 protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) 463 464 repos = append(repos, mConvert.GetRepoMeta(protoRepoMeta)) 465 } 466 467 return nil 468 }) 469 470 return repos, err 471 } 472 473 func getProtoImageMeta(imageBuck *bbolt.Bucket, digest string) (*proto_go.ImageMeta, error) { 474 imageMetaBlob := imageBuck.Get([]byte(digest)) 475 476 if len(imageMetaBlob) == 0 { 477 return nil, zerr.ErrImageMetaNotFound 478 } 479 480 imageMeta := proto_go.ImageMeta{} 481 482 err := proto.Unmarshal(imageMetaBlob, &imageMeta) 483 if err != nil { 484 return nil, err 485 } 486 487 return &imageMeta, nil 488 } 489 490 func (bdw *BoltDB) SearchTags(ctx context.Context, searchText string, 491 ) ([]mTypes.FullImageMeta, error) { 492 images := []mTypes.FullImageMeta{} 493 494 searchedRepo, searchedTag, err := common.GetRepoTag(searchText) 495 if err != nil { 496 return []mTypes.FullImageMeta{}, 497 fmt.Errorf("metadb: error while parsing search text, invalid format %w", err) 498 } 499 500 err = bdw.DB.View(func(transaction *bbolt.Tx) error { 501 var ( 502 repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) 503 imageBuck = transaction.Bucket([]byte(ImageMetaBuck)) 504 userBookmarks = getUserBookmarks(ctx, transaction) 505 userStars = getUserStars(ctx, transaction) 506 ) 507 508 repoName, repoMetaBlob := repoBuck.Cursor().Seek([]byte(searchedRepo)) 509 510 if string(repoName) != searchedRepo { 511 return nil 512 } 513 514 if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { 515 return err 516 } 517 518 protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) 519 if err != nil { 520 return err 521 } 522 523 delete(protoRepoMeta.Tags, "") 524 525 protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) 526 protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) 527 528 for tag, descriptor := range protoRepoMeta.Tags { 529 if !strings.HasPrefix(tag, searchedTag) || tag == "" { 530 continue 531 } 532 533 var protoImageMeta *proto_go.ImageMeta 534 535 switch descriptor.MediaType { 536 case ispec.MediaTypeImageManifest: 537 manifestDigest := descriptor.Digest 538 539 imageManifestData, err := getProtoImageMeta(imageBuck, manifestDigest) 540 if err != nil { 541 return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", 542 manifestDigest, err) 543 } 544 545 protoImageMeta = imageManifestData 546 case ispec.MediaTypeImageIndex: 547 indexDigest := descriptor.Digest 548 549 imageIndexData, err := getProtoImageMeta(imageBuck, indexDigest) 550 if err != nil { 551 return fmt.Errorf("metadb: error fetching manifest meta for manifest with digest %s %w", 552 indexDigest, err) 553 } 554 555 manifestDataList := make([]*proto_go.ManifestMeta, 0, len(imageIndexData.Index.Index.Manifests)) 556 557 for _, manifest := range imageIndexData.Index.Index.Manifests { 558 imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) 559 if err != nil { 560 return err 561 } 562 563 manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) 564 } 565 566 imageIndexData.Manifests = manifestDataList 567 568 protoImageMeta = imageIndexData 569 default: 570 bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") 571 572 continue 573 } 574 575 images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta)) 576 } 577 578 return nil 579 }) 580 581 return images, err 582 } 583 584 func (bdw *BoltDB) FilterTags(ctx context.Context, filterRepoTag mTypes.FilterRepoTagFunc, 585 filterFunc mTypes.FilterFunc, 586 ) ([]mTypes.FullImageMeta, error) { 587 images := []mTypes.FullImageMeta{} 588 589 err := bdw.DB.View(func(transaction *bbolt.Tx) error { 590 var ( 591 repoBuck = transaction.Bucket([]byte(RepoMetaBuck)) 592 imageMetaBuck = transaction.Bucket([]byte(ImageMetaBuck)) 593 userBookmarks = getUserBookmarks(ctx, transaction) 594 userStars = getUserStars(ctx, transaction) 595 viewError error 596 ) 597 598 cursor := repoBuck.Cursor() 599 repoName, repoMetaBlob := cursor.First() 600 601 for ; repoName != nil; repoName, repoMetaBlob = cursor.Next() { 602 if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { 603 continue 604 } 605 606 protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) 607 if err != nil { 608 viewError = errors.Join(viewError, err) 609 610 continue 611 } 612 613 delete(protoRepoMeta.Tags, "") 614 protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, protoRepoMeta.Name) 615 protoRepoMeta.IsStarred = zcommon.Contains(userStars, protoRepoMeta.Name) 616 repoMeta := mConvert.GetRepoMeta(protoRepoMeta) 617 618 for tag, descriptor := range protoRepoMeta.Tags { 619 if !filterRepoTag(string(repoName), tag) { 620 continue 621 } 622 623 switch descriptor.MediaType { 624 case ispec.MediaTypeImageManifest: 625 manifestDigest := descriptor.Digest 626 627 imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest) 628 if err != nil { 629 viewError = errors.Join(viewError, err) 630 631 continue 632 } 633 634 imageMeta := mConvert.GetImageMeta(imageManifestData) 635 636 if filterFunc(repoMeta, imageMeta) { 637 images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, imageManifestData)) 638 } 639 case ispec.MediaTypeImageIndex: 640 indexDigest := descriptor.Digest 641 642 imageIndexData, err := getProtoImageMeta(imageMetaBuck, indexDigest) 643 if err != nil { 644 viewError = errors.Join(viewError, err) 645 646 continue 647 } 648 649 matchedManifests := []*proto_go.ManifestMeta{} 650 651 for _, manifest := range imageIndexData.Index.Index.Manifests { 652 manifestDigest := manifest.Digest 653 654 imageManifestData, err := getProtoImageMeta(imageMetaBuck, manifestDigest) 655 if err != nil { 656 viewError = errors.Join(viewError, err) 657 658 continue 659 } 660 661 imageMeta := mConvert.GetImageMeta(imageManifestData) 662 663 if filterFunc(repoMeta, imageMeta) { 664 matchedManifests = append(matchedManifests, imageManifestData.Manifests[0]) 665 } 666 } 667 668 if len(matchedManifests) > 0 { 669 imageIndexData.Manifests = matchedManifests 670 671 images = append(images, mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, imageIndexData)) 672 } 673 default: 674 bdw.Log.Error().Str("mediaType", descriptor.MediaType).Msg("Unsupported media type") 675 676 continue 677 } 678 } 679 } 680 681 return viewError 682 }) 683 684 return images, err 685 } 686 687 func (bdw *BoltDB) FilterRepos(ctx context.Context, acceptName mTypes.FilterRepoNameFunc, 688 filter mTypes.FilterFullRepoFunc, 689 ) ([]mTypes.RepoMeta, error) { 690 repos := []mTypes.RepoMeta{} 691 692 err := bdw.DB.View(func(tx *bbolt.Tx) error { 693 var ( 694 buck = tx.Bucket([]byte(RepoMetaBuck)) 695 cursor = buck.Cursor() 696 userBookmarks = getUserBookmarks(ctx, tx) 697 userStars = getUserStars(ctx, tx) 698 ) 699 700 for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { 701 if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { 702 continue 703 } 704 705 if !acceptName(string(repoName)) { 706 continue 707 } 708 709 repoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) 710 if err != nil { 711 return err 712 } 713 714 repoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repoMeta.Name) 715 repoMeta.IsStarred = zcommon.Contains(userStars, repoMeta.Name) 716 717 fullRepoMeta := mConvert.GetRepoMeta(repoMeta) 718 719 if filter(fullRepoMeta) { 720 repos = append(repos, fullRepoMeta) 721 } 722 } 723 724 return nil 725 }) 726 if err != nil { 727 return []mTypes.RepoMeta{}, err 728 } 729 730 return repos, err 731 } 732 733 func (bdw *BoltDB) GetRepoMeta(ctx context.Context, repo string) (mTypes.RepoMeta, error) { 734 var protoRepoMeta *proto_go.RepoMeta 735 736 err := bdw.DB.View(func(tx *bbolt.Tx) error { 737 buck := tx.Bucket([]byte(RepoMetaBuck)) 738 userBookmarks := getUserBookmarks(ctx, tx) 739 userStars := getUserStars(ctx, tx) 740 741 repoMetaBlob := buck.Get([]byte(repo)) 742 743 var err error 744 745 protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob) 746 if err != nil { 747 return err 748 } 749 750 delete(protoRepoMeta.Tags, "") 751 protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) 752 protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) 753 754 return nil 755 }) 756 757 return mConvert.GetRepoMeta(protoRepoMeta), err 758 } 759 760 func (bdw *BoltDB) GetFullImageMeta(ctx context.Context, repo string, tag string) (mTypes.FullImageMeta, error) { 761 protoRepoMeta := &proto_go.RepoMeta{} 762 protoImageMeta := &proto_go.ImageMeta{} 763 764 err := bdw.DB.View(func(tx *bbolt.Tx) error { 765 buck := tx.Bucket([]byte(RepoMetaBuck)) 766 imageBuck := tx.Bucket([]byte(ImageMetaBuck)) 767 userBookmarks := getUserBookmarks(ctx, tx) 768 userStars := getUserStars(ctx, tx) 769 770 repoMetaBlob := buck.Get([]byte(repo)) 771 772 // object not found 773 if len(repoMetaBlob) == 0 { 774 return zerr.ErrRepoMetaNotFound 775 } 776 777 var err error 778 779 protoRepoMeta, err = unmarshalProtoRepoMeta(repo, repoMetaBlob) 780 if err != nil { 781 return err 782 } 783 784 delete(protoRepoMeta.Tags, "") 785 protoRepoMeta.IsBookmarked = zcommon.Contains(userBookmarks, repo) 786 protoRepoMeta.IsStarred = zcommon.Contains(userStars, repo) 787 788 descriptor, ok := protoRepoMeta.Tags[tag] 789 if !ok { 790 return zerr.ErrImageMetaNotFound 791 } 792 793 protoImageMeta, err = getProtoImageMeta(imageBuck, descriptor.Digest) 794 if err != nil { 795 return err 796 } 797 798 if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { 799 manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) 800 801 for _, manifest := range protoImageMeta.Index.Index.Manifests { 802 imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) 803 if err != nil { 804 return err 805 } 806 807 manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) 808 } 809 810 protoImageMeta.Manifests = manifestDataList 811 } 812 813 return nil 814 }) 815 816 return mConvert.GetFullImageMetaFromProto(tag, protoRepoMeta, protoImageMeta), err 817 } 818 819 func (bdw *BoltDB) GetImageMeta(digest godigest.Digest) (mTypes.ImageMeta, error) { 820 imageMeta := mTypes.ImageMeta{} 821 822 err := bdw.DB.View(func(tx *bbolt.Tx) error { 823 imageBuck := tx.Bucket([]byte(ImageMetaBuck)) 824 825 protoImageMeta, err := getProtoImageMeta(imageBuck, digest.String()) 826 if err != nil { 827 return err 828 } 829 830 if protoImageMeta.MediaType == ispec.MediaTypeImageIndex { 831 manifestDataList := make([]*proto_go.ManifestMeta, 0, len(protoImageMeta.Index.Index.Manifests)) 832 833 for _, manifest := range protoImageMeta.Index.Index.Manifests { 834 imageManifestData, err := getProtoImageMeta(imageBuck, manifest.Digest) 835 if err != nil { 836 return err 837 } 838 839 manifestDataList = append(manifestDataList, imageManifestData.Manifests[0]) 840 } 841 842 protoImageMeta.Manifests = manifestDataList 843 } 844 845 imageMeta = mConvert.GetImageMeta(protoImageMeta) 846 847 return nil 848 }) 849 850 return imageMeta, err 851 } 852 853 func (bdw *BoltDB) GetMultipleRepoMeta(ctx context.Context, filter func(repoMeta mTypes.RepoMeta) bool, 854 ) ([]mTypes.RepoMeta, error) { 855 foundRepos := []mTypes.RepoMeta{} 856 857 err := bdw.DB.View(func(tx *bbolt.Tx) error { 858 buck := tx.Bucket([]byte(RepoMetaBuck)) 859 860 cursor := buck.Cursor() 861 862 for repoName, repoMetaBlob := cursor.First(); repoName != nil; repoName, repoMetaBlob = cursor.Next() { 863 if ok, err := reqCtx.RepoIsUserAvailable(ctx, string(repoName)); !ok || err != nil { 864 continue 865 } 866 867 protoRepoMeta, err := unmarshalProtoRepoMeta(string(repoName), repoMetaBlob) 868 if err != nil { 869 return err 870 } 871 872 delete(protoRepoMeta.Tags, "") 873 874 repoMeta := mConvert.GetRepoMeta(protoRepoMeta) 875 876 if filter(repoMeta) { 877 foundRepos = append(foundRepos, repoMeta) 878 } 879 } 880 881 return nil 882 }) 883 884 return foundRepos, err 885 } 886 887 func (bdw *BoltDB) AddManifestSignature(repo string, signedManifestDigest godigest.Digest, 888 sigMeta mTypes.SignatureMetadata, 889 ) error { 890 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 891 repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) 892 893 repoMetaBlob := repoMetaBuck.Get([]byte(repo)) 894 895 if len(repoMetaBlob) == 0 { 896 var err error 897 // create a new object 898 repoMeta := proto_go.RepoMeta{ 899 Name: repo, 900 Tags: map[string]*proto_go.TagDescriptor{"": {}}, 901 Signatures: map[string]*proto_go.ManifestSignatures{ 902 signedManifestDigest.String(): { 903 Map: map[string]*proto_go.SignaturesInfo{ 904 sigMeta.SignatureType: { 905 List: []*proto_go.SignatureInfo{ 906 { 907 SignatureManifestDigest: sigMeta.SignatureDigest, 908 LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), 909 }, 910 }, 911 }, 912 }, 913 }, 914 }, 915 Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, 916 Statistics: map[string]*proto_go.DescriptorStatistics{"": {}}, 917 } 918 919 repoMetaBlob, err = proto.Marshal(&repoMeta) 920 if err != nil { 921 return err 922 } 923 924 return repoMetaBuck.Put([]byte(repo), repoMetaBlob) 925 } 926 927 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 928 if err != nil { 929 return err 930 } 931 932 var ( 933 manifestSignatures *proto_go.ManifestSignatures 934 found bool 935 ) 936 937 if manifestSignatures, found = protoRepoMeta.Signatures[signedManifestDigest.String()]; !found { 938 manifestSignatures = &proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} 939 } 940 941 signatureSlice := &proto_go.SignaturesInfo{List: []*proto_go.SignatureInfo{}} 942 if sigSlice, found := manifestSignatures.Map[sigMeta.SignatureType]; found { 943 signatureSlice = sigSlice 944 } 945 946 if !common.ProtoSignatureAlreadyExists(signatureSlice.List, sigMeta) { 947 switch sigMeta.SignatureType { 948 case zcommon.NotationSignature: 949 signatureSlice.List = append(signatureSlice.List, &proto_go.SignatureInfo{ 950 SignatureManifestDigest: sigMeta.SignatureDigest, 951 LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), 952 }) 953 case zcommon.CosignSignature: 954 newCosignSig := &proto_go.SignatureInfo{ 955 SignatureManifestDigest: sigMeta.SignatureDigest, 956 LayersInfo: mConvert.GetProtoLayersInfo(sigMeta.LayersInfo), 957 } 958 959 if zcommon.IsCosignTag(sigMeta.SignatureTag) { 960 // the entry for "sha256-{digest}.sig" signatures should be overwritten if 961 // it exists or added on the first position if it doesn't exist 962 if len(signatureSlice.GetList()) == 0 { 963 signatureSlice.List = []*proto_go.SignatureInfo{newCosignSig} 964 } else { 965 signatureSlice.List[0] = newCosignSig 966 } 967 } else { 968 // the first position should be reserved for "sha256-{digest}.sig" signatures 969 if len(signatureSlice.GetList()) == 0 { 970 signatureSlice.List = []*proto_go.SignatureInfo{{ 971 SignatureManifestDigest: "", 972 LayersInfo: []*proto_go.LayersInfo{}, 973 }} 974 } 975 976 signatureSlice.List = append(signatureSlice.List, newCosignSig) 977 } 978 } 979 } 980 981 manifestSignatures.Map[sigMeta.SignatureType] = signatureSlice 982 protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures 983 984 return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) 985 }) 986 987 return err 988 } 989 990 func (bdw *BoltDB) DeleteSignature(repo string, signedManifestDigest godigest.Digest, 991 sigMeta mTypes.SignatureMetadata, 992 ) error { 993 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 994 repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) 995 996 repoMetaBlob := repoMetaBuck.Get([]byte(repo)) 997 if len(repoMetaBlob) == 0 { 998 return zerr.ErrManifestMetaNotFound 999 } 1000 1001 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1002 if err != nil { 1003 return err 1004 } 1005 1006 manifestSignatures, found := protoRepoMeta.Signatures[signedManifestDigest.String()] 1007 if !found { 1008 return zerr.ErrManifestMetaNotFound 1009 } 1010 1011 signatureSlice := manifestSignatures.Map[sigMeta.SignatureType] 1012 1013 newSignatureSlice := make([]*proto_go.SignatureInfo, 0, len(signatureSlice.List)) 1014 1015 for _, sigInfo := range signatureSlice.List { 1016 if sigInfo.SignatureManifestDigest != sigMeta.SignatureDigest { 1017 newSignatureSlice = append(newSignatureSlice, sigInfo) 1018 } 1019 } 1020 1021 manifestSignatures.Map[sigMeta.SignatureType] = &proto_go.SignaturesInfo{List: newSignatureSlice} 1022 1023 protoRepoMeta.Signatures[signedManifestDigest.String()] = manifestSignatures 1024 1025 return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) 1026 }) 1027 1028 return err 1029 } 1030 1031 func (bdw *BoltDB) IncrementRepoStars(repo string) error { 1032 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 1033 repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) 1034 1035 repoMetaBlob := repoMetaBuck.Get([]byte(repo)) 1036 if len(repoMetaBlob) == 0 { 1037 return zerr.ErrRepoMetaNotFound 1038 } 1039 1040 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1041 if err != nil { 1042 return err 1043 } 1044 1045 protoRepoMeta.Stars++ 1046 1047 return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) 1048 }) 1049 1050 return err 1051 } 1052 1053 func (bdw *BoltDB) DecrementRepoStars(repo string) error { 1054 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 1055 repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) 1056 1057 repoMetaBlob := repoMetaBuck.Get([]byte(repo)) 1058 if len(repoMetaBlob) == 0 { 1059 return zerr.ErrRepoMetaNotFound 1060 } 1061 1062 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1063 if err != nil { 1064 return err 1065 } 1066 1067 if protoRepoMeta.Stars == 0 { 1068 return nil 1069 } 1070 1071 protoRepoMeta.Stars-- 1072 1073 return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) 1074 }) 1075 1076 return err 1077 } 1078 1079 func (bdw *BoltDB) SetRepoMeta(repo string, repoMeta mTypes.RepoMeta) error { 1080 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 1081 buck := tx.Bucket([]byte(RepoMetaBuck)) 1082 repoLastUpdatedBuck := tx.Bucket([]byte(RepoBlobsBuck)).Bucket([]byte(RepoLastUpdatedBuck)) 1083 1084 repoMeta.Name = repo 1085 1086 repoMetaBlob, err := proto.Marshal(mConvert.GetProtoRepoMeta(repoMeta)) 1087 if err != nil { 1088 return err 1089 } 1090 1091 err = buck.Put([]byte(repo), repoMetaBlob) 1092 if err != nil { 1093 return err 1094 } 1095 1096 // The last update time is set to 0 in order to force an update in case of a next storage parsing 1097 return setRepoLastUpdated(repo, time.Time{}, repoLastUpdatedBuck) 1098 }) 1099 1100 return err 1101 } 1102 1103 func (bdw *BoltDB) DeleteRepoMeta(repo string) error { 1104 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 1105 repoBuck := tx.Bucket([]byte(RepoMetaBuck)) 1106 repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) 1107 repoLastUpdatedBuck := repoBlobsBuck.Bucket([]byte(RepoLastUpdatedBuck)) 1108 1109 err := repoBuck.Delete([]byte(repo)) 1110 if err != nil { 1111 return err 1112 } 1113 1114 err = repoBlobsBuck.Delete([]byte(repo)) 1115 if err != nil { 1116 return err 1117 } 1118 1119 return repoLastUpdatedBuck.Delete([]byte(repo)) 1120 }) 1121 1122 return err 1123 } 1124 1125 func (bdw *BoltDB) ResetRepoReferences(repo string) error { 1126 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 1127 buck := tx.Bucket([]byte(RepoMetaBuck)) 1128 1129 repoMetaBlob := buck.Get([]byte(repo)) 1130 1131 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1132 if err != nil && !errors.Is(err, zerr.ErrRepoMetaNotFound) { 1133 return err 1134 } 1135 1136 repoMetaBlob, err = proto.Marshal(&proto_go.RepoMeta{ 1137 Name: repo, 1138 Statistics: protoRepoMeta.Statistics, 1139 Stars: protoRepoMeta.Stars, 1140 Tags: map[string]*proto_go.TagDescriptor{"": {}}, 1141 Signatures: map[string]*proto_go.ManifestSignatures{"": {Map: map[string]*proto_go.SignaturesInfo{"": {}}}}, 1142 Referrers: map[string]*proto_go.ReferrersInfo{"": {}}, 1143 }) 1144 if err != nil { 1145 return err 1146 } 1147 1148 return buck.Put([]byte(repo), repoMetaBlob) 1149 }) 1150 1151 return err 1152 } 1153 1154 func (bdw *BoltDB) GetReferrersInfo(repo string, referredDigest godigest.Digest, artifactTypes []string, 1155 ) ([]mTypes.ReferrerInfo, error) { 1156 referrersInfoResult := []mTypes.ReferrerInfo{} 1157 1158 err := bdw.DB.View(func(tx *bbolt.Tx) error { 1159 buck := tx.Bucket([]byte(RepoMetaBuck)) 1160 1161 repoMetaBlob := buck.Get([]byte(repo)) 1162 1163 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1164 if err != nil { 1165 return err 1166 } 1167 1168 referrersInfo := protoRepoMeta.Referrers[referredDigest.String()].List 1169 1170 for i := range referrersInfo { 1171 if !common.MatchesArtifactTypes(referrersInfo[i].ArtifactType, artifactTypes) { 1172 continue 1173 } 1174 1175 referrersInfoResult = append(referrersInfoResult, mTypes.ReferrerInfo{ 1176 Digest: referrersInfo[i].Digest, 1177 MediaType: referrersInfo[i].MediaType, 1178 ArtifactType: referrersInfo[i].ArtifactType, 1179 Size: int(referrersInfo[i].Size), 1180 Annotations: referrersInfo[i].Annotations, 1181 }) 1182 } 1183 1184 return nil 1185 }) 1186 1187 return referrersInfoResult, err 1188 } 1189 1190 func (bdw *BoltDB) UpdateStatsOnDownload(repo string, reference string) error { 1191 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 1192 repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) 1193 1194 repoMetaBlob := repoMetaBuck.Get([]byte(repo)) 1195 if len(repoMetaBlob) == 0 { 1196 return zerr.ErrRepoMetaNotFound 1197 } 1198 1199 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1200 if err != nil { 1201 return err 1202 } 1203 1204 manifestDigest := reference 1205 1206 if common.ReferenceIsTag(reference) { 1207 descriptor, found := protoRepoMeta.Tags[reference] 1208 1209 if !found { 1210 return zerr.ErrManifestMetaNotFound 1211 } 1212 1213 manifestDigest = descriptor.Digest 1214 } 1215 1216 manifestStatistics, ok := protoRepoMeta.Statistics[manifestDigest] 1217 if !ok { 1218 return zerr.ErrManifestMetaNotFound 1219 } 1220 1221 manifestStatistics.DownloadCount++ 1222 manifestStatistics.LastPullTimestamp = timestamppb.Now() 1223 protoRepoMeta.Statistics[manifestDigest] = manifestStatistics 1224 1225 return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) 1226 }) 1227 1228 return err 1229 } 1230 1231 func (bdw *BoltDB) UpdateSignaturesValidity(ctx context.Context, repo string, manifestDigest godigest.Digest) error { 1232 err := bdw.DB.Update(func(transaction *bbolt.Tx) error { 1233 imgTrustStore := bdw.ImageTrustStore() 1234 1235 if imgTrustStore == nil { 1236 return nil 1237 } 1238 1239 // get ManifestData of signed manifest 1240 imageMetaBuck := transaction.Bucket([]byte(ImageMetaBuck)) 1241 idBlob := imageMetaBuck.Get([]byte(manifestDigest)) 1242 1243 if len(idBlob) == 0 { 1244 // manifest meta not found, updating signatures with details about validity and author will not be performed 1245 return nil 1246 } 1247 1248 protoImageMeta := proto_go.ImageMeta{} 1249 1250 err := proto.Unmarshal(idBlob, &protoImageMeta) 1251 if err != nil { 1252 return err 1253 } 1254 1255 // update signatures with details about validity and author 1256 repoBuck := transaction.Bucket([]byte(RepoMetaBuck)) 1257 1258 repoMetaBlob := repoBuck.Get([]byte(repo)) 1259 if len(repoMetaBlob) == 0 { 1260 return zerr.ErrRepoMetaNotFound 1261 } 1262 1263 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1264 if err != nil { 1265 return err 1266 } 1267 1268 manifestSignatures := proto_go.ManifestSignatures{Map: map[string]*proto_go.SignaturesInfo{"": {}}} 1269 for sigType, sigs := range protoRepoMeta.Signatures[manifestDigest.String()].Map { 1270 if zcommon.IsContextDone(ctx) { 1271 return ctx.Err() 1272 } 1273 1274 signaturesInfo := []*proto_go.SignatureInfo{} 1275 1276 for _, sigInfo := range sigs.List { 1277 layersInfo := []*proto_go.LayersInfo{} 1278 1279 for _, layerInfo := range sigInfo.LayersInfo { 1280 author, date, isTrusted, _ := imgTrustStore.VerifySignature(sigType, layerInfo.LayerContent, 1281 layerInfo.SignatureKey, manifestDigest, mConvert.GetImageMeta(&protoImageMeta), repo) 1282 1283 if isTrusted { 1284 layerInfo.Signer = author 1285 } 1286 1287 if !date.IsZero() { 1288 layerInfo.Signer = author 1289 layerInfo.Date = timestamppb.New(date) 1290 } 1291 1292 layersInfo = append(layersInfo, layerInfo) 1293 } 1294 1295 signaturesInfo = append(signaturesInfo, &proto_go.SignatureInfo{ 1296 SignatureManifestDigest: sigInfo.SignatureManifestDigest, 1297 LayersInfo: layersInfo, 1298 }) 1299 } 1300 1301 manifestSignatures.Map[sigType] = &proto_go.SignaturesInfo{List: signaturesInfo} 1302 } 1303 1304 protoRepoMeta.Signatures[manifestDigest.String()] = &manifestSignatures 1305 1306 return setProtoRepoMeta(protoRepoMeta, repoBuck) 1307 }) 1308 1309 return err 1310 } 1311 1312 func (bdw *BoltDB) RemoveRepoReference(repo, reference string, manifestDigest godigest.Digest) error { 1313 err := bdw.DB.Update(func(tx *bbolt.Tx) error { 1314 repoMetaBuck := tx.Bucket([]byte(RepoMetaBuck)) 1315 imageMetaBuck := tx.Bucket([]byte(ImageMetaBuck)) 1316 repoBlobsBuck := tx.Bucket([]byte(RepoBlobsBuck)) 1317 repoLastUpdatedBuck := repoBlobsBuck.Bucket([]byte(RepoLastUpdatedBuck)) 1318 1319 protoRepoMeta, err := getProtoRepoMeta(repo, repoMetaBuck) 1320 if err != nil { 1321 if errors.Is(err, zerr.ErrRepoMetaNotFound) { 1322 return nil 1323 } 1324 1325 return err 1326 } 1327 1328 protoImageMeta, err := getProtoImageMeta(imageMetaBuck, manifestDigest.String()) 1329 if err != nil { 1330 if errors.Is(err, zerr.ErrImageMetaNotFound) { 1331 return nil 1332 } 1333 1334 return err 1335 } 1336 1337 // Remove Referrers 1338 if subject := mConvert.GetImageSubject(protoImageMeta); subject != nil { 1339 referredDigest := subject.Digest.String() 1340 refInfo := &proto_go.ReferrersInfo{} 1341 1342 if protoRepoMeta.Referrers[referredDigest] != nil { 1343 refInfo = protoRepoMeta.Referrers[referredDigest] 1344 } 1345 1346 referrers := refInfo.List 1347 1348 for i := range referrers { 1349 if referrers[i].Digest == manifestDigest.String() { 1350 referrers[i].Count -= 1 1351 1352 if referrers[i].Count == 0 || common.ReferenceIsDigest(reference) { 1353 referrers = append(referrers[:i], referrers[i+1:]...) 1354 } 1355 1356 break 1357 } 1358 } 1359 1360 refInfo.List = referrers 1361 1362 protoRepoMeta.Referrers[referredDigest] = refInfo 1363 } 1364 1365 if !common.ReferenceIsDigest(reference) { 1366 delete(protoRepoMeta.Tags, reference) 1367 } else { 1368 // remove all tags pointing to this digest 1369 for tag, desc := range protoRepoMeta.Tags { 1370 if desc.Digest == reference { 1371 delete(protoRepoMeta.Tags, tag) 1372 } 1373 } 1374 } 1375 1376 /* try to find at least one tag pointing to manifestDigest 1377 if not found then we can also remove everything related to this digest */ 1378 var foundTag bool 1379 for _, desc := range protoRepoMeta.Tags { 1380 if desc.Digest == manifestDigest.String() { 1381 foundTag = true 1382 } 1383 } 1384 1385 if !foundTag { 1386 delete(protoRepoMeta.Statistics, manifestDigest.String()) 1387 delete(protoRepoMeta.Signatures, manifestDigest.String()) 1388 delete(protoRepoMeta.Referrers, manifestDigest.String()) 1389 } 1390 1391 repoBlobsBytes := repoBlobsBuck.Get([]byte(protoRepoMeta.Name)) 1392 1393 repoBlobs, err := unmarshalProtoRepoBlobs(repo, repoBlobsBytes) 1394 if err != nil { 1395 return err 1396 } 1397 1398 err = setRepoLastUpdated(repo, time.Now(), repoLastUpdatedBuck) 1399 if err != nil { 1400 return err 1401 } 1402 1403 protoRepoMeta, repoBlobs = common.RemoveImageFromRepoMeta(protoRepoMeta, repoBlobs, reference) 1404 1405 repoBlobsBytes, err = proto.Marshal(repoBlobs) 1406 if err != nil { 1407 return err 1408 } 1409 1410 err = repoBlobsBuck.Put([]byte(protoRepoMeta.Name), repoBlobsBytes) 1411 if err != nil { 1412 return err 1413 } 1414 1415 return setProtoRepoMeta(protoRepoMeta, repoMetaBuck) 1416 }) 1417 1418 return err 1419 } 1420 1421 func (bdw *BoltDB) ImageTrustStore() mTypes.ImageTrustStore { 1422 return bdw.imgTrustStore 1423 } 1424 1425 func (bdw *BoltDB) SetImageTrustStore(imgTrustStore mTypes.ImageTrustStore) { 1426 bdw.imgTrustStore = imgTrustStore 1427 } 1428 1429 func (bdw *BoltDB) ToggleStarRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) { 1430 userAc, err := reqCtx.UserAcFromContext(ctx) 1431 if err != nil { 1432 return mTypes.NotChanged, err 1433 } 1434 1435 if userAc.IsAnonymous() || !userAc.Can(constants.ReadPermission, repo) { 1436 return mTypes.NotChanged, zerr.ErrUserDataNotAllowed 1437 } 1438 1439 userid := userAc.GetUsername() 1440 1441 var res mTypes.ToggleState 1442 1443 if err := bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:varnamelen 1444 var userData mTypes.UserData 1445 1446 err := bdw.getUserData(userid, tx, &userData) 1447 if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { 1448 return err 1449 } 1450 1451 isRepoStarred := zcommon.Contains(userData.StarredRepos, repo) 1452 1453 if isRepoStarred { 1454 res = mTypes.Removed 1455 userData.StarredRepos = zcommon.RemoveFrom(userData.StarredRepos, repo) 1456 } else { 1457 res = mTypes.Added 1458 userData.StarredRepos = append(userData.StarredRepos, repo) 1459 } 1460 1461 err = bdw.setUserData(userid, tx, userData) 1462 if err != nil { 1463 return err 1464 } 1465 1466 repoBuck := tx.Bucket([]byte(RepoMetaBuck)) 1467 1468 repoMetaBlob := repoBuck.Get([]byte(repo)) 1469 if len(repoMetaBlob) == 0 { 1470 return zerr.ErrRepoMetaNotFound 1471 } 1472 1473 protoRepoMeta, err := unmarshalProtoRepoMeta(repo, repoMetaBlob) 1474 if err != nil { 1475 return err 1476 } 1477 1478 switch res { 1479 case mTypes.Added: 1480 protoRepoMeta.Stars++ 1481 case mTypes.Removed: 1482 protoRepoMeta.Stars-- 1483 } 1484 1485 return setProtoRepoMeta(protoRepoMeta, repoBuck) 1486 }); err != nil { 1487 return mTypes.NotChanged, err 1488 } 1489 1490 return res, nil 1491 } 1492 1493 func (bdw *BoltDB) GetStarredRepos(ctx context.Context) ([]string, error) { 1494 userData, err := bdw.GetUserData(ctx) 1495 if errors.Is(err, zerr.ErrUserDataNotFound) || errors.Is(err, zerr.ErrUserDataNotAllowed) { 1496 return []string{}, nil 1497 } 1498 1499 return userData.StarredRepos, err 1500 } 1501 1502 func (bdw *BoltDB) ToggleBookmarkRepo(ctx context.Context, repo string) (mTypes.ToggleState, error) { 1503 userAc, err := reqCtx.UserAcFromContext(ctx) 1504 if err != nil { 1505 return mTypes.NotChanged, err 1506 } 1507 1508 if userAc.IsAnonymous() || !userAc.Can(constants.ReadPermission, repo) { 1509 return mTypes.NotChanged, zerr.ErrUserDataNotAllowed 1510 } 1511 1512 userid := userAc.GetUsername() 1513 1514 var res mTypes.ToggleState 1515 1516 if err := bdw.DB.Update(func(transaction *bbolt.Tx) error { //nolint:dupl 1517 var userData mTypes.UserData 1518 1519 err := bdw.getUserData(userid, transaction, &userData) 1520 if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { 1521 return err 1522 } 1523 1524 isRepoBookmarked := zcommon.Contains(userData.BookmarkedRepos, repo) 1525 1526 if isRepoBookmarked { 1527 res = mTypes.Removed 1528 userData.BookmarkedRepos = zcommon.RemoveFrom(userData.BookmarkedRepos, repo) 1529 } else { 1530 res = mTypes.Added 1531 userData.BookmarkedRepos = append(userData.BookmarkedRepos, repo) 1532 } 1533 1534 return bdw.setUserData(userid, transaction, userData) 1535 }); err != nil { 1536 return mTypes.NotChanged, err 1537 } 1538 1539 return res, nil 1540 } 1541 1542 func (bdw *BoltDB) GetBookmarkedRepos(ctx context.Context) ([]string, error) { 1543 userData, err := bdw.GetUserData(ctx) 1544 if errors.Is(err, zerr.ErrUserDataNotFound) || errors.Is(err, zerr.ErrUserDataNotAllowed) { 1545 return []string{}, nil 1546 } 1547 1548 return userData.BookmarkedRepos, err 1549 } 1550 1551 func (bdw *BoltDB) PatchDB() error { 1552 var DBVersion string 1553 1554 err := bdw.DB.View(func(tx *bbolt.Tx) error { 1555 versionBuck := tx.Bucket([]byte(VersionBucket)) 1556 DBVersion = string(versionBuck.Get([]byte(version.DBVersionKey))) 1557 1558 return nil 1559 }) 1560 if err != nil { 1561 return fmt.Errorf("patching the database failed, can't read db version %w", err) 1562 } 1563 1564 if version.GetVersionIndex(DBVersion) == -1 { 1565 return fmt.Errorf("DB has broken format, no version found %w", err) 1566 } 1567 1568 for patchIndex, patch := range bdw.Patches { 1569 if patchIndex < version.GetVersionIndex(DBVersion) { 1570 continue 1571 } 1572 1573 err := patch(bdw.DB) 1574 if err != nil { 1575 return err 1576 } 1577 } 1578 1579 return nil 1580 } 1581 1582 func getUserStars(ctx context.Context, transaction *bbolt.Tx) []string { 1583 userAc, err := reqCtx.UserAcFromContext(ctx) 1584 if err != nil { 1585 return []string{} 1586 } 1587 1588 var ( 1589 userData mTypes.UserData 1590 userid = userAc.GetUsername() 1591 userdb = transaction.Bucket([]byte(UserDataBucket)) 1592 ) 1593 1594 if userid == "" || userdb == nil { 1595 return []string{} 1596 } 1597 1598 mdata := userdb.Get([]byte(userid)) 1599 if mdata == nil { 1600 return []string{} 1601 } 1602 1603 if err := json.Unmarshal(mdata, &userData); err != nil { 1604 return []string{} 1605 } 1606 1607 return userData.StarredRepos 1608 } 1609 1610 func getUserBookmarks(ctx context.Context, transaction *bbolt.Tx) []string { 1611 userAc, err := reqCtx.UserAcFromContext(ctx) 1612 if err != nil { 1613 return []string{} 1614 } 1615 1616 var ( 1617 userData mTypes.UserData 1618 userid = userAc.GetUsername() 1619 userdb = transaction.Bucket([]byte(UserDataBucket)) 1620 ) 1621 1622 if userid == "" || userdb == nil { 1623 return []string{} 1624 } 1625 1626 mdata := userdb.Get([]byte(userid)) 1627 if mdata == nil { 1628 return []string{} 1629 } 1630 1631 if err := json.Unmarshal(mdata, &userData); err != nil { 1632 return []string{} 1633 } 1634 1635 return userData.BookmarkedRepos 1636 } 1637 1638 func (bdw *BoltDB) SetUserGroups(ctx context.Context, groups []string) error { 1639 userAc, err := reqCtx.UserAcFromContext(ctx) 1640 if err != nil { 1641 return err 1642 } 1643 1644 if userAc.IsAnonymous() { 1645 return zerr.ErrUserDataNotAllowed 1646 } 1647 1648 userid := userAc.GetUsername() 1649 1650 err = bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:varnamelen 1651 var userData mTypes.UserData 1652 1653 err := bdw.getUserData(userid, tx, &userData) 1654 if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { 1655 return err 1656 } 1657 1658 userData.Groups = append(userData.Groups, groups...) 1659 1660 err = bdw.setUserData(userid, tx, userData) 1661 1662 return err 1663 }) 1664 1665 return err 1666 } 1667 1668 func (bdw *BoltDB) GetUserGroups(ctx context.Context) ([]string, error) { 1669 userData, err := bdw.GetUserData(ctx) 1670 1671 return userData.Groups, err 1672 } 1673 1674 func (bdw *BoltDB) UpdateUserAPIKeyLastUsed(ctx context.Context, hashedKey string) error { 1675 userAc, err := reqCtx.UserAcFromContext(ctx) 1676 if err != nil { 1677 return err 1678 } 1679 1680 if userAc.IsAnonymous() { 1681 return zerr.ErrUserDataNotAllowed 1682 } 1683 1684 userid := userAc.GetUsername() 1685 1686 err = bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:varnamelen 1687 var userData mTypes.UserData 1688 1689 err := bdw.getUserData(userid, tx, &userData) 1690 if err != nil { 1691 return err 1692 } 1693 1694 apiKeyDetails := userData.APIKeys[hashedKey] 1695 apiKeyDetails.LastUsed = time.Now() 1696 1697 userData.APIKeys[hashedKey] = apiKeyDetails 1698 1699 err = bdw.setUserData(userid, tx, userData) 1700 1701 return err 1702 }) 1703 1704 return err 1705 } 1706 1707 func (bdw *BoltDB) IsAPIKeyExpired(ctx context.Context, hashedKey string) (bool, error) { 1708 userAc, err := reqCtx.UserAcFromContext(ctx) 1709 if err != nil { 1710 return false, err 1711 } 1712 1713 if userAc.IsAnonymous() { 1714 return false, zerr.ErrUserDataNotAllowed 1715 } 1716 1717 userid := userAc.GetUsername() 1718 1719 var isExpired bool 1720 1721 err = bdw.DB.Update(func(tx *bbolt.Tx) error { //nolint:varnamelen 1722 var userData mTypes.UserData 1723 1724 err := bdw.getUserData(userid, tx, &userData) 1725 if err != nil { 1726 return err 1727 } 1728 1729 apiKeyDetails := userData.APIKeys[hashedKey] 1730 if apiKeyDetails.IsExpired { 1731 isExpired = true 1732 1733 return nil 1734 } 1735 1736 // if expiresAt is not nil value 1737 if !apiKeyDetails.ExpirationDate.Equal(time.Time{}) && time.Now().After(apiKeyDetails.ExpirationDate) { 1738 isExpired = true 1739 apiKeyDetails.IsExpired = true 1740 } 1741 1742 userData.APIKeys[hashedKey] = apiKeyDetails 1743 1744 err = bdw.setUserData(userid, tx, userData) 1745 1746 return err 1747 }) 1748 1749 return isExpired, err 1750 } 1751 1752 func (bdw *BoltDB) GetUserAPIKeys(ctx context.Context) ([]mTypes.APIKeyDetails, error) { 1753 apiKeys := make([]mTypes.APIKeyDetails, 0) 1754 1755 userAc, err := reqCtx.UserAcFromContext(ctx) 1756 if err != nil { 1757 return nil, err 1758 } 1759 1760 if userAc.IsAnonymous() { 1761 return nil, zerr.ErrUserDataNotAllowed 1762 } 1763 1764 userid := userAc.GetUsername() 1765 1766 err = bdw.DB.Update(func(transaction *bbolt.Tx) error { 1767 var userData mTypes.UserData 1768 1769 err = bdw.getUserData(userid, transaction, &userData) 1770 if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { 1771 return err 1772 } 1773 1774 for hashedKey, apiKeyDetails := range userData.APIKeys { 1775 // if expiresAt is not nil value 1776 if !apiKeyDetails.ExpirationDate.Equal(time.Time{}) && time.Now().After(apiKeyDetails.ExpirationDate) { 1777 apiKeyDetails.IsExpired = true 1778 } 1779 1780 userData.APIKeys[hashedKey] = apiKeyDetails 1781 1782 err = bdw.setUserData(userid, transaction, userData) 1783 if err != nil { 1784 return err 1785 } 1786 1787 apiKeys = append(apiKeys, apiKeyDetails) 1788 } 1789 1790 return nil 1791 }) 1792 1793 return apiKeys, err 1794 } 1795 1796 func (bdw *BoltDB) AddUserAPIKey(ctx context.Context, hashedKey string, apiKeyDetails *mTypes.APIKeyDetails) error { 1797 userAc, err := reqCtx.UserAcFromContext(ctx) 1798 if err != nil { 1799 return err 1800 } 1801 1802 if userAc.IsAnonymous() { 1803 return zerr.ErrUserDataNotAllowed 1804 } 1805 1806 userid := userAc.GetUsername() 1807 1808 err = bdw.DB.Update(func(transaction *bbolt.Tx) error { 1809 var userData mTypes.UserData 1810 1811 apiKeysbuck := transaction.Bucket([]byte(UserAPIKeysBucket)) 1812 if apiKeysbuck == nil { 1813 return zerr.ErrBucketDoesNotExist 1814 } 1815 1816 err := apiKeysbuck.Put([]byte(hashedKey), []byte(userid)) 1817 if err != nil { 1818 return fmt.Errorf("metaDB: error while setting userData for identity %s %w", userid, err) 1819 } 1820 1821 err = bdw.getUserData(userid, transaction, &userData) 1822 if err != nil && !errors.Is(err, zerr.ErrUserDataNotFound) { 1823 return err 1824 } 1825 1826 if userData.APIKeys == nil { 1827 userData.APIKeys = make(map[string]mTypes.APIKeyDetails) 1828 } 1829 1830 userData.APIKeys[hashedKey] = *apiKeyDetails 1831 1832 err = bdw.setUserData(userid, transaction, userData) 1833 1834 return err 1835 }) 1836 1837 return err 1838 } 1839 1840 func (bdw *BoltDB) DeleteUserAPIKey(ctx context.Context, keyID string) error { 1841 userAc, err := reqCtx.UserAcFromContext(ctx) 1842 if err != nil { 1843 return err 1844 } 1845 1846 if userAc.IsAnonymous() { 1847 return zerr.ErrUserDataNotAllowed 1848 } 1849 1850 userid := userAc.GetUsername() 1851 1852 err = bdw.DB.Update(func(transaction *bbolt.Tx) error { 1853 var userData mTypes.UserData 1854 1855 apiKeysbuck := transaction.Bucket([]byte(UserAPIKeysBucket)) 1856 if apiKeysbuck == nil { 1857 return zerr.ErrBucketDoesNotExist 1858 } 1859 1860 err := bdw.getUserData(userid, transaction, &userData) 1861 if err != nil { 1862 return err 1863 } 1864 1865 for hash, apiKeyDetails := range userData.APIKeys { 1866 if apiKeyDetails.UUID == keyID { 1867 delete(userData.APIKeys, hash) 1868 1869 err := apiKeysbuck.Delete([]byte(hash)) 1870 if err != nil { 1871 return fmt.Errorf("userDB: error while deleting userAPIKey entry for hash %s %w", hash, err) 1872 } 1873 } 1874 } 1875 1876 return bdw.setUserData(userid, transaction, userData) 1877 }) 1878 1879 return err 1880 } 1881 1882 func (bdw *BoltDB) GetUserAPIKeyInfo(hashedKey string) (string, error) { 1883 var userid string 1884 err := bdw.DB.View(func(tx *bbolt.Tx) error { 1885 buck := tx.Bucket([]byte(UserAPIKeysBucket)) 1886 if buck == nil { 1887 return zerr.ErrBucketDoesNotExist 1888 } 1889 1890 uiBlob := buck.Get([]byte(hashedKey)) 1891 if len(uiBlob) == 0 { 1892 return zerr.ErrUserAPIKeyNotFound 1893 } 1894 1895 userid = string(uiBlob) 1896 1897 return nil 1898 }) 1899 1900 return userid, err 1901 } 1902 1903 func (bdw *BoltDB) GetUserData(ctx context.Context) (mTypes.UserData, error) { 1904 var userData mTypes.UserData 1905 1906 userAc, err := reqCtx.UserAcFromContext(ctx) 1907 if err != nil { 1908 return userData, err 1909 } 1910 1911 if userAc.IsAnonymous() { 1912 return userData, zerr.ErrUserDataNotAllowed 1913 } 1914 1915 userid := userAc.GetUsername() 1916 1917 err = bdw.DB.View(func(tx *bbolt.Tx) error { 1918 return bdw.getUserData(userid, tx, &userData) 1919 }) 1920 1921 return userData, err 1922 } 1923 1924 func (bdw *BoltDB) getUserData(userid string, transaction *bbolt.Tx, res *mTypes.UserData) error { 1925 buck := transaction.Bucket([]byte(UserDataBucket)) 1926 if buck == nil { 1927 return zerr.ErrBucketDoesNotExist 1928 } 1929 1930 upBlob := buck.Get([]byte(userid)) 1931 1932 if len(upBlob) == 0 { 1933 return zerr.ErrUserDataNotFound 1934 } 1935 1936 err := json.Unmarshal(upBlob, res) 1937 if err != nil { 1938 return err 1939 } 1940 1941 return nil 1942 } 1943 1944 func (bdw *BoltDB) SetUserData(ctx context.Context, userData mTypes.UserData) error { 1945 userAc, err := reqCtx.UserAcFromContext(ctx) 1946 if err != nil { 1947 return err 1948 } 1949 1950 if userAc.IsAnonymous() { 1951 return zerr.ErrUserDataNotAllowed 1952 } 1953 1954 userid := userAc.GetUsername() 1955 1956 err = bdw.DB.Update(func(tx *bbolt.Tx) error { 1957 return bdw.setUserData(userid, tx, userData) 1958 }) 1959 1960 return err 1961 } 1962 1963 func (bdw *BoltDB) setUserData(userid string, transaction *bbolt.Tx, userData mTypes.UserData) error { 1964 buck := transaction.Bucket([]byte(UserDataBucket)) 1965 if buck == nil { 1966 return zerr.ErrBucketDoesNotExist 1967 } 1968 1969 upBlob, err := json.Marshal(userData) 1970 if err != nil { 1971 return err 1972 } 1973 1974 err = buck.Put([]byte(userid), upBlob) 1975 if err != nil { 1976 return fmt.Errorf("metaDB: error while setting userData for identity %s %w", userid, err) 1977 } 1978 1979 return nil 1980 } 1981 1982 func (bdw *BoltDB) DeleteUserData(ctx context.Context) error { 1983 userAc, err := reqCtx.UserAcFromContext(ctx) 1984 if err != nil { 1985 return err 1986 } 1987 1988 if userAc.IsAnonymous() { 1989 return zerr.ErrUserDataNotAllowed 1990 } 1991 1992 userid := userAc.GetUsername() 1993 1994 err = bdw.DB.Update(func(tx *bbolt.Tx) error { 1995 buck := tx.Bucket([]byte(UserDataBucket)) 1996 if buck == nil { 1997 return zerr.ErrBucketDoesNotExist 1998 } 1999 2000 err := buck.Delete([]byte(userid)) 2001 if err != nil { 2002 return fmt.Errorf("metaDB: error while deleting userData for identity %s %w", userid, err) 2003 } 2004 2005 return nil 2006 }) 2007 2008 return err 2009 } 2010 2011 func (bdw *BoltDB) ResetDB() error { 2012 err := bdw.DB.Update(func(transaction *bbolt.Tx) error { 2013 err := resetBucket(transaction, RepoMetaBuck) 2014 if err != nil { 2015 return err 2016 } 2017 2018 err = resetBucket(transaction, ImageMetaBuck) 2019 if err != nil { 2020 return err 2021 } 2022 2023 err = resetBucket(transaction, RepoBlobsBuck) 2024 if err != nil { 2025 return err 2026 } 2027 2028 err = resetBucket(transaction, UserAPIKeysBucket) 2029 if err != nil { 2030 return err 2031 } 2032 2033 err = resetBucket(transaction, UserDataBucket) 2034 if err != nil { 2035 return err 2036 } 2037 2038 return nil 2039 }) 2040 2041 return err 2042 } 2043 2044 func resetBucket(transaction *bbolt.Tx, bucketName string) error { 2045 bucket := transaction.Bucket([]byte(bucketName)) 2046 if bucket == nil { 2047 return nil 2048 } 2049 2050 // we need to create the sub buckets if they exits, we'll presume the sub-buckets are not nested more than 1 layer 2051 subBuckets := [][]byte{} 2052 2053 err := bucket.ForEachBucket(func(bucketName []byte) error { 2054 subBuckets = append(subBuckets, bucketName) 2055 2056 return nil 2057 }) 2058 if err != nil { 2059 return err 2060 } 2061 2062 err = transaction.DeleteBucket([]byte(bucketName)) 2063 if err != nil { 2064 return err 2065 } 2066 2067 bucket, err = transaction.CreateBucketIfNotExists([]byte(bucketName)) 2068 if err != nil { 2069 return err 2070 } 2071 2072 for _, subBucket := range subBuckets { 2073 _, err := bucket.CreateBucketIfNotExists(subBucket) 2074 if err != nil { 2075 return err 2076 } 2077 } 2078 2079 return err 2080 }