zotregistry.dev/zot@v1.4.4-0.20240314164342-eec277e14d20/pkg/storage/s3/s3_test.go (about)

     1  package s3_test
     2  
     3  import (
     4  	"bytes"
     5  	"context"
     6  	_ "crypto/sha256"
     7  	"encoding/json"
     8  	"errors"
     9  	"fmt"
    10  	"io"
    11  	"os"
    12  	"path"
    13  	"strings"
    14  	"testing"
    15  	"time"
    16  
    17  	"github.com/docker/distribution/registry/storage/driver"
    18  	"github.com/docker/distribution/registry/storage/driver/factory"
    19  	_ "github.com/docker/distribution/registry/storage/driver/s3-aws"
    20  	guuid "github.com/gofrs/uuid"
    21  	godigest "github.com/opencontainers/go-digest"
    22  	ispec "github.com/opencontainers/image-spec/specs-go/v1"
    23  	"github.com/rs/zerolog"
    24  	. "github.com/smartystreets/goconvey/convey"
    25  	"gopkg.in/resty.v1"
    26  
    27  	zerr "zotregistry.dev/zot/errors"
    28  	"zotregistry.dev/zot/pkg/api"
    29  	"zotregistry.dev/zot/pkg/api/config"
    30  	"zotregistry.dev/zot/pkg/extensions/monitoring"
    31  	"zotregistry.dev/zot/pkg/log"
    32  	"zotregistry.dev/zot/pkg/scheduler"
    33  	"zotregistry.dev/zot/pkg/storage"
    34  	"zotregistry.dev/zot/pkg/storage/cache"
    35  	storageConstants "zotregistry.dev/zot/pkg/storage/constants"
    36  	"zotregistry.dev/zot/pkg/storage/s3"
    37  	storageTypes "zotregistry.dev/zot/pkg/storage/types"
    38  	. "zotregistry.dev/zot/pkg/test/image-utils"
    39  	"zotregistry.dev/zot/pkg/test/inject"
    40  	"zotregistry.dev/zot/pkg/test/mocks"
    41  	tskip "zotregistry.dev/zot/pkg/test/skip"
    42  )
    43  
    44  //nolint:gochecknoglobals
    45  var (
    46  	testImage      = "test"
    47  	fileWriterSize = 12
    48  	fileInfoSize   = 10
    49  	errorText      = "new s3 error"
    50  	errS3          = errors.New(errorText)
    51  	errCache       = errors.New("new cache error")
    52  	zotStorageTest = "zot-storage-test"
    53  	s3Region       = "us-east-2"
    54  )
    55  
    56  func cleanupStorage(store driver.StorageDriver, name string) {
    57  	_ = store.Delete(context.Background(), name)
    58  }
    59  
    60  func createMockStorage(rootDir string, cacheDir string, dedupe bool, store driver.StorageDriver,
    61  ) storageTypes.ImageStore {
    62  	log := log.Logger{Logger: zerolog.New(os.Stdout)}
    63  	metrics := monitoring.NewMetricsServer(true, log)
    64  
    65  	var cacheDriver cache.Cache
    66  
    67  	// from pkg/cli/server/root.go/applyDefaultValues, s3 magic
    68  	if _, err := os.Stat(path.Join(cacheDir,
    69  		storageConstants.BoltdbName+storageConstants.DBExtensionName)); dedupe || (!dedupe && err == nil) {
    70  		cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{
    71  			RootDir:     cacheDir,
    72  			Name:        "cache",
    73  			UseRelPaths: false,
    74  		}, log)
    75  	}
    76  
    77  	il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver)
    78  
    79  	return il
    80  }
    81  
    82  func createMockStorageWithMockCache(rootDir string, dedupe bool, store driver.StorageDriver,
    83  	cacheDriver cache.Cache,
    84  ) storageTypes.ImageStore {
    85  	log := log.Logger{Logger: zerolog.New(os.Stdout)}
    86  	metrics := monitoring.NewMetricsServer(false, log)
    87  
    88  	il := s3.NewImageStore(rootDir, "", dedupe, false, log, metrics, nil, store, cacheDriver)
    89  
    90  	return il
    91  }
    92  
    93  func createStoreDriver(rootDir string) driver.StorageDriver {
    94  	bucket := zotStorageTest
    95  	endpoint := os.Getenv("S3MOCK_ENDPOINT")
    96  	storageDriverParams := map[string]interface{}{
    97  		"rootDir":        rootDir,
    98  		"name":           "s3",
    99  		"region":         s3Region,
   100  		"bucket":         bucket,
   101  		"regionendpoint": endpoint,
   102  		"accesskey":      "minioadmin",
   103  		"secretkey":      "minioadmin",
   104  		"secure":         false,
   105  		"skipverify":     false,
   106  	}
   107  
   108  	storeName := fmt.Sprintf("%v", storageDriverParams["name"])
   109  
   110  	store, err := factory.Create(storeName, storageDriverParams)
   111  	if err != nil {
   112  		panic(err)
   113  	}
   114  
   115  	// create bucket if it doesn't exists
   116  	_, err = resty.R().Put("http://" + endpoint + "/" + bucket)
   117  	if err != nil {
   118  		panic(err)
   119  	}
   120  
   121  	return store
   122  }
   123  
   124  func createObjectsStore(rootDir string, cacheDir string, dedupe bool) (
   125  	driver.StorageDriver,
   126  	storageTypes.ImageStore,
   127  	error,
   128  ) {
   129  	store := createStoreDriver(rootDir)
   130  
   131  	log := log.Logger{Logger: zerolog.New(os.Stdout)}
   132  	metrics := monitoring.NewMetricsServer(false, log)
   133  
   134  	var cacheDriver cache.Cache
   135  
   136  	var err error
   137  
   138  	// from pkg/cli/server/root.go/applyDefaultValues, s3 magic
   139  	s3CacheDBPath := path.Join(cacheDir, storageConstants.BoltdbName+storageConstants.DBExtensionName)
   140  	if _, err = os.Stat(s3CacheDBPath); dedupe || (!dedupe && err == nil) {
   141  		cacheDriver, _ = storage.Create("boltdb", cache.BoltDBDriverParameters{
   142  			RootDir:     cacheDir,
   143  			Name:        "cache",
   144  			UseRelPaths: false,
   145  		}, log)
   146  	}
   147  
   148  	il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver)
   149  
   150  	return store, il, err
   151  }
   152  
   153  func createObjectsStoreDynamo(rootDir string, cacheDir string, dedupe bool, tableName string) (
   154  	driver.StorageDriver,
   155  	storageTypes.ImageStore,
   156  	error,
   157  ) {
   158  	store := createStoreDriver(rootDir)
   159  
   160  	log := log.Logger{Logger: zerolog.New(os.Stdout)}
   161  	metrics := monitoring.NewMetricsServer(false, log)
   162  
   163  	var cacheDriver cache.Cache
   164  
   165  	// from pkg/cli/server/root.go/applyDefaultValues, s3 magic
   166  	tableName = strings.ReplaceAll(tableName, "/", "")
   167  
   168  	cacheDriver, _ = storage.Create("dynamodb", cache.DynamoDBDriverParameters{
   169  		Endpoint:  os.Getenv("DYNAMODBMOCK_ENDPOINT"),
   170  		Region:    s3Region,
   171  		TableName: tableName,
   172  	}, log)
   173  
   174  	//nolint:errcheck
   175  	cacheDriverDynamo, _ := cacheDriver.(*cache.DynamoDBDriver)
   176  
   177  	err := cacheDriverDynamo.NewTable(tableName)
   178  	if err != nil {
   179  		panic(err)
   180  	}
   181  
   182  	il := s3.NewImageStore(rootDir, cacheDir, dedupe, false, log, metrics, nil, store, cacheDriver)
   183  
   184  	return store, il, err
   185  }
   186  
   187  func runAndGetScheduler() *scheduler.Scheduler {
   188  	logger := log.Logger{}
   189  	metrics := monitoring.NewMetricsServer(false, logger)
   190  	taskScheduler := scheduler.NewScheduler(config.New(), metrics, logger)
   191  	taskScheduler.RateLimit = 50 * time.Millisecond
   192  
   193  	taskScheduler.RunScheduler()
   194  
   195  	return taskScheduler
   196  }
   197  
   198  type FileInfoMock struct {
   199  	IsDirFn func() bool
   200  	SizeFn  func() int64
   201  	PathFn  func() string
   202  }
   203  
   204  func (f *FileInfoMock) Path() string {
   205  	if f != nil && f.PathFn != nil {
   206  		return f.PathFn()
   207  	}
   208  
   209  	return ""
   210  }
   211  
   212  func (f *FileInfoMock) Size() int64 {
   213  	if f != nil && f.SizeFn != nil {
   214  		return f.SizeFn()
   215  	}
   216  
   217  	return int64(fileInfoSize)
   218  }
   219  
   220  func (f *FileInfoMock) ModTime() time.Time {
   221  	return time.Now()
   222  }
   223  
   224  func (f *FileInfoMock) IsDir() bool {
   225  	if f != nil && f.IsDirFn != nil {
   226  		return f.IsDirFn()
   227  	}
   228  
   229  	return true
   230  }
   231  
   232  type FileWriterMock struct {
   233  	WriteFn  func([]byte) (int, error)
   234  	CancelFn func() error
   235  	CommitFn func() error
   236  	CloseFn  func() error
   237  }
   238  
   239  func (f *FileWriterMock) Size() int64 {
   240  	return int64(fileWriterSize)
   241  }
   242  
   243  func (f *FileWriterMock) Cancel() error {
   244  	if f != nil && f.CancelFn != nil {
   245  		return f.CancelFn()
   246  	}
   247  
   248  	return nil
   249  }
   250  
   251  func (f *FileWriterMock) Commit() error {
   252  	if f != nil && f.CommitFn != nil {
   253  		return f.CommitFn()
   254  	}
   255  
   256  	return nil
   257  }
   258  
   259  func (f *FileWriterMock) Write(p []byte) (int, error) {
   260  	if f != nil && f.WriteFn != nil {
   261  		return f.WriteFn(p)
   262  	}
   263  
   264  	return 10, nil
   265  }
   266  
   267  func (f *FileWriterMock) Close() error {
   268  	if f != nil && f.CloseFn != nil {
   269  		return f.CloseFn()
   270  	}
   271  
   272  	return nil
   273  }
   274  
   275  type StorageDriverMock struct {
   276  	NameFn       func() string
   277  	GetContentFn func(ctx context.Context, path string) ([]byte, error)
   278  	PutContentFn func(ctx context.Context, path string, content []byte) error
   279  	ReaderFn     func(ctx context.Context, path string, offset int64) (io.ReadCloser, error)
   280  	WriterFn     func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error)
   281  	StatFn       func(ctx context.Context, path string) (driver.FileInfo, error)
   282  	ListFn       func(ctx context.Context, path string) ([]string, error)
   283  	MoveFn       func(ctx context.Context, sourcePath, destPath string) error
   284  	DeleteFn     func(ctx context.Context, path string) error
   285  	WalkFn       func(ctx context.Context, path string, f driver.WalkFn) error
   286  }
   287  
   288  func (s *StorageDriverMock) Name() string {
   289  	if s != nil && s.NameFn != nil {
   290  		return s.NameFn()
   291  	}
   292  
   293  	return ""
   294  }
   295  
   296  func (s *StorageDriverMock) GetContent(ctx context.Context, path string) ([]byte, error) {
   297  	if s != nil && s.GetContentFn != nil {
   298  		return s.GetContentFn(ctx, path)
   299  	}
   300  
   301  	return []byte{}, nil
   302  }
   303  
   304  func (s *StorageDriverMock) PutContent(ctx context.Context, path string, content []byte) error {
   305  	if s != nil && s.PutContentFn != nil {
   306  		return s.PutContentFn(ctx, path, content)
   307  	}
   308  
   309  	return nil
   310  }
   311  
   312  func (s *StorageDriverMock) Reader(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
   313  	if s != nil && s.ReaderFn != nil {
   314  		return s.ReaderFn(ctx, path, offset)
   315  	}
   316  
   317  	return io.NopCloser(strings.NewReader("")), nil
   318  }
   319  
   320  func (s *StorageDriverMock) Writer(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
   321  	if s != nil && s.WriterFn != nil {
   322  		return s.WriterFn(ctx, path, isAppend)
   323  	}
   324  
   325  	return &FileWriterMock{}, nil
   326  }
   327  
   328  func (s *StorageDriverMock) Stat(ctx context.Context, path string) (driver.FileInfo, error) {
   329  	if s != nil && s.StatFn != nil {
   330  		return s.StatFn(ctx, path)
   331  	}
   332  
   333  	return &FileInfoMock{}, nil
   334  }
   335  
   336  func (s *StorageDriverMock) List(ctx context.Context, path string) ([]string, error) {
   337  	if s != nil && s.ListFn != nil {
   338  		return s.ListFn(ctx, path)
   339  	}
   340  
   341  	return []string{"a"}, nil
   342  }
   343  
   344  func (s *StorageDriverMock) Move(ctx context.Context, sourcePath, destPath string) error {
   345  	if s != nil && s.MoveFn != nil {
   346  		return s.MoveFn(ctx, sourcePath, destPath)
   347  	}
   348  
   349  	return nil
   350  }
   351  
   352  func (s *StorageDriverMock) Delete(ctx context.Context, path string) error {
   353  	if s != nil && s.DeleteFn != nil {
   354  		return s.DeleteFn(ctx, path)
   355  	}
   356  
   357  	return nil
   358  }
   359  
   360  func (s *StorageDriverMock) URLFor(ctx context.Context, path string, options map[string]interface{}) (string, error) {
   361  	return "", nil
   362  }
   363  
   364  func (s *StorageDriverMock) Walk(ctx context.Context, path string, f driver.WalkFn) error {
   365  	if s != nil && s.WalkFn != nil {
   366  		return s.WalkFn(ctx, path, f)
   367  	}
   368  
   369  	return nil
   370  }
   371  
   372  func TestStorageDriverStatFunction(t *testing.T) {
   373  	tskip.SkipS3(t)
   374  
   375  	uuid, err := guuid.NewV4()
   376  	if err != nil {
   377  		panic(err)
   378  	}
   379  
   380  	testDir := path.Join("/oci-repo-test", uuid.String())
   381  
   382  	storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
   383  	defer cleanupStorage(storeDriver, testDir)
   384  
   385  	/* There is an issue with storageDriver.Stat() that returns a storageDriver.FileInfo()
   386  	which falsely reports isDir() as true for paths under certain circumstances
   387  	for example:
   388  	1) create a file, eg: repo/testImageA/file
   389  	2) run storageDriver.Stat() on a partial path, eg: storageDriver.Stat("repo/testImage") - without 'A' char
   390  	3) the returned storageDriver.FileInfo will report that isDir() is true.
   391  	*/
   392  	Convey("Validate storageDriver.Stat() and isDir() functions with zot storage API", t, func(c C) {
   393  		repo1 := "repo/testimagea"
   394  		repo2 := "repo/testimage"
   395  
   396  		So(imgStore, ShouldNotBeNil)
   397  
   398  		err = imgStore.InitRepo(repo1)
   399  		So(err, ShouldBeNil)
   400  
   401  		isValid, err := imgStore.ValidateRepo(repo1)
   402  		So(err, ShouldBeNil)
   403  		So(isValid, ShouldBeTrue)
   404  
   405  		err = imgStore.InitRepo(repo2)
   406  		So(err, ShouldBeNil)
   407  
   408  		isValid, err = imgStore.ValidateRepo(repo2)
   409  		So(err, ShouldBeNil)
   410  		So(isValid, ShouldBeTrue)
   411  	})
   412  
   413  	Convey("Validate storageDriver.Stat() and isDir() functions with storageDriver API", t, func(c C) {
   414  		testFile := "/ab/cd/file"
   415  
   416  		shouldBeDirectoryPath1 := "/ab/cd"
   417  		shouldBeDirectoryPath2 := "/ab"
   418  
   419  		shouldNotBeDirectoryPath1 := "/ab/c"
   420  		shouldNotBeDirectoryPath2 := "/a"
   421  
   422  		err := storeDriver.PutContent(context.Background(), testFile, []byte("file contents"))
   423  		So(err, ShouldBeNil)
   424  
   425  		fileInfo, err := storeDriver.Stat(context.Background(), testFile)
   426  		So(err, ShouldBeNil)
   427  
   428  		So(fileInfo.IsDir(), ShouldBeFalse)
   429  
   430  		fileInfo, err = storeDriver.Stat(context.Background(), shouldBeDirectoryPath1)
   431  		So(err, ShouldBeNil)
   432  		So(fileInfo.IsDir(), ShouldBeTrue)
   433  
   434  		fileInfo, err = storeDriver.Stat(context.Background(), shouldBeDirectoryPath2)
   435  		So(err, ShouldBeNil)
   436  		So(fileInfo.IsDir(), ShouldBeTrue)
   437  
   438  		fileInfo, err = storeDriver.Stat(context.Background(), shouldNotBeDirectoryPath1)
   439  		// err should actually be storageDriver.PathNotFoundError but it's nil
   440  		So(err, ShouldBeNil)
   441  		// should be false instead
   442  		So(fileInfo.IsDir(), ShouldBeTrue)
   443  
   444  		fileInfo, err = storeDriver.Stat(context.Background(), shouldNotBeDirectoryPath2)
   445  		// err should actually be storageDriver.PathNotFoundError but it's nils
   446  		So(err, ShouldBeNil)
   447  		// should be false instead
   448  		So(fileInfo.IsDir(), ShouldBeTrue)
   449  	})
   450  }
   451  
   452  func TestGetOCIReferrers(t *testing.T) {
   453  	tskip.SkipS3(t)
   454  
   455  	repo := "zot-test"
   456  
   457  	uuid, err := guuid.NewV4()
   458  	if err != nil {
   459  		panic(err)
   460  	}
   461  
   462  	tdir := t.TempDir()
   463  	testDir := path.Join("/oci-repo-test", uuid.String())
   464  
   465  	_, imgStore, _ := createObjectsStore(testDir, tdir, true)
   466  
   467  	Convey("Upload test image", t, func(c C) {
   468  		image := CreateDefaultImage()
   469  
   470  		manifest := image.Manifest
   471  		cfg := image.Config
   472  		layers := image.Layers
   473  
   474  		for _, content := range layers {
   475  			upload, err := imgStore.NewBlobUpload(repo)
   476  			So(err, ShouldBeNil)
   477  			So(upload, ShouldNotBeEmpty)
   478  
   479  			buf := bytes.NewBuffer(content)
   480  			buflen := buf.Len()
   481  			digest := godigest.FromBytes(content)
   482  
   483  			blob, err := imgStore.PutBlobChunkStreamed(repo, upload, buf)
   484  			So(err, ShouldBeNil)
   485  			So(blob, ShouldEqual, buflen)
   486  			blobDigest1 := digest
   487  			So(blobDigest1, ShouldNotBeEmpty)
   488  
   489  			err = imgStore.FinishBlobUpload(repo, upload, buf, digest)
   490  			So(err, ShouldBeNil)
   491  			So(blob, ShouldEqual, buflen)
   492  		}
   493  
   494  		// upload config blob
   495  		cblob, err := json.Marshal(cfg)
   496  		So(err, ShouldBeNil)
   497  
   498  		buf := bytes.NewBuffer(cblob)
   499  		buflen := buf.Len()
   500  		digest := godigest.FromBytes(cblob)
   501  
   502  		_, clen, err := imgStore.FullBlobUpload(repo, buf, digest)
   503  		So(err, ShouldBeNil)
   504  		So(clen, ShouldEqual, buflen)
   505  
   506  		// upload manifest
   507  		mblob, err := json.Marshal(manifest)
   508  		So(err, ShouldBeNil)
   509  
   510  		mbuf := bytes.NewBuffer(mblob)
   511  		mbuflen := mbuf.Len()
   512  		mdigest := godigest.FromBytes(mblob)
   513  
   514  		d, _, err := imgStore.PutImageManifest(repo, "1.0", ispec.MediaTypeImageManifest, mbuf.Bytes())
   515  		So(d, ShouldEqual, mdigest)
   516  		So(err, ShouldBeNil)
   517  
   518  		body := []byte("this is an artifact")
   519  		digest = godigest.FromBytes(body)
   520  		buf = bytes.NewBuffer(body)
   521  		buflen = buf.Len()
   522  
   523  		_, n, err := imgStore.FullBlobUpload(repo, buf, digest)
   524  		So(err, ShouldBeNil)
   525  		So(n, ShouldEqual, buflen)
   526  
   527  		Convey("Get OCI Referrers - application/vnd.oci.image.manifest.v1+json", func(c C) {
   528  			artifactType := "application/vnd.example.icecream.v1"
   529  			// push artifact config blob
   530  			configBody := []byte("{}")
   531  			configDigest := godigest.FromBytes(configBody)
   532  			configBuf := bytes.NewBuffer(configBody)
   533  			configBufLen := configBuf.Len()
   534  
   535  			_, n, err := imgStore.FullBlobUpload(repo, configBuf, configDigest)
   536  			So(err, ShouldBeNil)
   537  			So(n, ShouldEqual, configBufLen)
   538  
   539  			artifactManifest := ispec.Manifest{
   540  				MediaType: ispec.MediaTypeImageManifest,
   541  				Config: ispec.Descriptor{
   542  					MediaType: artifactType,
   543  					Size:      int64(configBufLen),
   544  					Digest:    configDigest,
   545  				},
   546  				Layers: []ispec.Descriptor{
   547  					{
   548  						MediaType: "application/octet-stream",
   549  						Size:      int64(buflen),
   550  						Digest:    digest,
   551  					},
   552  				},
   553  				Subject: &ispec.Descriptor{
   554  					MediaType: ispec.MediaTypeImageManifest,
   555  					Size:      int64(mbuflen),
   556  					Digest:    mdigest,
   557  				},
   558  			}
   559  
   560  			artifactManifest.SchemaVersion = 2
   561  
   562  			manBuf, err := json.Marshal(artifactManifest)
   563  			So(err, ShouldBeNil)
   564  
   565  			manBufLen := len(manBuf)
   566  			manDigest := godigest.FromBytes(manBuf)
   567  
   568  			_, _, err = imgStore.PutImageManifest(repo, manDigest.Encoded(), ispec.MediaTypeImageManifest, manBuf)
   569  			So(err, ShouldBeNil)
   570  
   571  			index, err := imgStore.GetReferrers(repo, mdigest, []string{artifactType})
   572  			So(err, ShouldBeNil)
   573  			So(index, ShouldNotBeEmpty)
   574  			So(index.Manifests[0].ArtifactType, ShouldEqual, artifactType)
   575  			So(index.Manifests[0].MediaType, ShouldEqual, ispec.MediaTypeImageManifest)
   576  			So(index.Manifests[0].Size, ShouldEqual, manBufLen)
   577  			So(index.Manifests[0].Digest, ShouldEqual, manDigest)
   578  		})
   579  	})
   580  }
   581  
   582  func TestNegativeCasesObjectsStorage(t *testing.T) {
   583  	tskip.SkipS3(t)
   584  
   585  	uuid, err := guuid.NewV4()
   586  	if err != nil {
   587  		panic(err)
   588  	}
   589  
   590  	tdir := t.TempDir()
   591  	testDir := path.Join("/oci-repo-test", uuid.String())
   592  
   593  	Convey("With dedupe", t, func(c C) {
   594  		storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, true)
   595  		defer cleanupStorage(storeDriver, testDir)
   596  
   597  		Convey("Invalid repo name", func(c C) {
   598  			// Validate repo should fail if repo name does not match spec
   599  			_, err := imgStore.ValidateRepo(".")
   600  			So(err, ShouldNotBeNil)
   601  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   602  
   603  			_, err = imgStore.ValidateRepo("..")
   604  			So(err, ShouldNotBeNil)
   605  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   606  
   607  			_, err = imgStore.ValidateRepo("_test-dir")
   608  			So(err, ShouldNotBeNil)
   609  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   610  
   611  			_, err = imgStore.ValidateRepo(".test-dir")
   612  			So(err, ShouldNotBeNil)
   613  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   614  
   615  			_, err = imgStore.ValidateRepo("-test-dir")
   616  			So(err, ShouldNotBeNil)
   617  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   618  
   619  			// Init repo should fail if repo name does not match spec
   620  			err = imgStore.InitRepo(".")
   621  			So(err, ShouldNotBeNil)
   622  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   623  
   624  			err = imgStore.InitRepo("..")
   625  			So(err, ShouldNotBeNil)
   626  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   627  
   628  			err = imgStore.InitRepo("_test-dir")
   629  			So(err, ShouldNotBeNil)
   630  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   631  
   632  			err = imgStore.InitRepo(".test-dir")
   633  			So(err, ShouldNotBeNil)
   634  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   635  
   636  			err = imgStore.InitRepo("-test-dir")
   637  			So(err, ShouldNotBeNil)
   638  			So(errors.Is(err, zerr.ErrInvalidRepositoryName), ShouldBeTrue)
   639  		})
   640  
   641  		Convey("Invalid validate repo", func(c C) {
   642  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   643  			objects, err := storeDriver.List(context.Background(), path.Join(imgStore.RootDir(), testImage))
   644  			So(err, ShouldBeNil)
   645  
   646  			for _, object := range objects {
   647  				t.Logf("Removing object: %s", object)
   648  				err := storeDriver.Delete(context.Background(), object)
   649  				So(err, ShouldBeNil)
   650  			}
   651  
   652  			_, err = imgStore.ValidateRepo(testImage)
   653  			So(err, ShouldNotBeNil)
   654  			_, err = imgStore.GetRepositories()
   655  			So(err, ShouldBeNil)
   656  		})
   657  
   658  		Convey("Unable to create subpath cache db", func(c C) {
   659  			bucket := zotStorageTest
   660  			endpoint := os.Getenv("S3MOCK_ENDPOINT")
   661  
   662  			storageDriverParams := config.GlobalStorageConfig{
   663  				StorageConfig: config.StorageConfig{
   664  					Dedupe:        true,
   665  					RootDirectory: t.TempDir(),
   666  					RemoteCache:   false,
   667  				},
   668  				SubPaths: map[string]config.StorageConfig{
   669  					"/a": {
   670  						Dedupe:        true,
   671  						RootDirectory: t.TempDir(),
   672  						StorageDriver: map[string]interface{}{
   673  							"rootDir":        "/a",
   674  							"name":           "s3",
   675  							"region":         s3Region,
   676  							"bucket":         bucket,
   677  							"regionendpoint": endpoint,
   678  							"accesskey":      "minioadmin",
   679  							"secretkey":      "minioadmin",
   680  							"secure":         false,
   681  							"skipverify":     false,
   682  						},
   683  						RemoteCache: false,
   684  					},
   685  				},
   686  			}
   687  			conf := config.New()
   688  			conf.Storage = storageDriverParams
   689  			controller := api.NewController(conf)
   690  			So(controller, ShouldNotBeNil)
   691  
   692  			err = controller.InitImageStore()
   693  			So(err, ShouldBeNil)
   694  		})
   695  
   696  		Convey("Invalid get image tags", func(c C) {
   697  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   698  
   699  			So(storeDriver.Move(context.Background(), path.Join(testDir, testImage, "index.json"),
   700  				path.Join(testDir, testImage, "blobs")), ShouldBeNil)
   701  			ok, _ := imgStore.ValidateRepo(testImage)
   702  			So(ok, ShouldBeFalse)
   703  			_, err = imgStore.GetImageTags(testImage)
   704  			So(err, ShouldNotBeNil)
   705  
   706  			So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
   707  
   708  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   709  			So(storeDriver.PutContent(context.Background(), path.Join(testDir, testImage, "index.json"), []byte{}), ShouldBeNil)
   710  			_, err = imgStore.GetImageTags(testImage)
   711  			So(err, ShouldNotBeNil)
   712  		})
   713  	})
   714  
   715  	Convey("Without dedupe", t, func(c C) {
   716  		tdir := t.TempDir()
   717  		storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false)
   718  		defer cleanupStorage(storeDriver, testDir)
   719  
   720  		Convey("Invalid get image manifest", func(c C) {
   721  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   722  			So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage, "index.json")), ShouldBeNil)
   723  			_, _, _, err = imgStore.GetImageManifest(testImage, "")
   724  			So(err, ShouldNotBeNil)
   725  			So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
   726  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   727  			So(storeDriver.PutContent(context.Background(), path.Join(testDir, testImage, "index.json"), []byte{}), ShouldBeNil)
   728  			_, _, _, err = imgStore.GetImageManifest(testImage, "")
   729  			So(err, ShouldNotBeNil)
   730  		})
   731  
   732  		Convey("Invalid validate repo", func(c C) {
   733  			So(imgStore, ShouldNotBeNil)
   734  
   735  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   736  			So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage, "index.json")), ShouldBeNil)
   737  			_, err = imgStore.ValidateRepo(testImage)
   738  			So(err, ShouldNotBeNil)
   739  			So(storeDriver.Delete(context.Background(), path.Join(testDir, testImage)), ShouldBeNil)
   740  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   741  			So(storeDriver.Move(context.Background(), path.Join(testDir, testImage, "index.json"),
   742  				path.Join(testDir, testImage, "_index.json")), ShouldBeNil)
   743  			ok, err := imgStore.ValidateRepo(testImage)
   744  			So(err, ShouldBeNil)
   745  			So(ok, ShouldBeFalse)
   746  		})
   747  
   748  		Convey("Invalid finish blob upload", func(c C) {
   749  			So(imgStore, ShouldNotBeNil)
   750  
   751  			So(imgStore.InitRepo(testImage), ShouldBeNil)
   752  			upload, err := imgStore.NewBlobUpload(testImage)
   753  			So(err, ShouldBeNil)
   754  			So(upload, ShouldNotBeEmpty)
   755  
   756  			content := []byte("test-data1")
   757  			buf := bytes.NewBuffer(content)
   758  			buflen := buf.Len()
   759  			digest := godigest.FromBytes(content)
   760  
   761  			blob, err := imgStore.PutBlobChunk(testImage, upload, 0, int64(buflen), buf)
   762  			So(err, ShouldBeNil)
   763  			So(blob, ShouldEqual, buflen)
   764  
   765  			src := imgStore.BlobUploadPath(testImage, upload)
   766  			stwr, err := storeDriver.Writer(context.Background(), src, true)
   767  			So(err, ShouldBeNil)
   768  
   769  			_, err = stwr.Write([]byte("another-chunk-of-data"))
   770  			So(err, ShouldBeNil)
   771  
   772  			err = stwr.Close()
   773  			So(err, ShouldBeNil)
   774  
   775  			err = imgStore.FinishBlobUpload(testImage, upload, buf, digest)
   776  			So(err, ShouldNotBeNil)
   777  		})
   778  
   779  		Convey("Test storage driver errors", func(c C) {
   780  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   781  				ListFn: func(ctx context.Context, path string) ([]string, error) {
   782  					return []string{testImage}, errS3
   783  				},
   784  				MoveFn: func(ctx context.Context, sourcePath, destPath string) error {
   785  					return errS3
   786  				},
   787  				GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
   788  					return []byte{}, errS3
   789  				},
   790  				PutContentFn: func(ctx context.Context, path string, content []byte) error {
   791  					return errS3
   792  				},
   793  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
   794  					return &FileWriterMock{}, errS3
   795  				},
   796  				ReaderFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
   797  					return io.NopCloser(strings.NewReader("")), errS3
   798  				},
   799  				WalkFn: func(ctx context.Context, path string, f driver.WalkFn) error {
   800  					return errS3
   801  				},
   802  				StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
   803  					return &FileInfoMock{}, errS3
   804  				},
   805  				DeleteFn: func(ctx context.Context, path string) error {
   806  					return errS3
   807  				},
   808  			})
   809  			So(imgStore, ShouldNotBeNil)
   810  
   811  			So(imgStore.InitRepo(testImage), ShouldNotBeNil)
   812  			_, err := imgStore.ValidateRepo(testImage)
   813  			So(err, ShouldNotBeNil)
   814  
   815  			upload, err := imgStore.NewBlobUpload(testImage)
   816  			So(err, ShouldNotBeNil)
   817  
   818  			content := []byte("test-data1")
   819  			buf := bytes.NewBuffer(content)
   820  			buflen := buf.Len()
   821  			digest := godigest.FromBytes(content)
   822  
   823  			_, err = imgStore.PutBlobChunk(testImage, upload, 0, int64(buflen), buf)
   824  			So(err, ShouldNotBeNil)
   825  
   826  			err = imgStore.FinishBlobUpload(testImage, upload, buf, digest)
   827  			So(err, ShouldNotBeNil)
   828  
   829  			err = imgStore.DeleteBlob(testImage, digest)
   830  			So(err, ShouldNotBeNil)
   831  
   832  			err = imgStore.DeleteBlobUpload(testImage, upload)
   833  			So(err, ShouldNotBeNil)
   834  
   835  			err = imgStore.DeleteImageManifest(testImage, "1.0", false)
   836  			So(err, ShouldNotBeNil)
   837  
   838  			_, _, err = imgStore.PutImageManifest(testImage, "1.0", "application/json", []byte{})
   839  			So(err, ShouldNotBeNil)
   840  
   841  			_, err = imgStore.PutBlobChunkStreamed(testImage, upload, bytes.NewBufferString(testImage))
   842  			So(err, ShouldNotBeNil)
   843  
   844  			_, _, err = imgStore.FullBlobUpload(testImage, bytes.NewBuffer([]byte{}), "inexistent")
   845  			So(err, ShouldNotBeNil)
   846  
   847  			_, _, err = imgStore.CheckBlob(testImage, digest)
   848  			So(err, ShouldNotBeNil)
   849  
   850  			_, _, _, err = imgStore.StatBlob(testImage, digest)
   851  			So(err, ShouldNotBeNil)
   852  		})
   853  
   854  		Convey("Test ValidateRepo", func(c C) {
   855  			tdir := t.TempDir()
   856  
   857  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   858  				ListFn: func(ctx context.Context, path string) ([]string, error) {
   859  					return []string{testImage, testImage}, errS3
   860  				},
   861  			})
   862  
   863  			_, err := imgStore.ValidateRepo(testImage)
   864  			So(err, ShouldNotBeNil)
   865  
   866  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   867  				ListFn: func(ctx context.Context, path string) ([]string, error) {
   868  					return []string{testImage, testImage}, nil
   869  				},
   870  				StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
   871  					return nil, errS3
   872  				},
   873  			})
   874  
   875  			_, err = imgStore.ValidateRepo(testImage)
   876  			So(err, ShouldNotBeNil)
   877  		})
   878  
   879  		Convey("Test ValidateRepo2", func(c C) {
   880  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   881  				ListFn: func(ctx context.Context, path string) ([]string, error) {
   882  					return []string{"test/test/oci-layout", "test/test/index.json"}, nil
   883  				},
   884  				StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
   885  					return &FileInfoMock{}, nil
   886  				},
   887  			})
   888  			_, err := imgStore.ValidateRepo(testImage)
   889  			So(err, ShouldNotBeNil)
   890  		})
   891  
   892  		Convey("Test ValidateRepo3", func(c C) {
   893  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   894  				ListFn: func(ctx context.Context, path string) ([]string, error) {
   895  					return []string{"test/test/oci-layout", "test/test/index.json"}, nil
   896  				},
   897  				StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
   898  					return &FileInfoMock{}, nil
   899  				},
   900  				GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
   901  					return []byte{}, errS3
   902  				},
   903  			})
   904  			_, err := imgStore.ValidateRepo(testImage)
   905  			So(err, ShouldNotBeNil)
   906  		})
   907  
   908  		Convey("Test ValidateRepo4", func(c C) {
   909  			ociLayout := []byte(`{"imageLayoutVersion": "9.9.9"}`)
   910  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   911  				ListFn: func(ctx context.Context, path string) ([]string, error) {
   912  					return []string{"test/test/oci-layout", "test/test/index.json"}, nil
   913  				},
   914  				StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
   915  					return &FileInfoMock{}, nil
   916  				},
   917  				GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
   918  					return ociLayout, nil
   919  				},
   920  			})
   921  			_, err := imgStore.ValidateRepo(testImage)
   922  			So(err, ShouldNotBeNil)
   923  		})
   924  
   925  		Convey("Test GetRepositories", func(c C) {
   926  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   927  				WalkFn: func(ctx context.Context, path string, f driver.WalkFn) error {
   928  					return f(new(FileInfoMock))
   929  				},
   930  			})
   931  			repos, err := imgStore.GetRepositories()
   932  			So(repos, ShouldBeEmpty)
   933  			So(err, ShouldBeNil)
   934  		})
   935  
   936  		Convey("Test DeleteImageManifest", func(c C) {
   937  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   938  				GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
   939  					return []byte{}, errS3
   940  				},
   941  			})
   942  			err := imgStore.DeleteImageManifest(testImage, "1.0", false)
   943  			So(err, ShouldNotBeNil)
   944  		})
   945  
   946  		Convey("Test GetIndexContent", func(c C) {
   947  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   948  				GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
   949  					return []byte{}, driver.PathNotFoundError{}
   950  				},
   951  			})
   952  			_, err := imgStore.GetIndexContent(testImage)
   953  			So(err, ShouldNotBeNil)
   954  		})
   955  
   956  		Convey("Test DeleteImageManifest2", func(c C) {
   957  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{})
   958  			err := imgStore.DeleteImageManifest(testImage, "1.0", false)
   959  			So(err, ShouldNotBeNil)
   960  		})
   961  
   962  		Convey("Test NewBlobUpload", func(c C) {
   963  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   964  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
   965  					return nil, errS3
   966  				},
   967  			})
   968  			_, err := imgStore.NewBlobUpload(testImage)
   969  			So(err, ShouldNotBeNil)
   970  		})
   971  
   972  		Convey("Test GetBlobUpload", func(c C) {
   973  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   974  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
   975  					return nil, errS3
   976  				},
   977  			})
   978  			_, err := imgStore.GetBlobUpload(testImage, "uuid")
   979  			So(err, ShouldNotBeNil)
   980  		})
   981  
   982  		Convey("Test BlobUploadInfo", func(c C) {
   983  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   984  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
   985  					return nil, errS3
   986  				},
   987  			})
   988  			_, err := imgStore.BlobUploadInfo(testImage, "uuid")
   989  			So(err, ShouldNotBeNil)
   990  		})
   991  
   992  		Convey("Test PutBlobChunkStreamed", func(c C) {
   993  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
   994  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
   995  					return &FileWriterMock{}, errS3
   996  				},
   997  			})
   998  			_, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", io.NopCloser(strings.NewReader("")))
   999  			So(err, ShouldNotBeNil)
  1000  		})
  1001  
  1002  		Convey("Test PutBlobChunkStreamed2", func(c C) {
  1003  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1004  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1005  					return &FileWriterMock{WriteFn: func(b []byte) (int, error) {
  1006  						return 0, errS3
  1007  					}}, errS3
  1008  				},
  1009  			})
  1010  			_, err := imgStore.PutBlobChunkStreamed(testImage, "uuid", io.NopCloser(strings.NewReader("")))
  1011  			So(err, ShouldNotBeNil)
  1012  		})
  1013  
  1014  		Convey("Test PutBlobChunk", func(c C) {
  1015  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1016  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1017  					return &FileWriterMock{}, errS3
  1018  				},
  1019  			})
  1020  			_, err := imgStore.PutBlobChunk(testImage, "uuid", 0, 100, io.NopCloser(strings.NewReader("")))
  1021  			So(err, ShouldNotBeNil)
  1022  		})
  1023  
  1024  		Convey("Test PutBlobChunk2", func(c C) {
  1025  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1026  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1027  					return &FileWriterMock{
  1028  						WriteFn: func(b []byte) (int, error) {
  1029  							return 0, errS3
  1030  						},
  1031  						CancelFn: func() error {
  1032  							return errS3
  1033  						},
  1034  					}, nil
  1035  				},
  1036  			})
  1037  			_, err := imgStore.PutBlobChunk(testImage, "uuid", 0, 100, io.NopCloser(strings.NewReader("")))
  1038  			So(err, ShouldNotBeNil)
  1039  		})
  1040  
  1041  		Convey("Test PutBlobChunk3", func(c C) {
  1042  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1043  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1044  					return &FileWriterMock{
  1045  						WriteFn: func(b []byte) (int, error) {
  1046  							return 0, errS3
  1047  						},
  1048  					}, errS3
  1049  				},
  1050  			})
  1051  			_, err := imgStore.PutBlobChunk(testImage, "uuid", 12, 100, io.NopCloser(strings.NewReader("")))
  1052  			So(err, ShouldNotBeNil)
  1053  		})
  1054  
  1055  		Convey("Test PutBlobChunk4", func(c C) {
  1056  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1057  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1058  					return &FileWriterMock{}, driver.PathNotFoundError{}
  1059  				},
  1060  			})
  1061  			_, err := imgStore.PutBlobChunk(testImage, "uuid", 0, 100, io.NopCloser(strings.NewReader("")))
  1062  			So(err, ShouldNotBeNil)
  1063  		})
  1064  
  1065  		Convey("Test FinishBlobUpload", func(c C) {
  1066  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1067  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1068  					return &FileWriterMock{
  1069  						CommitFn: func() error {
  1070  							return errS3
  1071  						},
  1072  					}, nil
  1073  				},
  1074  			})
  1075  			d := godigest.FromBytes([]byte("test"))
  1076  			err := imgStore.FinishBlobUpload(testImage, "uuid", io.NopCloser(strings.NewReader("")), d)
  1077  			So(err, ShouldNotBeNil)
  1078  		})
  1079  
  1080  		Convey("Test FinishBlobUpload2", func(c C) {
  1081  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1082  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1083  					return &FileWriterMock{
  1084  						CloseFn: func() error {
  1085  							return errS3
  1086  						},
  1087  					}, nil
  1088  				},
  1089  			})
  1090  			d := godigest.FromBytes([]byte("test"))
  1091  			err := imgStore.FinishBlobUpload(testImage, "uuid", io.NopCloser(strings.NewReader("")), d)
  1092  			So(err, ShouldNotBeNil)
  1093  		})
  1094  
  1095  		Convey("Test FinishBlobUpload3", func(c C) {
  1096  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1097  				ReaderFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
  1098  					return nil, errS3
  1099  				},
  1100  			})
  1101  			d := godigest.FromBytes([]byte("test"))
  1102  			err := imgStore.FinishBlobUpload(testImage, "uuid", io.NopCloser(strings.NewReader("")), d)
  1103  			So(err, ShouldNotBeNil)
  1104  		})
  1105  
  1106  		Convey("Test FinishBlobUpload4", func(c C) {
  1107  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1108  				MoveFn: func(ctx context.Context, sourcePath, destPath string) error {
  1109  					return errS3
  1110  				},
  1111  			})
  1112  			d := godigest.FromBytes([]byte(""))
  1113  			err := imgStore.FinishBlobUpload(testImage, "uuid", io.NopCloser(strings.NewReader("")), d)
  1114  			So(err, ShouldNotBeNil)
  1115  		})
  1116  
  1117  		Convey("Test FullBlobUpload", func(c C) {
  1118  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1119  				WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  1120  					return &FileWriterMock{}, errS3
  1121  				},
  1122  			})
  1123  			d := godigest.FromBytes([]byte(""))
  1124  			_, _, err := imgStore.FullBlobUpload(testImage, io.NopCloser(strings.NewReader("")), d)
  1125  			So(err, ShouldNotBeNil)
  1126  		})
  1127  
  1128  		Convey("Test FullBlobUpload2", func(c C) {
  1129  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{})
  1130  			d := godigest.FromBytes([]byte(" "))
  1131  			_, _, err := imgStore.FullBlobUpload(testImage, io.NopCloser(strings.NewReader("")), d)
  1132  			So(err, ShouldNotBeNil)
  1133  		})
  1134  
  1135  		Convey("Test FullBlobUpload3", func(c C) {
  1136  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1137  				MoveFn: func(ctx context.Context, sourcePath, destPath string) error {
  1138  					return errS3
  1139  				},
  1140  			})
  1141  			d := godigest.FromBytes([]byte(""))
  1142  			_, _, err := imgStore.FullBlobUpload(testImage, io.NopCloser(strings.NewReader("")), d)
  1143  			So(err, ShouldNotBeNil)
  1144  		})
  1145  
  1146  		Convey("Test GetBlob", func(c C) {
  1147  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1148  				ReaderFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
  1149  					return io.NopCloser(strings.NewReader("")), errS3
  1150  				},
  1151  			})
  1152  			d := godigest.FromBytes([]byte(""))
  1153  			_, _, err := imgStore.GetBlob(testImage, d, "")
  1154  			So(err, ShouldNotBeNil)
  1155  		})
  1156  
  1157  		Convey("Test GetBlobContent", func(c C) {
  1158  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1159  				GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
  1160  					return []byte{}, errS3
  1161  				},
  1162  			})
  1163  
  1164  			d := godigest.FromBytes([]byte(""))
  1165  			_, err := imgStore.GetBlobContent(testImage, d)
  1166  			So(err, ShouldNotBeNil)
  1167  		})
  1168  
  1169  		Convey("Test DeleteBlob", func(c C) {
  1170  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{
  1171  				DeleteFn: func(ctx context.Context, path string) error {
  1172  					return errS3
  1173  				},
  1174  			})
  1175  			d := godigest.FromBytes([]byte(""))
  1176  			err := imgStore.DeleteBlob(testImage, d)
  1177  			So(err, ShouldNotBeNil)
  1178  		})
  1179  
  1180  		Convey("Test GetReferrers", func(c C) {
  1181  			imgStore = createMockStorage(testDir, tdir, false, &StorageDriverMock{})
  1182  			d := godigest.FromBytes([]byte(""))
  1183  			_, err := imgStore.GetReferrers(testImage, d, []string{"application/image"})
  1184  			So(err, ShouldNotBeNil)
  1185  			So(err, ShouldEqual, zerr.ErrRepoBadVersion)
  1186  		})
  1187  	})
  1188  }
  1189  
  1190  func TestS3Dedupe(t *testing.T) {
  1191  	tskip.SkipS3(t)
  1192  	tskip.SkipDynamo(t)
  1193  	Convey("Dedupe", t, func(c C) {
  1194  		uuid, err := guuid.NewV4()
  1195  		if err != nil {
  1196  			panic(err)
  1197  		}
  1198  
  1199  		testDir := path.Join("/oci-repo-test", uuid.String())
  1200  
  1201  		tdir := t.TempDir()
  1202  
  1203  		storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, true)
  1204  		defer cleanupStorage(storeDriver, testDir)
  1205  
  1206  		// manifest1
  1207  		upload, err := imgStore.NewBlobUpload("dedupe1")
  1208  		So(err, ShouldBeNil)
  1209  		So(upload, ShouldNotBeEmpty)
  1210  
  1211  		content := []byte("test-data3")
  1212  		buf := bytes.NewBuffer(content)
  1213  		buflen := buf.Len()
  1214  		digest := godigest.FromBytes(content)
  1215  		blob, err := imgStore.PutBlobChunkStreamed("dedupe1", upload, buf)
  1216  		So(err, ShouldBeNil)
  1217  		So(blob, ShouldEqual, buflen)
  1218  		blobDigest1 := digest
  1219  		So(blobDigest1, ShouldNotBeEmpty)
  1220  
  1221  		err = imgStore.FinishBlobUpload("dedupe1", upload, buf, digest)
  1222  		So(err, ShouldBeNil)
  1223  		So(blob, ShouldEqual, buflen)
  1224  
  1225  		ok, checkBlobSize1, err := imgStore.CheckBlob("dedupe1", digest)
  1226  		So(ok, ShouldBeTrue)
  1227  		So(checkBlobSize1, ShouldBeGreaterThan, 0)
  1228  		So(err, ShouldBeNil)
  1229  
  1230  		ok, checkBlobSize1, _, err = imgStore.StatBlob("dedupe1", digest)
  1231  		So(ok, ShouldBeTrue)
  1232  		So(checkBlobSize1, ShouldBeGreaterThan, 0)
  1233  		So(err, ShouldBeNil)
  1234  
  1235  		blobReadCloser, getBlobSize1, err := imgStore.GetBlob("dedupe1", digest,
  1236  			"application/vnd.oci.image.layer.v1.tar+gzip")
  1237  		So(getBlobSize1, ShouldBeGreaterThan, 0)
  1238  		So(err, ShouldBeNil)
  1239  		err = blobReadCloser.Close()
  1240  		So(err, ShouldBeNil)
  1241  
  1242  		cblob, cdigest := GetRandomImageConfig()
  1243  		_, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest)
  1244  		So(err, ShouldBeNil)
  1245  		So(clen, ShouldEqual, len(cblob))
  1246  		hasBlob, _, err := imgStore.CheckBlob("dedupe1", cdigest)
  1247  		So(err, ShouldBeNil)
  1248  		So(hasBlob, ShouldEqual, true)
  1249  
  1250  		manifest := ispec.Manifest{
  1251  			Config: ispec.Descriptor{
  1252  				MediaType: "application/vnd.oci.image.config.v1+json",
  1253  				Digest:    cdigest,
  1254  				Size:      int64(len(cblob)),
  1255  			},
  1256  			Layers: []ispec.Descriptor{
  1257  				{
  1258  					MediaType: "application/vnd.oci.image.layer.v1.tar",
  1259  					Digest:    digest,
  1260  					Size:      int64(buflen),
  1261  				},
  1262  			},
  1263  		}
  1264  
  1265  		manifest.SchemaVersion = 2
  1266  		manifestBuf, err := json.Marshal(manifest)
  1267  		So(err, ShouldBeNil)
  1268  		manifestDigest := godigest.FromBytes(manifestBuf)
  1269  		_, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(),
  1270  			ispec.MediaTypeImageManifest, manifestBuf)
  1271  		So(err, ShouldBeNil)
  1272  
  1273  		_, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String())
  1274  		So(err, ShouldBeNil)
  1275  
  1276  		// manifest2
  1277  		upload, err = imgStore.NewBlobUpload("dedupe2")
  1278  		So(err, ShouldBeNil)
  1279  		So(upload, ShouldNotBeEmpty)
  1280  
  1281  		content = []byte("test-data3")
  1282  		buf = bytes.NewBuffer(content)
  1283  		buflen = buf.Len()
  1284  		digest = godigest.FromBytes(content)
  1285  
  1286  		blob, err = imgStore.PutBlobChunkStreamed("dedupe2", upload, buf)
  1287  		So(err, ShouldBeNil)
  1288  		So(blob, ShouldEqual, buflen)
  1289  		blobDigest2 := digest
  1290  		So(blobDigest2, ShouldNotBeEmpty)
  1291  
  1292  		err = imgStore.FinishBlobUpload("dedupe2", upload, buf, digest)
  1293  		So(err, ShouldBeNil)
  1294  		So(blob, ShouldEqual, buflen)
  1295  
  1296  		_, checkBlobSize2, err := imgStore.CheckBlob("dedupe2", digest)
  1297  		So(err, ShouldBeNil)
  1298  		So(checkBlobSize2, ShouldBeGreaterThan, 0)
  1299  
  1300  		blobReadCloser, getBlobSize2, err := imgStore.GetBlob("dedupe2", digest,
  1301  			"application/vnd.oci.image.layer.v1.tar+gzip")
  1302  		So(err, ShouldBeNil)
  1303  		So(getBlobSize2, ShouldBeGreaterThan, 0)
  1304  		So(checkBlobSize1, ShouldEqual, checkBlobSize2)
  1305  		So(getBlobSize1, ShouldEqual, getBlobSize2)
  1306  		err = blobReadCloser.Close()
  1307  		So(err, ShouldBeNil)
  1308  
  1309  		blobContent, err := imgStore.GetBlobContent("dedupe2", digest)
  1310  		So(err, ShouldBeNil)
  1311  		So(len(blobContent), ShouldBeGreaterThan, 0)
  1312  		So(checkBlobSize1, ShouldEqual, len(blobContent))
  1313  		So(getBlobSize1, ShouldEqual, len(blobContent))
  1314  		err = blobReadCloser.Close()
  1315  		So(err, ShouldBeNil)
  1316  
  1317  		cblob, cdigest = GetRandomImageConfig()
  1318  		_, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest)
  1319  		So(err, ShouldBeNil)
  1320  		So(clen, ShouldEqual, len(cblob))
  1321  		hasBlob, _, err = imgStore.CheckBlob("dedupe2", cdigest)
  1322  		So(err, ShouldBeNil)
  1323  		So(hasBlob, ShouldEqual, true)
  1324  
  1325  		manifest = ispec.Manifest{
  1326  			Config: ispec.Descriptor{
  1327  				MediaType: "application/vnd.oci.image.config.v1+json",
  1328  				Digest:    cdigest,
  1329  				Size:      int64(len(cblob)),
  1330  			},
  1331  			Layers: []ispec.Descriptor{
  1332  				{
  1333  					MediaType: "application/vnd.oci.image.layer.v1.tar",
  1334  					Digest:    digest,
  1335  					Size:      int64(buflen),
  1336  				},
  1337  			},
  1338  		}
  1339  		manifest.SchemaVersion = 2
  1340  		manifestBuf, err = json.Marshal(manifest)
  1341  		So(err, ShouldBeNil)
  1342  		digest = godigest.FromBytes(manifestBuf)
  1343  		_, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest,
  1344  			manifestBuf)
  1345  		So(err, ShouldBeNil)
  1346  
  1347  		_, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String())
  1348  		So(err, ShouldBeNil)
  1349  
  1350  		fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1351  			blobDigest1.Encoded()))
  1352  		So(err, ShouldBeNil)
  1353  
  1354  		fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1355  			blobDigest2.Encoded()))
  1356  		So(err, ShouldBeNil)
  1357  
  1358  		// original blob should have the real content of blob
  1359  		So(fi1.Size(), ShouldNotEqual, fi2.Size())
  1360  		So(fi1.Size(), ShouldBeGreaterThan, 0)
  1361  		// deduped blob should be of size 0
  1362  		So(fi2.Size(), ShouldEqual, 0)
  1363  
  1364  		Convey("delete blobs from storage/cache should work when dedupe is true", func() {
  1365  			So(blobDigest1, ShouldEqual, blobDigest2)
  1366  
  1367  			// to not trigger BlobInUse err, delete manifest first
  1368  			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
  1369  			So(err, ShouldBeNil)
  1370  
  1371  			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
  1372  			So(err, ShouldBeNil)
  1373  
  1374  			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
  1375  			So(err, ShouldBeNil)
  1376  
  1377  			err = imgStore.DeleteBlob("dedupe2", blobDigest2)
  1378  			So(err, ShouldBeNil)
  1379  		})
  1380  
  1381  		Convey("Check that delete blobs moves the real content to the next contenders", func() {
  1382  			// to not trigger BlobInUse err, delete manifest first
  1383  			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
  1384  			So(err, ShouldBeNil)
  1385  
  1386  			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
  1387  			So(err, ShouldBeNil)
  1388  
  1389  			// if we delete blob1, the content should be moved to blob2
  1390  			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
  1391  			So(err, ShouldBeNil)
  1392  
  1393  			_, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1394  				blobDigest1.Encoded()))
  1395  			So(err, ShouldNotBeNil)
  1396  
  1397  			fi2, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1398  				blobDigest2.Encoded()))
  1399  			So(err, ShouldBeNil)
  1400  
  1401  			So(fi2.Size(), ShouldBeGreaterThan, 0)
  1402  			// the second blob should now be equal to the deleted blob.
  1403  			So(fi2.Size(), ShouldEqual, fi1.Size())
  1404  
  1405  			err = imgStore.DeleteBlob("dedupe2", blobDigest2)
  1406  			So(err, ShouldBeNil)
  1407  
  1408  			_, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1409  				blobDigest2.Encoded()))
  1410  			So(err, ShouldNotBeNil)
  1411  		})
  1412  
  1413  		Convey("Check backward compatibility - switch dedupe to false", func() {
  1414  			/* copy cache to the new storage with dedupe false (doing this because we
  1415  			already have a cache object holding the lock on cache db file) */
  1416  			input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
  1417  			So(err, ShouldBeNil)
  1418  
  1419  			tdir = t.TempDir()
  1420  
  1421  			err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600)
  1422  			So(err, ShouldBeNil)
  1423  
  1424  			storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false)
  1425  			defer cleanupStorage(storeDriver, testDir)
  1426  
  1427  			// manifest3 without dedupe
  1428  			upload, err = imgStore.NewBlobUpload("dedupe3")
  1429  			So(err, ShouldBeNil)
  1430  			So(upload, ShouldNotBeEmpty)
  1431  
  1432  			content = []byte("test-data3")
  1433  			buf = bytes.NewBuffer(content)
  1434  			buflen = buf.Len()
  1435  			digest = godigest.FromBytes(content)
  1436  
  1437  			blob, err = imgStore.PutBlobChunkStreamed("dedupe3", upload, buf)
  1438  			So(err, ShouldBeNil)
  1439  			So(blob, ShouldEqual, buflen)
  1440  			blobDigest2 := digest
  1441  			So(blobDigest2, ShouldNotBeEmpty)
  1442  
  1443  			err = imgStore.FinishBlobUpload("dedupe3", upload, buf, digest)
  1444  			So(err, ShouldBeNil)
  1445  			So(blob, ShouldEqual, buflen)
  1446  
  1447  			_, _, err = imgStore.CheckBlob("dedupe3", digest)
  1448  			So(err, ShouldBeNil)
  1449  
  1450  			// check that we retrieve the real dedupe2/blob (which is deduped earlier - 0 size) when switching to dedupe false
  1451  			blobReadCloser, getBlobSize2, err = imgStore.GetBlob("dedupe2", digest,
  1452  				"application/vnd.oci.image.layer.v1.tar+gzip")
  1453  			So(err, ShouldBeNil)
  1454  			So(getBlobSize1, ShouldEqual, getBlobSize2)
  1455  			err = blobReadCloser.Close()
  1456  			So(err, ShouldBeNil)
  1457  
  1458  			_, checkBlobSize2, err := imgStore.CheckBlob("dedupe2", digest)
  1459  			So(err, ShouldBeNil)
  1460  			So(checkBlobSize2, ShouldBeGreaterThan, 0)
  1461  			So(checkBlobSize2, ShouldEqual, getBlobSize2)
  1462  
  1463  			_, getBlobSize3, err := imgStore.GetBlob("dedupe3", digest, "application/vnd.oci.image.layer.v1.tar+gzip")
  1464  			So(err, ShouldBeNil)
  1465  			So(getBlobSize1, ShouldEqual, getBlobSize3)
  1466  
  1467  			blobContent, err := imgStore.GetBlobContent("dedupe3", digest)
  1468  			So(err, ShouldBeNil)
  1469  			So(getBlobSize1, ShouldEqual, len(blobContent))
  1470  
  1471  			_, checkBlobSize3, err := imgStore.CheckBlob("dedupe3", digest)
  1472  			So(err, ShouldBeNil)
  1473  			So(checkBlobSize3, ShouldBeGreaterThan, 0)
  1474  			So(checkBlobSize3, ShouldEqual, getBlobSize3)
  1475  
  1476  			cblob, cdigest = GetRandomImageConfig()
  1477  			_, clen, err = imgStore.FullBlobUpload("dedupe3", bytes.NewReader(cblob), cdigest)
  1478  			So(err, ShouldBeNil)
  1479  			So(clen, ShouldEqual, len(cblob))
  1480  			hasBlob, _, err = imgStore.CheckBlob("dedupe3", cdigest)
  1481  			So(err, ShouldBeNil)
  1482  			So(hasBlob, ShouldEqual, true)
  1483  
  1484  			manifest = ispec.Manifest{
  1485  				Config: ispec.Descriptor{
  1486  					MediaType: "application/vnd.oci.image.config.v1+json",
  1487  					Digest:    cdigest,
  1488  					Size:      int64(len(cblob)),
  1489  				},
  1490  				Layers: []ispec.Descriptor{
  1491  					{
  1492  						MediaType: "application/vnd.oci.image.layer.v1.tar",
  1493  						Digest:    digest,
  1494  						Size:      int64(buflen),
  1495  					},
  1496  				},
  1497  			}
  1498  			manifest.SchemaVersion = 2
  1499  			manifestBuf, err = json.Marshal(manifest)
  1500  			So(err, ShouldBeNil)
  1501  			digest = godigest.FromBytes(manifestBuf)
  1502  			_, _, err = imgStore.PutImageManifest("dedupe3", "1.0", ispec.MediaTypeImageManifest,
  1503  				manifestBuf)
  1504  			So(err, ShouldBeNil)
  1505  
  1506  			_, _, _, err = imgStore.GetImageManifest("dedupe3", digest.String())
  1507  			So(err, ShouldBeNil)
  1508  
  1509  			fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1510  				blobDigest1.Encoded()))
  1511  			So(err, ShouldBeNil)
  1512  
  1513  			fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1514  				blobDigest1.Encoded()))
  1515  			So(err, ShouldBeNil)
  1516  			So(fi2.Size(), ShouldEqual, 0)
  1517  
  1518  			fi3, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe3", "blobs", "sha256",
  1519  				blobDigest2.Encoded()))
  1520  			So(err, ShouldBeNil)
  1521  
  1522  			// the new blob with dedupe false should be equal with the origin blob from dedupe1
  1523  			So(fi1.Size(), ShouldEqual, fi3.Size())
  1524  
  1525  			Convey("delete blobs from storage/cache should work when dedupe is false", func() {
  1526  				So(blobDigest1, ShouldEqual, blobDigest2)
  1527  				// to not trigger BlobInUse err, delete manifest first
  1528  				err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
  1529  				So(err, ShouldBeNil)
  1530  
  1531  				err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
  1532  				So(err, ShouldBeNil)
  1533  
  1534  				err = imgStore.DeleteImageManifest("dedupe3", "1.0", false)
  1535  				So(err, ShouldBeNil)
  1536  
  1537  				err = imgStore.DeleteBlob("dedupe1", blobDigest1)
  1538  				So(err, ShouldBeNil)
  1539  
  1540  				err = imgStore.DeleteBlob("dedupe2", blobDigest2)
  1541  				So(err, ShouldBeNil)
  1542  
  1543  				err = imgStore.DeleteBlob("dedupe3", blobDigest2)
  1544  				So(err, ShouldBeNil)
  1545  			})
  1546  
  1547  			Convey("rebuild s3 dedupe index from true to false", func() { //nolint: dupl
  1548  				taskScheduler := runAndGetScheduler()
  1549  				defer taskScheduler.Shutdown()
  1550  
  1551  				storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), false)
  1552  				defer cleanupStorage(storeDriver, testDir)
  1553  
  1554  				// rebuild with dedupe false, should have all blobs with content
  1555  				imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  1556  				// wait until rebuild finishes
  1557  
  1558  				time.Sleep(10 * time.Second)
  1559  
  1560  				taskScheduler.Shutdown()
  1561  
  1562  				fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1563  					blobDigest1.Encoded()))
  1564  				So(fi1.Size(), ShouldBeGreaterThan, 0)
  1565  				So(err, ShouldBeNil)
  1566  
  1567  				fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1568  					blobDigest2.Encoded()))
  1569  				So(err, ShouldBeNil)
  1570  				So(fi2.Size(), ShouldEqual, fi1.Size())
  1571  
  1572  				blobContent, err := imgStore.GetBlobContent("dedupe2", blobDigest2)
  1573  				So(err, ShouldBeNil)
  1574  				So(len(blobContent), ShouldEqual, fi1.Size())
  1575  
  1576  				Convey("rebuild s3 dedupe index from false to true", func() {
  1577  					taskScheduler := runAndGetScheduler()
  1578  					defer taskScheduler.Shutdown()
  1579  
  1580  					storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  1581  					defer cleanupStorage(storeDriver, testDir)
  1582  
  1583  					// rebuild with dedupe false, should have all blobs with content
  1584  					imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  1585  					// wait until rebuild finishes
  1586  
  1587  					time.Sleep(10 * time.Second)
  1588  
  1589  					taskScheduler.Shutdown()
  1590  
  1591  					fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1592  						blobDigest2.Encoded()))
  1593  					So(err, ShouldBeNil)
  1594  					So(fi2.Size(), ShouldEqual, 0)
  1595  
  1596  					blobContent, err := imgStore.GetBlobContent("dedupe2", blobDigest2)
  1597  					So(err, ShouldBeNil)
  1598  					So(len(blobContent), ShouldBeGreaterThan, 0)
  1599  				})
  1600  			})
  1601  		})
  1602  	})
  1603  
  1604  	Convey("Dedupe with dynamodb", t, func(c C) {
  1605  		uuid, err := guuid.NewV4()
  1606  		if err != nil {
  1607  			panic(err)
  1608  		}
  1609  
  1610  		testDir := path.Join("/oci-repo-test", uuid.String())
  1611  
  1612  		tdir := t.TempDir()
  1613  
  1614  		storeDriver, imgStore, _ := createObjectsStoreDynamo(testDir, tdir, true, tdir)
  1615  		defer cleanupStorage(storeDriver, testDir)
  1616  
  1617  		// manifest1
  1618  		upload, err := imgStore.NewBlobUpload("dedupe1")
  1619  		So(err, ShouldBeNil)
  1620  		So(upload, ShouldNotBeEmpty)
  1621  
  1622  		content := []byte("test-data3")
  1623  		buf := bytes.NewBuffer(content)
  1624  		buflen := buf.Len()
  1625  		digest := godigest.FromBytes(content)
  1626  		blob, err := imgStore.PutBlobChunkStreamed("dedupe1", upload, buf)
  1627  		So(err, ShouldBeNil)
  1628  		So(blob, ShouldEqual, buflen)
  1629  		blobDigest1 := digest
  1630  		So(blobDigest1, ShouldNotBeEmpty)
  1631  
  1632  		err = imgStore.FinishBlobUpload("dedupe1", upload, buf, digest)
  1633  		So(err, ShouldBeNil)
  1634  		So(blob, ShouldEqual, buflen)
  1635  
  1636  		_, checkBlobSize1, err := imgStore.CheckBlob("dedupe1", digest)
  1637  		So(checkBlobSize1, ShouldBeGreaterThan, 0)
  1638  		So(err, ShouldBeNil)
  1639  
  1640  		blobReadCloser, getBlobSize1, err := imgStore.GetBlob("dedupe1", digest,
  1641  			"application/vnd.oci.image.layer.v1.tar+gzip")
  1642  		So(getBlobSize1, ShouldBeGreaterThan, 0)
  1643  		So(err, ShouldBeNil)
  1644  		err = blobReadCloser.Close()
  1645  		So(err, ShouldBeNil)
  1646  
  1647  		cblob, cdigest := GetRandomImageConfig()
  1648  		_, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest)
  1649  		So(err, ShouldBeNil)
  1650  		So(clen, ShouldEqual, len(cblob))
  1651  		hasBlob, _, err := imgStore.CheckBlob("dedupe1", cdigest)
  1652  		So(err, ShouldBeNil)
  1653  		So(hasBlob, ShouldEqual, true)
  1654  
  1655  		manifest := ispec.Manifest{
  1656  			Config: ispec.Descriptor{
  1657  				MediaType: "application/vnd.oci.image.config.v1+json",
  1658  				Digest:    cdigest,
  1659  				Size:      int64(len(cblob)),
  1660  			},
  1661  			Layers: []ispec.Descriptor{
  1662  				{
  1663  					MediaType: "application/vnd.oci.image.layer.v1.tar",
  1664  					Digest:    digest,
  1665  					Size:      int64(buflen),
  1666  				},
  1667  			},
  1668  		}
  1669  
  1670  		manifest.SchemaVersion = 2
  1671  		manifestBuf, err := json.Marshal(manifest)
  1672  		So(err, ShouldBeNil)
  1673  		manifestDigest := godigest.FromBytes(manifestBuf)
  1674  		_, _, err = imgStore.PutImageManifest("dedupe1", manifestDigest.String(),
  1675  			ispec.MediaTypeImageManifest, manifestBuf)
  1676  		So(err, ShouldBeNil)
  1677  
  1678  		_, _, _, err = imgStore.GetImageManifest("dedupe1", manifestDigest.String())
  1679  		So(err, ShouldBeNil)
  1680  
  1681  		// manifest2
  1682  		upload, err = imgStore.NewBlobUpload("dedupe2")
  1683  		So(err, ShouldBeNil)
  1684  		So(upload, ShouldNotBeEmpty)
  1685  
  1686  		content = []byte("test-data3")
  1687  		buf = bytes.NewBuffer(content)
  1688  		buflen = buf.Len()
  1689  		digest = godigest.FromBytes(content)
  1690  
  1691  		blob, err = imgStore.PutBlobChunkStreamed("dedupe2", upload, buf)
  1692  		So(err, ShouldBeNil)
  1693  		So(blob, ShouldEqual, buflen)
  1694  		blobDigest2 := digest
  1695  		So(blobDigest2, ShouldNotBeEmpty)
  1696  
  1697  		err = imgStore.FinishBlobUpload("dedupe2", upload, buf, digest)
  1698  		So(err, ShouldBeNil)
  1699  		So(blob, ShouldEqual, buflen)
  1700  
  1701  		_, checkBlobSize2, err := imgStore.CheckBlob("dedupe2", digest)
  1702  		So(err, ShouldBeNil)
  1703  		So(checkBlobSize2, ShouldBeGreaterThan, 0)
  1704  
  1705  		blobReadCloser, getBlobSize2, err := imgStore.GetBlob("dedupe2", digest,
  1706  			"application/vnd.oci.image.layer.v1.tar+gzip")
  1707  		So(err, ShouldBeNil)
  1708  		So(getBlobSize2, ShouldBeGreaterThan, 0)
  1709  		So(checkBlobSize1, ShouldEqual, checkBlobSize2)
  1710  		So(getBlobSize1, ShouldEqual, getBlobSize2)
  1711  		err = blobReadCloser.Close()
  1712  		So(err, ShouldBeNil)
  1713  
  1714  		cblob, cdigest = GetRandomImageConfig()
  1715  		_, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest)
  1716  		So(err, ShouldBeNil)
  1717  		So(clen, ShouldEqual, len(cblob))
  1718  		hasBlob, _, err = imgStore.CheckBlob("dedupe2", cdigest)
  1719  		So(err, ShouldBeNil)
  1720  		So(hasBlob, ShouldEqual, true)
  1721  
  1722  		manifest = ispec.Manifest{
  1723  			Config: ispec.Descriptor{
  1724  				MediaType: "application/vnd.oci.image.config.v1+json",
  1725  				Digest:    cdigest,
  1726  				Size:      int64(len(cblob)),
  1727  			},
  1728  			Layers: []ispec.Descriptor{
  1729  				{
  1730  					MediaType: "application/vnd.oci.image.layer.v1.tar",
  1731  					Digest:    digest,
  1732  					Size:      int64(buflen),
  1733  				},
  1734  			},
  1735  		}
  1736  		manifest.SchemaVersion = 2
  1737  		manifestBuf, err = json.Marshal(manifest)
  1738  		So(err, ShouldBeNil)
  1739  		digest = godigest.FromBytes(manifestBuf)
  1740  		_, _, err = imgStore.PutImageManifest("dedupe2", "1.0", ispec.MediaTypeImageManifest,
  1741  			manifestBuf)
  1742  		So(err, ShouldBeNil)
  1743  
  1744  		_, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String())
  1745  		So(err, ShouldBeNil)
  1746  
  1747  		fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1748  			blobDigest1.Encoded()))
  1749  		So(err, ShouldBeNil)
  1750  
  1751  		fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1752  			blobDigest2.Encoded()))
  1753  		So(err, ShouldBeNil)
  1754  
  1755  		// original blob should have the real content of blob
  1756  		So(fi1.Size(), ShouldNotEqual, fi2.Size())
  1757  		So(fi1.Size(), ShouldBeGreaterThan, 0)
  1758  		// deduped blob should be of size 0
  1759  		So(fi2.Size(), ShouldEqual, 0)
  1760  
  1761  		Convey("delete blobs from storage/cache should work when dedupe is true", func() {
  1762  			So(blobDigest1, ShouldEqual, blobDigest2)
  1763  
  1764  			// to not trigger BlobInUse err, delete manifest first
  1765  			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
  1766  			So(err, ShouldBeNil)
  1767  
  1768  			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
  1769  			So(err, ShouldBeNil)
  1770  
  1771  			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
  1772  			So(err, ShouldBeNil)
  1773  
  1774  			err = imgStore.DeleteBlob("dedupe2", blobDigest2)
  1775  			So(err, ShouldBeNil)
  1776  		})
  1777  
  1778  		Convey("rebuild s3 dedupe index from true to false", func() { //nolint: dupl
  1779  			taskScheduler := runAndGetScheduler()
  1780  			defer taskScheduler.Shutdown()
  1781  
  1782  			storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), false)
  1783  			defer cleanupStorage(storeDriver, testDir)
  1784  
  1785  			// rebuild with dedupe false, should have all blobs with content
  1786  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  1787  			// wait until rebuild finishes
  1788  
  1789  			time.Sleep(10 * time.Second)
  1790  
  1791  			taskScheduler.Shutdown()
  1792  
  1793  			fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1794  				blobDigest1.Encoded()))
  1795  			So(fi1.Size(), ShouldBeGreaterThan, 0)
  1796  			So(err, ShouldBeNil)
  1797  
  1798  			fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1799  				blobDigest2.Encoded()))
  1800  			So(err, ShouldBeNil)
  1801  			So(fi2.Size(), ShouldEqual, fi1.Size())
  1802  
  1803  			blobContent, err := imgStore.GetBlobContent("dedupe2", blobDigest2)
  1804  			So(err, ShouldBeNil)
  1805  			So(len(blobContent), ShouldEqual, fi1.Size())
  1806  
  1807  			Convey("delete blobs from storage/cache should work when dedupe is false", func() {
  1808  				So(blobDigest1, ShouldEqual, blobDigest2)
  1809  
  1810  				// to not trigger BlobInUse err, delete manifest first
  1811  				err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
  1812  				So(err, ShouldBeNil)
  1813  
  1814  				err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
  1815  				So(err, ShouldBeNil)
  1816  
  1817  				err = imgStore.DeleteBlob("dedupe1", blobDigest1)
  1818  				So(err, ShouldBeNil)
  1819  
  1820  				err = imgStore.DeleteBlob("dedupe2", blobDigest2)
  1821  				So(err, ShouldBeNil)
  1822  			})
  1823  
  1824  			Convey("rebuild s3 dedupe index from false to true", func() {
  1825  				taskScheduler := runAndGetScheduler()
  1826  				defer taskScheduler.Shutdown()
  1827  
  1828  				storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  1829  				defer cleanupStorage(storeDriver, testDir)
  1830  
  1831  				// rebuild with dedupe false, should have all blobs with content
  1832  				imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  1833  				// wait until rebuild finishes
  1834  
  1835  				time.Sleep(10 * time.Second)
  1836  
  1837  				taskScheduler.Shutdown()
  1838  
  1839  				fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1840  					blobDigest2.Encoded()))
  1841  				So(err, ShouldBeNil)
  1842  				So(fi2.Size(), ShouldEqual, 0)
  1843  
  1844  				blobContent, err := imgStore.GetBlobContent("dedupe2", blobDigest2)
  1845  				So(err, ShouldBeNil)
  1846  				So(len(blobContent), ShouldBeGreaterThan, 0)
  1847  			})
  1848  		})
  1849  
  1850  		Convey("Check that delete blobs moves the real content to the next contenders", func() {
  1851  			// if we delete blob1, the content should be moved to blob2
  1852  			// to not trigger BlobInUse err, delete manifest first
  1853  			err = imgStore.DeleteImageManifest("dedupe1", manifestDigest.String(), false)
  1854  			So(err, ShouldBeNil)
  1855  
  1856  			err = imgStore.DeleteImageManifest("dedupe2", "1.0", false)
  1857  			So(err, ShouldBeNil)
  1858  
  1859  			err = imgStore.DeleteBlob("dedupe1", blobDigest1)
  1860  			So(err, ShouldBeNil)
  1861  
  1862  			_, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1863  				blobDigest1.Encoded()))
  1864  			So(err, ShouldNotBeNil)
  1865  
  1866  			fi2, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1867  				blobDigest2.Encoded()))
  1868  			So(err, ShouldBeNil)
  1869  
  1870  			So(fi2.Size(), ShouldBeGreaterThan, 0)
  1871  			// the second blob should now be equal to the deleted blob.
  1872  			So(fi2.Size(), ShouldEqual, fi1.Size())
  1873  
  1874  			err = imgStore.DeleteBlob("dedupe2", blobDigest2)
  1875  			So(err, ShouldBeNil)
  1876  
  1877  			_, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1878  				blobDigest2.Encoded()))
  1879  			So(err, ShouldNotBeNil)
  1880  		})
  1881  	})
  1882  }
  1883  
  1884  func TestRebuildDedupeIndex(t *testing.T) {
  1885  	tskip.SkipS3(t)
  1886  
  1887  	Convey("Push images with dedupe true", t, func() {
  1888  		uuid, err := guuid.NewV4()
  1889  		if err != nil {
  1890  			panic(err)
  1891  		}
  1892  
  1893  		testDir := path.Join("/oci-repo-test", uuid.String())
  1894  
  1895  		tdir := t.TempDir()
  1896  
  1897  		storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, true)
  1898  		defer cleanupStorage(storeDriver, testDir)
  1899  
  1900  		// push image1
  1901  		content := []byte("test-data3")
  1902  		buf := bytes.NewBuffer(content)
  1903  		buflen := buf.Len()
  1904  		digest := godigest.FromBytes(content)
  1905  
  1906  		blobDigest1 := digest
  1907  
  1908  		_, blen, err := imgStore.FullBlobUpload("dedupe1", buf, digest)
  1909  		So(err, ShouldBeNil)
  1910  		So(blen, ShouldEqual, buflen)
  1911  
  1912  		hasBlob, blen1, err := imgStore.CheckBlob("dedupe1", digest)
  1913  		So(blen1, ShouldEqual, buflen)
  1914  		So(hasBlob, ShouldEqual, true)
  1915  		So(err, ShouldBeNil)
  1916  
  1917  		cblob, cdigest := GetRandomImageConfig()
  1918  		_, clen, err := imgStore.FullBlobUpload("dedupe1", bytes.NewReader(cblob), cdigest)
  1919  		So(err, ShouldBeNil)
  1920  		So(clen, ShouldEqual, len(cblob))
  1921  
  1922  		hasBlob, clen, err = imgStore.CheckBlob("dedupe1", cdigest)
  1923  		So(err, ShouldBeNil)
  1924  		So(hasBlob, ShouldEqual, true)
  1925  		So(clen, ShouldEqual, len(cblob))
  1926  
  1927  		manifest := ispec.Manifest{
  1928  			Config: ispec.Descriptor{
  1929  				MediaType: "application/vnd.oci.image.config.v1+json",
  1930  				Digest:    cdigest,
  1931  				Size:      int64(len(cblob)),
  1932  			},
  1933  			Layers: []ispec.Descriptor{
  1934  				{
  1935  					MediaType: "application/vnd.oci.image.layer.v1.tar",
  1936  					Digest:    digest,
  1937  					Size:      int64(buflen),
  1938  				},
  1939  			},
  1940  		}
  1941  
  1942  		manifest.SchemaVersion = 2
  1943  		manifestBuf, err := json.Marshal(manifest)
  1944  		So(err, ShouldBeNil)
  1945  		digest = godigest.FromBytes(manifestBuf)
  1946  		_, _, err = imgStore.PutImageManifest("dedupe1", digest.String(),
  1947  			ispec.MediaTypeImageManifest, manifestBuf)
  1948  		So(err, ShouldBeNil)
  1949  
  1950  		_, _, _, err = imgStore.GetImageManifest("dedupe1", digest.String())
  1951  		So(err, ShouldBeNil)
  1952  
  1953  		content = []byte("test-data3")
  1954  		buf = bytes.NewBuffer(content)
  1955  		buflen = buf.Len()
  1956  		digest = godigest.FromBytes(content)
  1957  
  1958  		blobDigest2 := digest
  1959  
  1960  		_, blen, err = imgStore.FullBlobUpload("dedupe2", buf, digest)
  1961  		So(err, ShouldBeNil)
  1962  		So(blen, ShouldEqual, buflen)
  1963  
  1964  		hasBlob, blen1, err = imgStore.CheckBlob("dedupe2", digest)
  1965  		So(blen1, ShouldEqual, buflen)
  1966  		So(hasBlob, ShouldEqual, true)
  1967  		So(err, ShouldBeNil)
  1968  
  1969  		_, clen, err = imgStore.FullBlobUpload("dedupe2", bytes.NewReader(cblob), cdigest)
  1970  		So(err, ShouldBeNil)
  1971  		So(clen, ShouldEqual, len(cblob))
  1972  
  1973  		hasBlob, clen, err = imgStore.CheckBlob("dedupe2", cdigest)
  1974  		So(err, ShouldBeNil)
  1975  		So(hasBlob, ShouldEqual, true)
  1976  		So(clen, ShouldEqual, len(cblob))
  1977  
  1978  		digest = godigest.FromBytes(manifestBuf)
  1979  		_, _, err = imgStore.PutImageManifest("dedupe2", digest.String(),
  1980  			ispec.MediaTypeImageManifest, manifestBuf)
  1981  		So(err, ShouldBeNil)
  1982  
  1983  		_, _, _, err = imgStore.GetImageManifest("dedupe2", digest.String())
  1984  		So(err, ShouldBeNil)
  1985  
  1986  		configFi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1987  			cdigest.Encoded()))
  1988  		So(err, ShouldBeNil)
  1989  
  1990  		configFi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1991  			cdigest.Encoded()))
  1992  		So(err, ShouldBeNil)
  1993  
  1994  		fi1, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe1", "blobs", "sha256",
  1995  			blobDigest1.Encoded()))
  1996  		So(err, ShouldBeNil)
  1997  
  1998  		fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  1999  			blobDigest2.Encoded()))
  2000  		So(err, ShouldBeNil)
  2001  
  2002  		// original blob should have the real content of blob
  2003  		So(fi1.Size(), ShouldNotEqual, fi2.Size())
  2004  		So(fi1.Size(), ShouldBeGreaterThan, 0)
  2005  		// deduped blob should be of size 0
  2006  		So(fi2.Size(), ShouldEqual, 0)
  2007  
  2008  		So(configFi1.Size(), ShouldNotEqual, configFi2.Size())
  2009  		So(configFi1.Size(), ShouldBeGreaterThan, 0)
  2010  		// deduped blob should be of size 0
  2011  		So(configFi2.Size(), ShouldEqual, 0)
  2012  
  2013  		Convey("Intrerrupt rebuilding and restart, checking idempotency", func() {
  2014  			for i := 0; i < 10; i++ {
  2015  				logger := log.Logger{}
  2016  				metrics := monitoring.NewMetricsServer(false, logger)
  2017  				taskScheduler := scheduler.NewScheduler(config.New(), metrics, logger)
  2018  				taskScheduler.RateLimit = 1 * time.Millisecond
  2019  
  2020  				taskScheduler.RunScheduler()
  2021  				defer taskScheduler.Shutdown()
  2022  
  2023  				storeDriver, imgStore, _ = createObjectsStore(testDir, t.TempDir(), false)
  2024  				defer cleanupStorage(storeDriver, testDir)
  2025  
  2026  				// rebuild with dedupe false, should have all blobs with content
  2027  				imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2028  				sleepValue := i * 5
  2029  				time.Sleep(time.Duration(sleepValue) * time.Millisecond)
  2030  
  2031  				taskScheduler.Shutdown()
  2032  			}
  2033  
  2034  			taskScheduler := runAndGetScheduler()
  2035  			defer taskScheduler.Shutdown()
  2036  
  2037  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2038  
  2039  			// wait until rebuild finishes
  2040  			time.Sleep(10 * time.Second)
  2041  
  2042  			taskScheduler.Shutdown()
  2043  
  2044  			fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  2045  				blobDigest2.Encoded()))
  2046  			So(err, ShouldBeNil)
  2047  			So(fi2.Size(), ShouldEqual, fi1.Size())
  2048  
  2049  			configFi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  2050  				cdigest.Encoded()))
  2051  			So(err, ShouldBeNil)
  2052  			So(configFi2.Size(), ShouldEqual, configFi1.Size())
  2053  
  2054  			// now from dedupe false to true
  2055  			for i := 0; i < 10; i++ {
  2056  				logger := log.Logger{}
  2057  				metrics := monitoring.NewMetricsServer(false, logger)
  2058  				taskScheduler := scheduler.NewScheduler(config.New(), metrics, logger)
  2059  				taskScheduler.RateLimit = 1 * time.Millisecond
  2060  
  2061  				taskScheduler.RunScheduler()
  2062  				defer taskScheduler.Shutdown()
  2063  
  2064  				storeDriver, imgStore, _ = createObjectsStore(testDir, t.TempDir(), true)
  2065  				defer cleanupStorage(storeDriver, testDir)
  2066  
  2067  				// rebuild with dedupe false, should have all blobs with content
  2068  				imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2069  
  2070  				sleepValue := i * 5
  2071  				time.Sleep(time.Duration(sleepValue) * time.Millisecond)
  2072  
  2073  				taskScheduler.Shutdown()
  2074  			}
  2075  
  2076  			taskScheduler = runAndGetScheduler()
  2077  			defer taskScheduler.Shutdown()
  2078  
  2079  			// rebuild with dedupe false, should have all blobs with content
  2080  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2081  
  2082  			// wait until rebuild finishes
  2083  			time.Sleep(10 * time.Second)
  2084  
  2085  			taskScheduler.Shutdown()
  2086  
  2087  			fi2, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  2088  				blobDigest2.Encoded()))
  2089  			So(err, ShouldBeNil)
  2090  			So(fi2.Size(), ShouldNotEqual, fi1.Size())
  2091  			So(fi2.Size(), ShouldEqual, 0)
  2092  
  2093  			configFi2, err = storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  2094  				cdigest.Encoded()))
  2095  			So(err, ShouldBeNil)
  2096  			So(configFi2.Size(), ShouldNotEqual, configFi1.Size())
  2097  			So(configFi2.Size(), ShouldEqual, 0)
  2098  		})
  2099  
  2100  		Convey("Trigger ErrDedupeRebuild because cache is nil", func() {
  2101  			storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, true)
  2102  			defer cleanupStorage(storeDriver, testDir)
  2103  
  2104  			taskScheduler := runAndGetScheduler()
  2105  			defer taskScheduler.Shutdown()
  2106  
  2107  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2108  
  2109  			// wait until rebuild finishes
  2110  			time.Sleep(3 * time.Second)
  2111  		})
  2112  
  2113  		Convey("Rebuild dedupe index already rebuilt", func() {
  2114  			taskScheduler := runAndGetScheduler()
  2115  			defer taskScheduler.Shutdown()
  2116  
  2117  			storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  2118  			defer cleanupStorage(storeDriver, testDir)
  2119  
  2120  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2121  
  2122  			// wait until rebuild finishes
  2123  			time.Sleep(5 * time.Second)
  2124  		})
  2125  
  2126  		Convey("Trigger Stat error while getting original blob", func() {
  2127  			tdir := t.TempDir()
  2128  			storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, false)
  2129  			defer cleanupStorage(storeDriver, testDir)
  2130  
  2131  			// remove original blob
  2132  			err := storeDriver.PutContent(context.Background(), fi1.Path(), []byte{})
  2133  			So(err, ShouldBeNil)
  2134  
  2135  			taskScheduler := runAndGetScheduler()
  2136  			defer taskScheduler.Shutdown()
  2137  
  2138  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2139  
  2140  			// wait until rebuild finishes
  2141  			time.Sleep(5 * time.Second)
  2142  		})
  2143  
  2144  		Convey("Trigger ErrDedupeRebuild while statting original blob", func() {
  2145  			// remove original blob
  2146  			err := storeDriver.Delete(context.Background(), fi1.Path())
  2147  			So(err, ShouldBeNil)
  2148  
  2149  			taskScheduler := runAndGetScheduler()
  2150  			defer taskScheduler.Shutdown()
  2151  
  2152  			storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  2153  			defer cleanupStorage(storeDriver, testDir)
  2154  
  2155  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2156  
  2157  			// wait until rebuild finishes
  2158  			time.Sleep(5 * time.Second)
  2159  		})
  2160  
  2161  		Convey("Trigger ErrDedupeRebuild when original blob has 0 size", func() {
  2162  			// remove original blob
  2163  			err := storeDriver.PutContent(context.Background(), fi1.Path(), []byte{})
  2164  			So(err, ShouldBeNil)
  2165  
  2166  			taskScheduler := runAndGetScheduler()
  2167  			defer taskScheduler.Shutdown()
  2168  
  2169  			storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  2170  			defer cleanupStorage(storeDriver, testDir)
  2171  
  2172  			// rebuild with dedupe false, should have all blobs with content
  2173  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2174  
  2175  			// wait until rebuild finishes
  2176  			time.Sleep(5 * time.Second)
  2177  		})
  2178  
  2179  		Convey("Trigger GetNextDigestWithBlobPaths path not found err", func() {
  2180  			tdir := t.TempDir()
  2181  			storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, true)
  2182  			defer cleanupStorage(storeDriver, testDir)
  2183  
  2184  			// remove rootDir
  2185  			err := storeDriver.Delete(context.Background(), imgStore.RootDir())
  2186  			So(err, ShouldBeNil)
  2187  
  2188  			taskScheduler := runAndGetScheduler()
  2189  			defer taskScheduler.Shutdown()
  2190  
  2191  			// rebuild with dedupe false, should have all blobs with content
  2192  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2193  
  2194  			// wait until rebuild finishes
  2195  			time.Sleep(5 * time.Second)
  2196  		})
  2197  
  2198  		Convey("Rebuild from true to false", func() {
  2199  			taskScheduler := runAndGetScheduler()
  2200  			defer taskScheduler.Shutdown()
  2201  
  2202  			storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), false)
  2203  			defer cleanupStorage(storeDriver, testDir)
  2204  
  2205  			// rebuild with dedupe false, should have all blobs with content
  2206  			imgStore.RunDedupeBlobs(time.Duration(0), taskScheduler)
  2207  
  2208  			// wait until rebuild finishes
  2209  			time.Sleep(10 * time.Second)
  2210  
  2211  			taskScheduler.Shutdown()
  2212  
  2213  			fi2, err := storeDriver.Stat(context.Background(), path.Join(testDir, "dedupe2", "blobs", "sha256",
  2214  				blobDigest2.Encoded()))
  2215  			So(err, ShouldBeNil)
  2216  			So(fi2.Size(), ShouldEqual, fi1.Size())
  2217  		})
  2218  	})
  2219  }
  2220  
  2221  func TestNextRepositoryMockStoreDriver(t *testing.T) {
  2222  	testDir := t.TempDir()
  2223  	tdir := t.TempDir()
  2224  
  2225  	// some s3 implementations (eg, digitalocean spaces) will return pathnotfounderror for walk but not list
  2226  	// This code cannot be reliably covered by end to end tests
  2227  	Convey("Trigger PathNotFound error when Walk() is called in GetNextRepository()", t, func() {
  2228  		imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{
  2229  			ListFn: func(ctx context.Context, path string) ([]string, error) {
  2230  				return []string{}, nil
  2231  			},
  2232  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2233  				return driver.PathNotFoundError{}
  2234  			},
  2235  		})
  2236  
  2237  		nextRepository, err := imgStore.GetNextRepository("testRepo")
  2238  		So(err, ShouldBeNil)
  2239  		So(nextRepository, ShouldEqual, "")
  2240  	})
  2241  }
  2242  
  2243  func TestRebuildDedupeMockStoreDriver(t *testing.T) {
  2244  	uuid, err := guuid.NewV4()
  2245  	if err != nil {
  2246  		panic(err)
  2247  	}
  2248  
  2249  	testDir := path.Join("/oci-repo-test", uuid.String())
  2250  
  2251  	tdir := t.TempDir()
  2252  
  2253  	validDigest := godigest.FromString("digest")
  2254  
  2255  	Convey("Trigger Stat error in getOriginalBlobFromDisk()", t, func() {
  2256  		imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{
  2257  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2258  				return &FileInfoMock{}, errS3
  2259  			},
  2260  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2261  				return walkFn(&FileInfoMock{
  2262  					IsDirFn: func() bool {
  2263  						return false
  2264  					},
  2265  					PathFn: func() string {
  2266  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2267  					},
  2268  				})
  2269  			},
  2270  		})
  2271  
  2272  		digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2273  		So(err, ShouldBeNil)
  2274  
  2275  		err = imgStore.RunDedupeForDigest(context.TODO(), digest, false, duplicateBlobs)
  2276  		So(err, ShouldNotBeNil)
  2277  	})
  2278  
  2279  	Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() {
  2280  		imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{
  2281  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2282  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2283  					return &FileInfoMock{
  2284  						SizeFn: func() int64 {
  2285  							return int64(0)
  2286  						},
  2287  					}, nil
  2288  				}
  2289  
  2290  				return &FileInfoMock{
  2291  					SizeFn: func() int64 {
  2292  						return int64(10)
  2293  					},
  2294  				}, nil
  2295  			},
  2296  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2297  				_ = walkFn(&FileInfoMock{
  2298  					IsDirFn: func() bool {
  2299  						return false
  2300  					},
  2301  					PathFn: func() string {
  2302  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2303  					},
  2304  				})
  2305  				_ = walkFn(&FileInfoMock{
  2306  					IsDirFn: func() bool {
  2307  						return false
  2308  					},
  2309  					PathFn: func() string {
  2310  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2311  					},
  2312  				})
  2313  
  2314  				return nil
  2315  			},
  2316  			GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
  2317  				return []byte{}, errS3
  2318  			},
  2319  		})
  2320  
  2321  		digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2322  		So(err, ShouldBeNil)
  2323  
  2324  		err = imgStore.RunDedupeForDigest(context.TODO(), digest, false, duplicateBlobs)
  2325  		So(err, ShouldNotBeNil)
  2326  	})
  2327  
  2328  	Convey("Trigger GetContent error in restoreDedupedBlobs()", t, func() {
  2329  		imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{
  2330  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2331  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2332  					return &FileInfoMock{
  2333  						SizeFn: func() int64 {
  2334  							return int64(0)
  2335  						},
  2336  					}, nil
  2337  				}
  2338  
  2339  				return &FileInfoMock{
  2340  					SizeFn: func() int64 {
  2341  						return int64(10)
  2342  					},
  2343  				}, nil
  2344  			},
  2345  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2346  				_ = walkFn(&FileInfoMock{
  2347  					IsDirFn: func() bool {
  2348  						return false
  2349  					},
  2350  					PathFn: func() string {
  2351  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2352  					},
  2353  				})
  2354  				_ = walkFn(&FileInfoMock{
  2355  					IsDirFn: func() bool {
  2356  						return false
  2357  					},
  2358  					PathFn: func() string {
  2359  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2360  					},
  2361  				})
  2362  
  2363  				return nil
  2364  			},
  2365  			WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  2366  				return &FileWriterMock{}, errS3
  2367  			},
  2368  		})
  2369  
  2370  		digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2371  		So(err, ShouldBeNil)
  2372  
  2373  		err = imgStore.RunDedupeForDigest(context.TODO(), digest, false, duplicateBlobs)
  2374  		So(err, ShouldNotBeNil)
  2375  	})
  2376  
  2377  	Convey("Trigger Stat() error in restoreDedupedBlobs()", t, func() {
  2378  		imgStore := createMockStorage(testDir, tdir, false, &StorageDriverMock{
  2379  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2380  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2381  					return &FileInfoMock{
  2382  						SizeFn: func() int64 {
  2383  							return int64(10)
  2384  						},
  2385  					}, nil
  2386  				}
  2387  
  2388  				return &FileInfoMock{
  2389  					SizeFn: func() int64 {
  2390  						return int64(10)
  2391  					},
  2392  				}, errS3
  2393  			},
  2394  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2395  				_ = walkFn(&FileInfoMock{
  2396  					IsDirFn: func() bool {
  2397  						return false
  2398  					},
  2399  					PathFn: func() string {
  2400  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2401  					},
  2402  				})
  2403  				_ = walkFn(&FileInfoMock{
  2404  					IsDirFn: func() bool {
  2405  						return false
  2406  					},
  2407  					PathFn: func() string {
  2408  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2409  					},
  2410  				})
  2411  
  2412  				return nil
  2413  			},
  2414  		})
  2415  
  2416  		digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2417  		So(err, ShouldBeNil)
  2418  
  2419  		err = imgStore.RunDedupeForDigest(context.TODO(), digest, false, duplicateBlobs)
  2420  		So(err, ShouldNotBeNil)
  2421  
  2422  		Convey("Trigger Stat() error in dedupeBlobs()", func() {
  2423  			imgStore := createMockStorage(testDir, t.TempDir(), true, &StorageDriverMock{
  2424  				StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2425  					if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2426  						return &FileInfoMock{
  2427  							SizeFn: func() int64 {
  2428  								return int64(10)
  2429  							},
  2430  						}, nil
  2431  					}
  2432  
  2433  					return &FileInfoMock{
  2434  						SizeFn: func() int64 {
  2435  							return int64(10)
  2436  						},
  2437  					}, errS3
  2438  				},
  2439  				WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2440  					_ = walkFn(&FileInfoMock{
  2441  						IsDirFn: func() bool {
  2442  							return false
  2443  						},
  2444  						PathFn: func() string {
  2445  							return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2446  						},
  2447  					})
  2448  					_ = walkFn(&FileInfoMock{
  2449  						IsDirFn: func() bool {
  2450  							return false
  2451  						},
  2452  						PathFn: func() string {
  2453  							return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2454  						},
  2455  					})
  2456  
  2457  					return nil
  2458  				},
  2459  			})
  2460  
  2461  			digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2462  			So(err, ShouldBeNil)
  2463  
  2464  			err = imgStore.RunDedupeForDigest(context.TODO(), digest, false, duplicateBlobs)
  2465  			So(err, ShouldNotBeNil)
  2466  		})
  2467  	})
  2468  
  2469  	Convey("Trigger PutContent() error in dedupeBlobs()", t, func() {
  2470  		tdir := t.TempDir()
  2471  		imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{
  2472  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2473  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2474  					return &FileInfoMock{
  2475  						SizeFn: func() int64 {
  2476  							return int64(0)
  2477  						},
  2478  					}, nil
  2479  				}
  2480  
  2481  				return &FileInfoMock{
  2482  					SizeFn: func() int64 {
  2483  						return int64(10)
  2484  					},
  2485  				}, nil
  2486  			},
  2487  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2488  				_ = walkFn(&FileInfoMock{
  2489  					IsDirFn: func() bool {
  2490  						return false
  2491  					},
  2492  					PathFn: func() string {
  2493  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2494  					},
  2495  				})
  2496  				_ = walkFn(&FileInfoMock{
  2497  					IsDirFn: func() bool {
  2498  						return false
  2499  					},
  2500  					PathFn: func() string {
  2501  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2502  					},
  2503  				})
  2504  
  2505  				return nil
  2506  			},
  2507  			PutContentFn: func(ctx context.Context, path string, content []byte) error {
  2508  				return errS3
  2509  			},
  2510  		})
  2511  
  2512  		digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2513  		So(err, ShouldBeNil)
  2514  
  2515  		err = imgStore.RunDedupeForDigest(context.TODO(), digest, true, duplicateBlobs)
  2516  		So(err, ShouldNotBeNil)
  2517  	})
  2518  
  2519  	//nolint: dupl
  2520  	Convey("Trigger getOriginalBlob() error in dedupeBlobs()", t, func() {
  2521  		tdir := t.TempDir()
  2522  		imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{
  2523  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2524  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2525  					return &FileInfoMock{
  2526  						SizeFn: func() int64 {
  2527  							return int64(0)
  2528  						},
  2529  					}, nil
  2530  				}
  2531  
  2532  				return &FileInfoMock{
  2533  					SizeFn: func() int64 {
  2534  						return int64(0)
  2535  					},
  2536  				}, nil
  2537  			},
  2538  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2539  				_ = walkFn(&FileInfoMock{
  2540  					IsDirFn: func() bool {
  2541  						return false
  2542  					},
  2543  					PathFn: func() string {
  2544  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2545  					},
  2546  				})
  2547  				_ = walkFn(&FileInfoMock{
  2548  					IsDirFn: func() bool {
  2549  						return false
  2550  					},
  2551  					PathFn: func() string {
  2552  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2553  					},
  2554  				})
  2555  
  2556  				return nil
  2557  			},
  2558  		})
  2559  
  2560  		digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2561  		So(err, ShouldBeNil)
  2562  
  2563  		err = imgStore.RunDedupeForDigest(context.TODO(), digest, true, duplicateBlobs)
  2564  		So(err, ShouldNotBeNil)
  2565  	})
  2566  
  2567  	//nolint: dupl
  2568  	Convey("Trigger Stat() error in dedupeBlobs()", t, func() {
  2569  		tdir := t.TempDir()
  2570  		imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{
  2571  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2572  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2573  					return &FileInfoMock{
  2574  						SizeFn: func() int64 {
  2575  							return int64(10)
  2576  						},
  2577  					}, nil
  2578  				}
  2579  
  2580  				return &FileInfoMock{
  2581  					SizeFn: func() int64 {
  2582  						return int64(10)
  2583  					},
  2584  				}, errS3
  2585  			},
  2586  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2587  				_ = walkFn(&FileInfoMock{
  2588  					IsDirFn: func() bool {
  2589  						return false
  2590  					},
  2591  					PathFn: func() string {
  2592  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2593  					},
  2594  				})
  2595  				_ = walkFn(&FileInfoMock{
  2596  					IsDirFn: func() bool {
  2597  						return false
  2598  					},
  2599  					PathFn: func() string {
  2600  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2601  					},
  2602  				})
  2603  
  2604  				return nil
  2605  			},
  2606  		})
  2607  
  2608  		digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2609  		So(err, ShouldBeNil)
  2610  
  2611  		err = imgStore.RunDedupeForDigest(context.TODO(), digest, true, duplicateBlobs)
  2612  		So(err, ShouldNotBeNil)
  2613  	})
  2614  
  2615  	Convey("Trigger getNextDigestWithBlobPaths err", t, func() {
  2616  		tdir := t.TempDir()
  2617  		imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{
  2618  			WalkFn: func(ctx context.Context, path string, f driver.WalkFn) error {
  2619  				return errS3
  2620  			},
  2621  		})
  2622  
  2623  		_, _, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2624  		So(err, ShouldNotBeNil)
  2625  	})
  2626  
  2627  	Convey("Trigger cache errors", t, func() {
  2628  		storageDriverMockIfBranch := &StorageDriverMock{
  2629  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2630  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2631  					return &FileInfoMock{
  2632  						SizeFn: func() int64 {
  2633  							return int64(0)
  2634  						},
  2635  					}, nil
  2636  				}
  2637  
  2638  				return &FileInfoMock{
  2639  					SizeFn: func() int64 {
  2640  						return int64(10)
  2641  					},
  2642  				}, nil
  2643  			},
  2644  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2645  				_ = walkFn(&FileInfoMock{
  2646  					IsDirFn: func() bool {
  2647  						return false
  2648  					},
  2649  					PathFn: func() string {
  2650  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2651  					},
  2652  				})
  2653  				_ = walkFn(&FileInfoMock{
  2654  					IsDirFn: func() bool {
  2655  						return false
  2656  					},
  2657  					PathFn: func() string {
  2658  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2659  					},
  2660  				})
  2661  
  2662  				return nil
  2663  			},
  2664  		}
  2665  
  2666  		storageDriverMockElseBranch := &StorageDriverMock{
  2667  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  2668  				if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2669  					return &FileInfoMock{
  2670  						SizeFn: func() int64 {
  2671  							return int64(10)
  2672  						},
  2673  					}, nil
  2674  				}
  2675  
  2676  				return &FileInfoMock{
  2677  					SizeFn: func() int64 {
  2678  						return int64(10)
  2679  					},
  2680  				}, nil
  2681  			},
  2682  			WalkFn: func(ctx context.Context, path string, walkFn driver.WalkFn) error {
  2683  				_ = walkFn(&FileInfoMock{
  2684  					IsDirFn: func() bool {
  2685  						return false
  2686  					},
  2687  					PathFn: func() string {
  2688  						return fmt.Sprintf("path/to/%s", validDigest.Encoded())
  2689  					},
  2690  				})
  2691  				_ = walkFn(&FileInfoMock{
  2692  					IsDirFn: func() bool {
  2693  						return false
  2694  					},
  2695  					PathFn: func() string {
  2696  						return fmt.Sprintf("path/to/second/%s", validDigest.Encoded())
  2697  					},
  2698  				})
  2699  
  2700  				return nil
  2701  			},
  2702  		}
  2703  
  2704  		Convey("on original blob", func() {
  2705  			imgStore := createMockStorageWithMockCache(testDir, true, storageDriverMockIfBranch,
  2706  				&mocks.CacheMock{
  2707  					HasBlobFn: func(digest godigest.Digest, path string) bool {
  2708  						return false
  2709  					},
  2710  					PutBlobFn: func(digest godigest.Digest, path string) error {
  2711  						return errCache
  2712  					},
  2713  				})
  2714  
  2715  			digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2716  			So(err, ShouldBeNil)
  2717  
  2718  			err = imgStore.RunDedupeForDigest(context.TODO(), digest, true, duplicateBlobs)
  2719  			So(err, ShouldNotBeNil)
  2720  		})
  2721  
  2722  		Convey("on dedupe blob", func() {
  2723  			imgStore := createMockStorageWithMockCache(testDir, true, storageDriverMockIfBranch,
  2724  				&mocks.CacheMock{
  2725  					HasBlobFn: func(digest godigest.Digest, path string) bool {
  2726  						return false
  2727  					},
  2728  					PutBlobFn: func(digest godigest.Digest, path string) error {
  2729  						if path == fmt.Sprintf("path/to/%s", validDigest.Encoded()) {
  2730  							return errCache
  2731  						}
  2732  
  2733  						return nil
  2734  					},
  2735  				})
  2736  
  2737  			digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2738  			So(err, ShouldBeNil)
  2739  
  2740  			err = imgStore.RunDedupeForDigest(context.TODO(), digest, true, duplicateBlobs)
  2741  			So(err, ShouldNotBeNil)
  2742  		})
  2743  
  2744  		Convey("on else branch", func() {
  2745  			imgStore := createMockStorageWithMockCache(testDir, true, storageDriverMockElseBranch,
  2746  				&mocks.CacheMock{
  2747  					HasBlobFn: func(digest godigest.Digest, path string) bool {
  2748  						return false
  2749  					},
  2750  					PutBlobFn: func(digest godigest.Digest, path string) error {
  2751  						return errCache
  2752  					},
  2753  				})
  2754  
  2755  			digest, duplicateBlobs, err := imgStore.GetNextDigestWithBlobPaths([]string{"path/to"}, []godigest.Digest{})
  2756  			So(err, ShouldBeNil)
  2757  
  2758  			err = imgStore.RunDedupeForDigest(context.TODO(), digest, true, duplicateBlobs)
  2759  			So(err, ShouldNotBeNil)
  2760  		})
  2761  	})
  2762  }
  2763  
  2764  func TestS3PullRange(t *testing.T) {
  2765  	tskip.SkipS3(t)
  2766  
  2767  	Convey("Test against s3 image store", t, func() {
  2768  		uuid, err := guuid.NewV4()
  2769  		if err != nil {
  2770  			panic(err)
  2771  		}
  2772  
  2773  		testDir := path.Join("/oci-repo-test", uuid.String())
  2774  
  2775  		storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  2776  		defer cleanupStorage(storeDriver, testDir)
  2777  
  2778  		// create a blob/layer
  2779  		upload, err := imgStore.NewBlobUpload("index")
  2780  		So(err, ShouldBeNil)
  2781  		So(upload, ShouldNotBeEmpty)
  2782  
  2783  		content := []byte("0123456789")
  2784  		buf := bytes.NewBuffer(content)
  2785  		buflen := buf.Len()
  2786  		digest := godigest.FromBytes(content)
  2787  		So(digest, ShouldNotBeNil)
  2788  		blob, err := imgStore.PutBlobChunkStreamed("index", upload, buf)
  2789  		So(err, ShouldBeNil)
  2790  		So(blob, ShouldEqual, buflen)
  2791  
  2792  		err = imgStore.FinishBlobUpload("index", upload, buf, digest)
  2793  		So(err, ShouldBeNil)
  2794  		So(blob, ShouldEqual, buflen)
  2795  
  2796  		Convey("Without Dedupe", func() {
  2797  			reader, _, _, err := imgStore.GetBlobPartial("index", digest, "*/*", 0, -1)
  2798  			So(err, ShouldBeNil)
  2799  			rdbuf, err := io.ReadAll(reader)
  2800  			So(err, ShouldBeNil)
  2801  			So(rdbuf, ShouldResemble, content)
  2802  			reader.Close()
  2803  
  2804  			reader, _, _, err = imgStore.GetBlobPartial("index", digest, "application/octet-stream", 0, -1)
  2805  			So(err, ShouldBeNil)
  2806  			rdbuf, err = io.ReadAll(reader)
  2807  			So(err, ShouldBeNil)
  2808  			So(rdbuf, ShouldResemble, content)
  2809  			reader.Close()
  2810  
  2811  			reader, _, _, err = imgStore.GetBlobPartial("index", digest, "*/*", 0, 100)
  2812  			So(err, ShouldBeNil)
  2813  			rdbuf, err = io.ReadAll(reader)
  2814  			So(err, ShouldBeNil)
  2815  			So(rdbuf, ShouldResemble, content)
  2816  			reader.Close()
  2817  
  2818  			reader, _, _, err = imgStore.GetBlobPartial("index", digest, "*/*", 0, 10)
  2819  			So(err, ShouldBeNil)
  2820  			rdbuf, err = io.ReadAll(reader)
  2821  			So(err, ShouldBeNil)
  2822  			So(rdbuf, ShouldResemble, content)
  2823  			reader.Close()
  2824  
  2825  			reader, _, _, err = imgStore.GetBlobPartial("index", digest, "*/*", 0, 0)
  2826  			So(err, ShouldBeNil)
  2827  			rdbuf, err = io.ReadAll(reader)
  2828  			So(err, ShouldBeNil)
  2829  			So(rdbuf, ShouldResemble, content[0:1])
  2830  			reader.Close()
  2831  
  2832  			reader, _, _, err = imgStore.GetBlobPartial("index", digest, "*/*", 0, 1)
  2833  			So(err, ShouldBeNil)
  2834  			rdbuf, err = io.ReadAll(reader)
  2835  			So(err, ShouldBeNil)
  2836  			So(rdbuf, ShouldResemble, content[0:2])
  2837  			reader.Close()
  2838  
  2839  			reader, _, _, err = imgStore.GetBlobPartial("index", digest, "*/*", 2, 3)
  2840  			So(err, ShouldBeNil)
  2841  			rdbuf, err = io.ReadAll(reader)
  2842  			So(err, ShouldBeNil)
  2843  			So(rdbuf, ShouldResemble, content[2:4])
  2844  			reader.Close()
  2845  		})
  2846  
  2847  		Convey("With Dedupe", func() {
  2848  			// create a blob/layer with same content
  2849  			upload, err := imgStore.NewBlobUpload("dupindex")
  2850  			So(err, ShouldBeNil)
  2851  			So(upload, ShouldNotBeEmpty)
  2852  
  2853  			dupcontent := []byte("0123456789")
  2854  			buf := bytes.NewBuffer(dupcontent)
  2855  			buflen := buf.Len()
  2856  			digest := godigest.FromBytes(dupcontent)
  2857  			So(digest, ShouldNotBeNil)
  2858  			blob, err := imgStore.PutBlobChunkStreamed("dupindex", upload, buf)
  2859  			So(err, ShouldBeNil)
  2860  			So(blob, ShouldEqual, buflen)
  2861  
  2862  			err = imgStore.FinishBlobUpload("dupindex", upload, buf, digest)
  2863  			So(err, ShouldBeNil)
  2864  			So(blob, ShouldEqual, buflen)
  2865  
  2866  			reader, _, _, err := imgStore.GetBlobPartial("dupindex", digest, "*/*", 0, -1)
  2867  			So(err, ShouldBeNil)
  2868  			rdbuf, err := io.ReadAll(reader)
  2869  			So(err, ShouldBeNil)
  2870  			So(rdbuf, ShouldResemble, content)
  2871  			reader.Close()
  2872  
  2873  			reader, _, _, err = imgStore.GetBlobPartial("dupindex", digest, "application/octet-stream", 0, -1)
  2874  			So(err, ShouldBeNil)
  2875  			rdbuf, err = io.ReadAll(reader)
  2876  			So(err, ShouldBeNil)
  2877  			So(rdbuf, ShouldResemble, content)
  2878  			reader.Close()
  2879  
  2880  			reader, _, _, err = imgStore.GetBlobPartial("dupindex", digest, "*/*", 0, 100)
  2881  			So(err, ShouldBeNil)
  2882  			rdbuf, err = io.ReadAll(reader)
  2883  			So(err, ShouldBeNil)
  2884  			So(rdbuf, ShouldResemble, content)
  2885  			reader.Close()
  2886  
  2887  			reader, _, _, err = imgStore.GetBlobPartial("dupindex", digest, "*/*", 0, 10)
  2888  			So(err, ShouldBeNil)
  2889  			rdbuf, err = io.ReadAll(reader)
  2890  			So(err, ShouldBeNil)
  2891  			So(rdbuf, ShouldResemble, content)
  2892  			reader.Close()
  2893  
  2894  			reader, _, _, err = imgStore.GetBlobPartial("dupindex", digest, "*/*", 0, 0)
  2895  			So(err, ShouldBeNil)
  2896  			rdbuf, err = io.ReadAll(reader)
  2897  			So(err, ShouldBeNil)
  2898  			So(rdbuf, ShouldResemble, content[0:1])
  2899  			reader.Close()
  2900  
  2901  			reader, _, _, err = imgStore.GetBlobPartial("dupindex", digest, "*/*", 0, 1)
  2902  			So(err, ShouldBeNil)
  2903  			rdbuf, err = io.ReadAll(reader)
  2904  			So(err, ShouldBeNil)
  2905  			So(rdbuf, ShouldResemble, content[0:2])
  2906  			reader.Close()
  2907  
  2908  			reader, _, _, err = imgStore.GetBlobPartial("dupindex", digest, "*/*", 2, 3)
  2909  			So(err, ShouldBeNil)
  2910  			rdbuf, err = io.ReadAll(reader)
  2911  			So(err, ShouldBeNil)
  2912  			So(rdbuf, ShouldResemble, content[2:4])
  2913  			reader.Close()
  2914  
  2915  			// delete original blob
  2916  			err = imgStore.DeleteBlob("index", digest)
  2917  			So(err, ShouldBeNil)
  2918  
  2919  			reader, _, _, err = imgStore.GetBlobPartial("dupindex", digest, "*/*", 2, 3)
  2920  			So(err, ShouldBeNil)
  2921  			rdbuf, err = io.ReadAll(reader)
  2922  			So(err, ShouldBeNil)
  2923  			So(rdbuf, ShouldResemble, content[2:4])
  2924  			reader.Close()
  2925  		})
  2926  
  2927  		Convey("Negative cases", func() {
  2928  			_, _, _, err := imgStore.GetBlobPartial("index", "deadBEEF", "*/*", 0, -1)
  2929  			So(err, ShouldNotBeNil)
  2930  
  2931  			content := []byte("invalid content")
  2932  			digest := godigest.FromBytes(content)
  2933  
  2934  			_, _, _, err = imgStore.GetBlobPartial("index", digest, "*/*", 0, -1)
  2935  			So(err, ShouldNotBeNil)
  2936  		})
  2937  	})
  2938  }
  2939  
  2940  func TestS3ManifestImageIndex(t *testing.T) {
  2941  	tskip.SkipS3(t)
  2942  
  2943  	Convey("Test against s3 image store", t, func() {
  2944  		uuid, err := guuid.NewV4()
  2945  		if err != nil {
  2946  			panic(err)
  2947  		}
  2948  
  2949  		testDir := path.Join("/oci-repo-test", uuid.String())
  2950  
  2951  		storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  2952  		defer cleanupStorage(storeDriver, testDir)
  2953  
  2954  		// create a blob/layer
  2955  		upload, err := imgStore.NewBlobUpload("index")
  2956  		So(err, ShouldBeNil)
  2957  		So(upload, ShouldNotBeEmpty)
  2958  
  2959  		content := []byte("this is a blob1")
  2960  		buf := bytes.NewBuffer(content)
  2961  		buflen := buf.Len()
  2962  		digest := godigest.FromBytes(content)
  2963  		So(digest, ShouldNotBeNil)
  2964  		blob, err := imgStore.PutBlobChunkStreamed("index", upload, buf)
  2965  		So(err, ShouldBeNil)
  2966  		So(blob, ShouldEqual, buflen)
  2967  		bdgst1 := digest
  2968  		bsize1 := len(content)
  2969  
  2970  		err = imgStore.FinishBlobUpload("index", upload, buf, digest)
  2971  		So(err, ShouldBeNil)
  2972  		So(blob, ShouldEqual, buflen)
  2973  
  2974  		// upload image config blob
  2975  		upload, err = imgStore.NewBlobUpload("index")
  2976  		So(err, ShouldBeNil)
  2977  		So(upload, ShouldNotBeEmpty)
  2978  
  2979  		cblob, cdigest := GetRandomImageConfig()
  2980  		buf = bytes.NewBuffer(cblob)
  2981  		buflen = buf.Len()
  2982  		blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf)
  2983  		So(err, ShouldBeNil)
  2984  		So(blob, ShouldEqual, buflen)
  2985  
  2986  		err = imgStore.FinishBlobUpload("index", upload, buf, cdigest)
  2987  		So(err, ShouldBeNil)
  2988  		So(blob, ShouldEqual, buflen)
  2989  
  2990  		// create a manifest
  2991  		manifest := ispec.Manifest{
  2992  			Config: ispec.Descriptor{
  2993  				MediaType: ispec.MediaTypeImageConfig,
  2994  				Digest:    cdigest,
  2995  				Size:      int64(len(cblob)),
  2996  			},
  2997  			Layers: []ispec.Descriptor{
  2998  				{
  2999  					MediaType: ispec.MediaTypeImageLayer,
  3000  					Digest:    bdgst1,
  3001  					Size:      int64(bsize1),
  3002  				},
  3003  			},
  3004  		}
  3005  		manifest.SchemaVersion = 2
  3006  		content, err = json.Marshal(manifest)
  3007  		So(err, ShouldBeNil)
  3008  		digest = godigest.FromBytes(content)
  3009  		So(digest, ShouldNotBeNil)
  3010  		m1content := content
  3011  		_, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageManifest, content)
  3012  		So(err, ShouldBeNil)
  3013  
  3014  		// create another manifest but upload using its sha256 reference
  3015  
  3016  		// upload image config blob
  3017  		upload, err = imgStore.NewBlobUpload("index")
  3018  		So(err, ShouldBeNil)
  3019  		So(upload, ShouldNotBeEmpty)
  3020  
  3021  		cblob, cdigest = GetRandomImageConfig()
  3022  		buf = bytes.NewBuffer(cblob)
  3023  		buflen = buf.Len()
  3024  		blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf)
  3025  		So(err, ShouldBeNil)
  3026  		So(blob, ShouldEqual, buflen)
  3027  
  3028  		err = imgStore.FinishBlobUpload("index", upload, buf, cdigest)
  3029  		So(err, ShouldBeNil)
  3030  		So(blob, ShouldEqual, buflen)
  3031  
  3032  		// create a manifest
  3033  		manifest = ispec.Manifest{
  3034  			Config: ispec.Descriptor{
  3035  				MediaType: ispec.MediaTypeImageConfig,
  3036  				Digest:    cdigest,
  3037  				Size:      int64(len(cblob)),
  3038  			},
  3039  			Layers: []ispec.Descriptor{
  3040  				{
  3041  					MediaType: ispec.MediaTypeImageLayer,
  3042  					Digest:    bdgst1,
  3043  					Size:      int64(bsize1),
  3044  				},
  3045  			},
  3046  		}
  3047  		manifest.SchemaVersion = 2
  3048  		content, err = json.Marshal(manifest)
  3049  		So(err, ShouldBeNil)
  3050  		digest = godigest.FromBytes(content)
  3051  		So(digest, ShouldNotBeNil)
  3052  		m2dgst := digest
  3053  		m2size := len(content)
  3054  		_, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content)
  3055  		So(err, ShouldBeNil)
  3056  
  3057  		Convey("Image index", func() {
  3058  			// upload image config blob
  3059  			upload, err = imgStore.NewBlobUpload("index")
  3060  			So(err, ShouldBeNil)
  3061  			So(upload, ShouldNotBeEmpty)
  3062  
  3063  			cblob, cdigest = GetRandomImageConfig()
  3064  			buf = bytes.NewBuffer(cblob)
  3065  			buflen = buf.Len()
  3066  			blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf)
  3067  			So(err, ShouldBeNil)
  3068  			So(blob, ShouldEqual, buflen)
  3069  
  3070  			err = imgStore.FinishBlobUpload("index", upload, buf, cdigest)
  3071  			So(err, ShouldBeNil)
  3072  			So(blob, ShouldEqual, buflen)
  3073  
  3074  			// create a manifest
  3075  			manifest := ispec.Manifest{
  3076  				Config: ispec.Descriptor{
  3077  					MediaType: ispec.MediaTypeImageConfig,
  3078  					Digest:    cdigest,
  3079  					Size:      int64(len(cblob)),
  3080  				},
  3081  				Layers: []ispec.Descriptor{
  3082  					{
  3083  						MediaType: ispec.MediaTypeImageLayer,
  3084  						Digest:    bdgst1,
  3085  						Size:      int64(bsize1),
  3086  					},
  3087  				},
  3088  			}
  3089  			manifest.SchemaVersion = 2
  3090  			content, err = json.Marshal(manifest)
  3091  			So(err, ShouldBeNil)
  3092  			digest = godigest.FromBytes(content)
  3093  			So(digest, ShouldNotBeNil)
  3094  			_, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content)
  3095  			So(err, ShouldBeNil)
  3096  
  3097  			var index ispec.Index
  3098  			index.SchemaVersion = 2
  3099  			index.Manifests = []ispec.Descriptor{
  3100  				{
  3101  					MediaType: ispec.MediaTypeImageIndex,
  3102  					Digest:    digest,
  3103  					Size:      int64(len(content)),
  3104  				},
  3105  				{
  3106  					MediaType: ispec.MediaTypeImageIndex,
  3107  					Digest:    m2dgst,
  3108  					Size:      int64(m2size),
  3109  				},
  3110  			}
  3111  
  3112  			content, err = json.Marshal(index)
  3113  			So(err, ShouldBeNil)
  3114  			digest = godigest.FromBytes(content)
  3115  			So(digest, ShouldNotBeNil)
  3116  			index1dgst := digest
  3117  			_, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content)
  3118  			So(err, ShouldBeNil)
  3119  			_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3120  			So(err, ShouldBeNil)
  3121  
  3122  			// upload another image config blob
  3123  			upload, err = imgStore.NewBlobUpload("index")
  3124  			So(err, ShouldBeNil)
  3125  			So(upload, ShouldNotBeEmpty)
  3126  
  3127  			cblob, cdigest = GetRandomImageConfig()
  3128  			buf = bytes.NewBuffer(cblob)
  3129  			buflen = buf.Len()
  3130  			blob, err = imgStore.PutBlobChunkStreamed("index", upload, buf)
  3131  			So(err, ShouldBeNil)
  3132  			So(blob, ShouldEqual, buflen)
  3133  
  3134  			err = imgStore.FinishBlobUpload("index", upload, buf, cdigest)
  3135  			So(err, ShouldBeNil)
  3136  			So(blob, ShouldEqual, buflen)
  3137  
  3138  			// create another manifest
  3139  			manifest = ispec.Manifest{
  3140  				Config: ispec.Descriptor{
  3141  					MediaType: ispec.MediaTypeImageConfig,
  3142  					Digest:    cdigest,
  3143  					Size:      int64(len(cblob)),
  3144  				},
  3145  				Layers: []ispec.Descriptor{
  3146  					{
  3147  						MediaType: ispec.MediaTypeImageLayer,
  3148  						Digest:    bdgst1,
  3149  						Size:      int64(bsize1),
  3150  					},
  3151  				},
  3152  			}
  3153  			manifest.SchemaVersion = 2
  3154  			content, err = json.Marshal(manifest)
  3155  			So(err, ShouldBeNil)
  3156  			digest = godigest.FromBytes(content)
  3157  			So(digest, ShouldNotBeNil)
  3158  			m4dgst := digest
  3159  			m4size := len(content)
  3160  			_, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content)
  3161  			So(err, ShouldBeNil)
  3162  
  3163  			index.SchemaVersion = 2
  3164  			index.Manifests = []ispec.Descriptor{
  3165  				{
  3166  					MediaType: ispec.MediaTypeImageIndex,
  3167  					Digest:    digest,
  3168  					Size:      int64(len(content)),
  3169  				},
  3170  				{
  3171  					MediaType: ispec.MediaTypeImageIndex,
  3172  					Digest:    m2dgst,
  3173  					Size:      int64(m2size),
  3174  				},
  3175  			}
  3176  
  3177  			content, err = json.Marshal(index)
  3178  			So(err, ShouldBeNil)
  3179  			digest = godigest.FromBytes(content)
  3180  			So(digest, ShouldNotBeNil)
  3181  			_, _, err = imgStore.PutImageManifest("index", "test:index2", ispec.MediaTypeImageIndex, content)
  3182  			So(err, ShouldBeNil)
  3183  			_, _, _, err = imgStore.GetImageManifest("index", "test:index2")
  3184  			So(err, ShouldBeNil)
  3185  
  3186  			Convey("List tags", func() {
  3187  				tags, err := imgStore.GetImageTags("index")
  3188  				So(err, ShouldBeNil)
  3189  				So(len(tags), ShouldEqual, 3)
  3190  				So(tags, ShouldContain, "test:1.0")
  3191  				So(tags, ShouldContain, "test:index1")
  3192  				So(tags, ShouldContain, "test:index2")
  3193  			})
  3194  
  3195  			Convey("Another index with same manifest", func() {
  3196  				var index ispec.Index
  3197  				index.SchemaVersion = 2
  3198  				index.Manifests = []ispec.Descriptor{
  3199  					{
  3200  						MediaType: ispec.MediaTypeImageIndex,
  3201  						Digest:    m4dgst,
  3202  						Size:      int64(m4size),
  3203  					},
  3204  				}
  3205  
  3206  				content, err = json.Marshal(index)
  3207  				So(err, ShouldBeNil)
  3208  				digest = godigest.FromBytes(content)
  3209  				So(digest, ShouldNotBeNil)
  3210  				_, _, err = imgStore.PutImageManifest("index", "test:index3", ispec.MediaTypeImageIndex, content)
  3211  				So(err, ShouldBeNil)
  3212  				_, _, _, err = imgStore.GetImageManifest("index", "test:index3")
  3213  				So(err, ShouldBeNil)
  3214  			})
  3215  
  3216  			Convey("Another index using digest with same manifest", func() {
  3217  				var index ispec.Index
  3218  				index.SchemaVersion = 2
  3219  				index.Manifests = []ispec.Descriptor{
  3220  					{
  3221  						MediaType: ispec.MediaTypeImageIndex,
  3222  						Digest:    m4dgst,
  3223  						Size:      int64(m4size),
  3224  					},
  3225  				}
  3226  
  3227  				content, err = json.Marshal(index)
  3228  				So(err, ShouldBeNil)
  3229  				digest = godigest.FromBytes(content)
  3230  				So(digest, ShouldNotBeNil)
  3231  				_, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageIndex, content)
  3232  				So(err, ShouldBeNil)
  3233  				_, _, _, err = imgStore.GetImageManifest("index", digest.String())
  3234  				So(err, ShouldBeNil)
  3235  			})
  3236  
  3237  			Convey("Deleting an image index", func() {
  3238  				// delete manifest by tag should pass
  3239  				err := imgStore.DeleteImageManifest("index", "test:index3", false)
  3240  				So(err, ShouldNotBeNil)
  3241  				_, _, _, err = imgStore.GetImageManifest("index", "test:index3")
  3242  				So(err, ShouldNotBeNil)
  3243  
  3244  				err = imgStore.DeleteImageManifest("index", "test:index1", false)
  3245  				So(err, ShouldBeNil)
  3246  
  3247  				_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3248  				So(err, ShouldNotBeNil)
  3249  
  3250  				_, _, _, err = imgStore.GetImageManifest("index", "test:index2")
  3251  				So(err, ShouldBeNil)
  3252  			})
  3253  
  3254  			Convey("Deleting an image index by digest", func() {
  3255  				// delete manifest by tag should pass
  3256  				err := imgStore.DeleteImageManifest("index", "test:index3", false)
  3257  				So(err, ShouldNotBeNil)
  3258  				_, _, _, err = imgStore.GetImageManifest("index", "test:index3")
  3259  				So(err, ShouldNotBeNil)
  3260  
  3261  				err = imgStore.DeleteImageManifest("index", index1dgst.String(), false)
  3262  				So(err, ShouldBeNil)
  3263  				_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3264  				So(err, ShouldNotBeNil)
  3265  
  3266  				_, _, _, err = imgStore.GetImageManifest("index", "test:index2")
  3267  				So(err, ShouldBeNil)
  3268  			})
  3269  
  3270  			Convey("Update an index tag with different manifest", func() {
  3271  				// create a blob/layer
  3272  				upload, err := imgStore.NewBlobUpload("index")
  3273  				So(err, ShouldBeNil)
  3274  				So(upload, ShouldNotBeEmpty)
  3275  
  3276  				content := []byte("this is another blob")
  3277  				buf := bytes.NewBuffer(content)
  3278  				buflen := buf.Len()
  3279  				digest := godigest.FromBytes(content)
  3280  				So(digest, ShouldNotBeNil)
  3281  				blob, err := imgStore.PutBlobChunkStreamed("index", upload, buf)
  3282  				So(err, ShouldBeNil)
  3283  				So(blob, ShouldEqual, buflen)
  3284  
  3285  				err = imgStore.FinishBlobUpload("index", upload, buf, digest)
  3286  				So(err, ShouldBeNil)
  3287  				So(blob, ShouldEqual, buflen)
  3288  
  3289  				// create a manifest with same blob but a different tag
  3290  				manifest = ispec.Manifest{
  3291  					Config: ispec.Descriptor{
  3292  						MediaType: ispec.MediaTypeImageConfig,
  3293  						Digest:    cdigest,
  3294  						Size:      int64(len(cblob)),
  3295  					},
  3296  					Layers: []ispec.Descriptor{
  3297  						{
  3298  							MediaType: ispec.MediaTypeImageLayer,
  3299  							Digest:    digest,
  3300  							Size:      int64(len(content)),
  3301  						},
  3302  					},
  3303  				}
  3304  				manifest.SchemaVersion = 2
  3305  				content, err = json.Marshal(manifest)
  3306  				So(err, ShouldBeNil)
  3307  				digest = godigest.FromBytes(content)
  3308  				So(digest, ShouldNotBeNil)
  3309  				_, _, err = imgStore.PutImageManifest("index", digest.String(), ispec.MediaTypeImageManifest, content)
  3310  				So(err, ShouldBeNil)
  3311  				_, _, _, err = imgStore.GetImageManifest("index", digest.String())
  3312  				So(err, ShouldBeNil)
  3313  
  3314  				index.SchemaVersion = 2
  3315  				index.Manifests = []ispec.Descriptor{
  3316  					{
  3317  						MediaType: ispec.MediaTypeImageIndex,
  3318  						Digest:    digest,
  3319  						Size:      int64(len(content)),
  3320  					},
  3321  				}
  3322  
  3323  				content, err = json.Marshal(index)
  3324  				So(err, ShouldBeNil)
  3325  				digest = godigest.FromBytes(content)
  3326  				So(digest, ShouldNotBeNil)
  3327  				_, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content)
  3328  				So(err, ShouldBeNil)
  3329  				_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3330  				So(err, ShouldBeNil)
  3331  
  3332  				err = imgStore.DeleteImageManifest("index", "test:index1", false)
  3333  				So(err, ShouldBeNil)
  3334  				_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3335  				So(err, ShouldNotBeNil)
  3336  			})
  3337  
  3338  			Convey("Negative test cases", func() {
  3339  				Convey("Delete index", func() {
  3340  					cleanupStorage(storeDriver, path.Join(testDir, "index", "blobs",
  3341  						index1dgst.Algorithm().String(), index1dgst.Encoded()))
  3342  
  3343  					err = imgStore.DeleteImageManifest("index", index1dgst.String(), false)
  3344  					So(err, ShouldNotBeNil)
  3345  					_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3346  					So(err, ShouldNotBeNil)
  3347  				})
  3348  
  3349  				Convey("Corrupt index", func() {
  3350  					wrtr, err := storeDriver.Writer(context.Background(),
  3351  						path.Join(testDir, "index", "blobs",
  3352  							index1dgst.Algorithm().String(), index1dgst.Encoded()),
  3353  						false)
  3354  					So(err, ShouldBeNil)
  3355  					_, err = wrtr.Write([]byte("deadbeef"))
  3356  					So(err, ShouldBeNil)
  3357  					wrtr.Close()
  3358  					err = imgStore.DeleteImageManifest("index", index1dgst.String(), false)
  3359  					So(err, ShouldBeNil)
  3360  					_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3361  					So(err, ShouldNotBeNil)
  3362  				})
  3363  
  3364  				Convey("Change media-type", func() {
  3365  					// previously a manifest, try writing an image index
  3366  					var index ispec.Index
  3367  					index.SchemaVersion = 2
  3368  					index.Manifests = []ispec.Descriptor{
  3369  						{
  3370  							MediaType: ispec.MediaTypeImageIndex,
  3371  							Digest:    m4dgst,
  3372  							Size:      int64(m4size),
  3373  						},
  3374  					}
  3375  
  3376  					content, err = json.Marshal(index)
  3377  					So(err, ShouldBeNil)
  3378  					digest = godigest.FromBytes(content)
  3379  					So(digest, ShouldNotBeNil)
  3380  					_, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageIndex, content)
  3381  					So(err, ShouldNotBeNil)
  3382  
  3383  					// previously an image index, try writing a manifest
  3384  					_, _, err = imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageManifest, m1content)
  3385  					So(err, ShouldNotBeNil)
  3386  				})
  3387  			})
  3388  		})
  3389  	})
  3390  
  3391  	Convey("Test image index as artifact with subject against s3 image store", t, func() {
  3392  		uuid, err := guuid.NewV4()
  3393  		if err != nil {
  3394  			panic(err)
  3395  		}
  3396  
  3397  		testDir := path.Join("/oci-repo-test", uuid.String())
  3398  
  3399  		storeDriver, imgStore, _ := createObjectsStore(testDir, t.TempDir(), true)
  3400  		defer cleanupStorage(storeDriver, testDir)
  3401  
  3402  		// create and upload a blob/layer
  3403  		// create and upload 2 configs
  3404  		// create and upload 2 manifests
  3405  		// index creation/testing is handled in the other conveys
  3406  
  3407  		// layer blob
  3408  		content := []byte("this is a blob1")
  3409  		buf := bytes.NewBuffer(content)
  3410  		buflen := buf.Len()
  3411  		bdigest := godigest.FromBytes(content)
  3412  		bsize := len(content)
  3413  		So(bdigest, ShouldNotBeNil)
  3414  
  3415  		_, clen, err := imgStore.FullBlobUpload("index", buf, bdigest)
  3416  		So(err, ShouldBeNil)
  3417  		So(clen, ShouldEqual, buflen)
  3418  
  3419  		// first config
  3420  		cblob, cdigest := GetRandomImageConfig()
  3421  		buf = bytes.NewBuffer(cblob)
  3422  		buflen = buf.Len()
  3423  
  3424  		_, clen, err = imgStore.FullBlobUpload("index", buf, cdigest)
  3425  		So(err, ShouldBeNil)
  3426  		So(clen, ShouldEqual, buflen)
  3427  
  3428  		// first manifest
  3429  		manifest := ispec.Manifest{
  3430  			Config: ispec.Descriptor{
  3431  				MediaType: ispec.MediaTypeImageConfig,
  3432  				Digest:    cdigest,
  3433  				Size:      int64(len(cblob)),
  3434  			},
  3435  			Layers: []ispec.Descriptor{
  3436  				{
  3437  					MediaType: ispec.MediaTypeImageLayer,
  3438  					Digest:    bdigest,
  3439  					Size:      int64(bsize),
  3440  				},
  3441  			},
  3442  		}
  3443  		manifest.SchemaVersion = 2
  3444  		content, err = json.Marshal(manifest)
  3445  		So(err, ShouldBeNil)
  3446  		m1digest := godigest.FromBytes(content)
  3447  		So(m1digest, ShouldNotBeNil)
  3448  		m1size := len(content)
  3449  
  3450  		_, _, err = imgStore.PutImageManifest("index", "test:1.0", ispec.MediaTypeImageManifest, content)
  3451  		So(err, ShouldBeNil)
  3452  
  3453  		// second config
  3454  		cblob, cdigest = GetRandomImageConfig()
  3455  		buf = bytes.NewBuffer(cblob)
  3456  		buflen = buf.Len()
  3457  
  3458  		_, clen, err = imgStore.FullBlobUpload("index", buf, cdigest)
  3459  		So(err, ShouldBeNil)
  3460  		So(clen, ShouldEqual, buflen)
  3461  
  3462  		// second manifest
  3463  		manifest = ispec.Manifest{
  3464  			Config: ispec.Descriptor{
  3465  				MediaType: ispec.MediaTypeImageConfig,
  3466  				Digest:    cdigest,
  3467  				Size:      int64(len(cblob)),
  3468  			},
  3469  			Layers: []ispec.Descriptor{
  3470  				{
  3471  					MediaType: ispec.MediaTypeImageLayer,
  3472  					Digest:    bdigest,
  3473  					Size:      int64(bsize),
  3474  				},
  3475  			},
  3476  		}
  3477  		manifest.SchemaVersion = 2
  3478  		content, err = json.Marshal(manifest)
  3479  		So(err, ShouldBeNil)
  3480  		m2digest := godigest.FromBytes(content)
  3481  		So(m2digest, ShouldNotBeNil)
  3482  		m2size := len(content)
  3483  		_, _, err = imgStore.PutImageManifest("index", m2digest.String(), ispec.MediaTypeImageManifest, content)
  3484  		So(err, ShouldBeNil)
  3485  
  3486  		Convey("Put image index with valid subject", func() {
  3487  			// create an image index containing the 2nd manifest, having the 1st manifest as subject
  3488  
  3489  			var index ispec.Index
  3490  			index.SchemaVersion = 2
  3491  			index.Manifests = []ispec.Descriptor{
  3492  				{
  3493  					MediaType: ispec.MediaTypeImageManifest,
  3494  					Digest:    m2digest,
  3495  					Size:      int64(m2size),
  3496  				},
  3497  			}
  3498  			index.Subject = &ispec.Descriptor{
  3499  				MediaType: ispec.MediaTypeImageManifest,
  3500  				Digest:    m1digest,
  3501  				Size:      int64(m1size),
  3502  			}
  3503  
  3504  			content, err := json.Marshal(index)
  3505  			So(err, ShouldBeNil)
  3506  			idigest := godigest.FromBytes(content)
  3507  			So(idigest, ShouldNotBeNil)
  3508  
  3509  			digest1, digest2, err := imgStore.PutImageManifest("index", "test:index1", ispec.MediaTypeImageIndex, content)
  3510  			So(err, ShouldBeNil)
  3511  			So(digest1.String(), ShouldEqual, idigest.String())
  3512  			So(digest2.String(), ShouldEqual, m1digest.String())
  3513  
  3514  			_, _, _, err = imgStore.GetImageManifest("index", "test:index1")
  3515  			So(err, ShouldBeNil)
  3516  		})
  3517  	})
  3518  }
  3519  
  3520  func TestS3DedupeErr(t *testing.T) {
  3521  	tskip.SkipS3(t)
  3522  
  3523  	uuid, err := guuid.NewV4()
  3524  	if err != nil {
  3525  		panic(err)
  3526  	}
  3527  
  3528  	testDir := path.Join("/oci-repo-test", uuid.String())
  3529  
  3530  	tdir := t.TempDir()
  3531  
  3532  	storeDriver, imgStore, _ := createObjectsStore(testDir, tdir, true)
  3533  	defer cleanupStorage(storeDriver, testDir)
  3534  
  3535  	Convey("Test DedupeBlob", t, func(c C) {
  3536  		tdir := t.TempDir()
  3537  
  3538  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{})
  3539  
  3540  		err = os.Remove(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
  3541  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest")
  3542  
  3543  		// trigger unable to insert blob record
  3544  		err := imgStore.DedupeBlob("", digest, "", "")
  3545  		So(err, ShouldNotBeNil)
  3546  
  3547  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3548  			MoveFn: func(ctx context.Context, sourcePath string, destPath string) error {
  3549  				return errS3
  3550  			},
  3551  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3552  				return driver.FileInfoInternal{}, errS3
  3553  			},
  3554  		})
  3555  
  3556  		// trigger unable to rename blob
  3557  		err = imgStore.DedupeBlob("", digest, "", "dst")
  3558  		So(err, ShouldNotBeNil)
  3559  
  3560  		// trigger retry
  3561  		err = imgStore.DedupeBlob("", digest, "", "dst")
  3562  		So(err, ShouldNotBeNil)
  3563  	})
  3564  
  3565  	Convey("Test DedupeBlob - error on second store.Stat()", t, func(c C) {
  3566  		tdir := t.TempDir()
  3567  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3568  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3569  				if path == "dst2" {
  3570  					return driver.FileInfoInternal{}, errS3
  3571  				}
  3572  
  3573  				return driver.FileInfoInternal{}, nil
  3574  			},
  3575  		})
  3576  
  3577  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest")
  3578  		err := imgStore.DedupeBlob("", digest, "", "dst")
  3579  		So(err, ShouldBeNil)
  3580  
  3581  		// error will be triggered in driver.SameFile()
  3582  		err = imgStore.DedupeBlob("", digest, "", "dst2")
  3583  		So(err, ShouldBeNil)
  3584  	})
  3585  
  3586  	Convey("Test DedupeBlob - error on store.PutContent()", t, func(c C) {
  3587  		tdir := t.TempDir()
  3588  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3589  			PutContentFn: func(ctx context.Context, path string, content []byte) error {
  3590  				return errS3
  3591  			},
  3592  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3593  				return nil, nil
  3594  			},
  3595  		})
  3596  
  3597  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest")
  3598  		err := imgStore.DedupeBlob("", digest, "", "dst")
  3599  		So(err, ShouldBeNil)
  3600  
  3601  		err = imgStore.DedupeBlob("", digest, "", "dst2")
  3602  		So(err, ShouldNotBeNil)
  3603  	})
  3604  
  3605  	Convey("Test DedupeBlob - error on cache.PutBlob()", t, func(c C) {
  3606  		tdir := t.TempDir()
  3607  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3608  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3609  				return nil, nil
  3610  			},
  3611  		})
  3612  
  3613  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest")
  3614  		err := imgStore.DedupeBlob("", digest, "", "dst")
  3615  		So(err, ShouldBeNil)
  3616  
  3617  		err = imgStore.DedupeBlob("", digest, "", "")
  3618  		So(err, ShouldNotBeNil)
  3619  	})
  3620  
  3621  	Convey("Test DedupeBlob - error on store.Delete()", t, func(c C) {
  3622  		tdir := t.TempDir()
  3623  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3624  			DeleteFn: func(ctx context.Context, path string) error {
  3625  				return errS3
  3626  			},
  3627  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3628  				return nil, nil
  3629  			},
  3630  		})
  3631  
  3632  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, "digest")
  3633  		err := imgStore.DedupeBlob("", digest, "", "dst")
  3634  		So(err, ShouldBeNil)
  3635  
  3636  		err = imgStore.DedupeBlob("", digest, "", "dst")
  3637  		So(err, ShouldNotBeNil)
  3638  	})
  3639  
  3640  	Convey("Test copyBlob() - error on initRepo()", t, func(c C) {
  3641  		tdir := t.TempDir()
  3642  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3643  			PutContentFn: func(ctx context.Context, path string, content []byte) error {
  3644  				return errS3
  3645  			},
  3646  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3647  				return driver.FileInfoInternal{}, errS3
  3648  			},
  3649  			WriterFn: func(ctx context.Context, path string, isAppend bool) (driver.FileWriter, error) {
  3650  				return &FileWriterMock{}, errS3
  3651  			},
  3652  		})
  3653  
  3654  		digest := godigest.NewDigestFromEncoded(godigest.SHA256,
  3655  			"7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc")
  3656  
  3657  		err := imgStore.DedupeBlob("repo", digest, "", "dst")
  3658  		So(err, ShouldBeNil)
  3659  
  3660  		_, _, err = imgStore.CheckBlob("repo", digest)
  3661  		So(err, ShouldNotBeNil)
  3662  	})
  3663  
  3664  	Convey("Test copyBlob() - error on store.PutContent()", t, func(c C) {
  3665  		tdir := t.TempDir()
  3666  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3667  			PutContentFn: func(ctx context.Context, path string, content []byte) error {
  3668  				return errS3
  3669  			},
  3670  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3671  				return driver.FileInfoInternal{}, errS3
  3672  			},
  3673  		})
  3674  
  3675  		digest := godigest.NewDigestFromEncoded(godigest.SHA256,
  3676  			"7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc")
  3677  
  3678  		err := imgStore.DedupeBlob("repo", digest, "", "dst")
  3679  		So(err, ShouldBeNil)
  3680  
  3681  		_, _, err = imgStore.CheckBlob("repo", digest)
  3682  		So(err, ShouldNotBeNil)
  3683  	})
  3684  
  3685  	Convey("Test copyBlob() - error on store.Stat()", t, func(c C) {
  3686  		tdir := t.TempDir()
  3687  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3688  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3689  				return driver.FileInfoInternal{}, errS3
  3690  			},
  3691  		})
  3692  
  3693  		digest := godigest.NewDigestFromEncoded(godigest.SHA256,
  3694  			"7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc")
  3695  
  3696  		err := imgStore.DedupeBlob("repo", digest, "", "dst")
  3697  		So(err, ShouldBeNil)
  3698  
  3699  		_, _, err = imgStore.CheckBlob("repo", digest)
  3700  		So(err, ShouldNotBeNil)
  3701  	})
  3702  
  3703  	Convey("Test GetBlob() - error on second store.Stat()", t, func(c C) {
  3704  		tdir := t.TempDir()
  3705  
  3706  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{})
  3707  
  3708  		digest := godigest.NewDigestFromEncoded(godigest.SHA256,
  3709  			"7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc")
  3710  
  3711  		err := imgStore.DedupeBlob("/src/dst", digest, "", "/repo1/dst1")
  3712  		So(err, ShouldBeNil)
  3713  
  3714  		err = imgStore.DedupeBlob("/src/dst", digest, "", "/repo2/dst2")
  3715  		So(err, ShouldBeNil)
  3716  
  3717  		// copy cache db to the new imagestore
  3718  		input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
  3719  		So(err, ShouldBeNil)
  3720  
  3721  		tdir = t.TempDir()
  3722  
  3723  		err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600)
  3724  		So(err, ShouldBeNil)
  3725  
  3726  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3727  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3728  				if strings.Contains(path, "repo1/dst1") {
  3729  					return driver.FileInfoInternal{}, driver.PathNotFoundError{}
  3730  				}
  3731  
  3732  				return driver.FileInfoInternal{}, nil
  3733  			},
  3734  		})
  3735  
  3736  		_, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip")
  3737  		So(err, ShouldNotBeNil)
  3738  
  3739  		// now it should move content from /repo1/dst1 to /repo2/dst2
  3740  		_, err = imgStore.GetBlobContent("repo2", digest)
  3741  		So(err, ShouldBeNil)
  3742  
  3743  		_, _, _, err = imgStore.StatBlob("repo2", digest)
  3744  		So(err, ShouldBeNil)
  3745  
  3746  		// it errors out because of bad range, as mock store returns a driver.FileInfo with 0 size
  3747  		_, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1)
  3748  		So(err, ShouldNotBeNil)
  3749  	})
  3750  
  3751  	Convey("Test GetBlob() - error on store.Reader()", t, func(c C) {
  3752  		tdir := t.TempDir()
  3753  
  3754  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{})
  3755  
  3756  		digest := godigest.NewDigestFromEncoded(godigest.SHA256,
  3757  			"7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc")
  3758  
  3759  		err := imgStore.DedupeBlob("/src/dst", digest, "", "/repo1/dst1")
  3760  		So(err, ShouldBeNil)
  3761  
  3762  		err = imgStore.DedupeBlob("/src/dst", digest, "", "/repo2/dst2")
  3763  		So(err, ShouldBeNil)
  3764  
  3765  		// copy cache db to the new imagestore
  3766  		input, err := os.ReadFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName))
  3767  		So(err, ShouldBeNil)
  3768  
  3769  		tdir = t.TempDir()
  3770  
  3771  		err = os.WriteFile(path.Join(tdir, storageConstants.BoltdbName+storageConstants.DBExtensionName), input, 0o600)
  3772  		So(err, ShouldBeNil)
  3773  
  3774  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3775  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3776  				return &FileInfoMock{
  3777  					SizeFn: func() int64 {
  3778  						return 0
  3779  					},
  3780  					PathFn: func() string {
  3781  						return "repo1/dst1"
  3782  					},
  3783  				}, nil
  3784  			},
  3785  			ReaderFn: func(ctx context.Context, path string, offset int64) (io.ReadCloser, error) {
  3786  				if strings.Contains(path, "repo1/dst1") {
  3787  					return io.NopCloser(strings.NewReader("")), errS3
  3788  				}
  3789  
  3790  				return io.NopCloser(strings.NewReader("")), nil
  3791  			},
  3792  
  3793  			GetContentFn: func(ctx context.Context, path string) ([]byte, error) {
  3794  				if strings.Contains(path, "repo1/dst1") {
  3795  					return []byte{}, errS3
  3796  				}
  3797  
  3798  				return []byte{}, nil
  3799  			},
  3800  		})
  3801  
  3802  		_, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip")
  3803  		So(err, ShouldNotBeNil)
  3804  
  3805  		_, err = imgStore.GetBlobContent("repo2", digest)
  3806  		So(err, ShouldNotBeNil)
  3807  
  3808  		_, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1)
  3809  		So(err, ShouldNotBeNil)
  3810  	})
  3811  
  3812  	Convey("Test GetBlob() - error on checkCacheBlob()", t, func(c C) {
  3813  		tdir := t.TempDir()
  3814  
  3815  		digest := godigest.NewDigestFromEncoded(godigest.SHA256,
  3816  			"7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc")
  3817  
  3818  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3819  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3820  				return &FileInfoMock{
  3821  					SizeFn: func() int64 {
  3822  						return 0
  3823  					},
  3824  				}, nil
  3825  			},
  3826  		})
  3827  
  3828  		_, _, err = imgStore.GetBlob("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip")
  3829  		So(err, ShouldNotBeNil)
  3830  
  3831  		_, err = imgStore.GetBlobContent("repo2", digest)
  3832  		So(err, ShouldNotBeNil)
  3833  
  3834  		_, _, _, err = imgStore.StatBlob("repo2", digest)
  3835  		So(err, ShouldNotBeNil)
  3836  
  3837  		_, _, _, err = imgStore.GetBlobPartial("repo2", digest, "application/vnd.oci.image.layer.v1.tar+gzip", 0, 1)
  3838  		So(err, ShouldNotBeNil)
  3839  	})
  3840  
  3841  	Convey("Test DeleteBlob() - error on store.Move()", t, func(c C) {
  3842  		tdir := t.TempDir()
  3843  		hash := "7173b809ca12ec5dee4506cd86be934c4596dd234ee82c0662eac04a8c2c71dc" // #nosec G101
  3844  
  3845  		digest := godigest.NewDigestFromEncoded(godigest.SHA256, hash)
  3846  
  3847  		blobPath := path.Join(testDir, "repo/blobs/sha256", hash)
  3848  
  3849  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3850  			MoveFn: func(ctx context.Context, sourcePath, destPath string) error {
  3851  				if destPath == blobPath {
  3852  					return nil
  3853  				}
  3854  
  3855  				return errS3
  3856  			},
  3857  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3858  				if path != blobPath {
  3859  					return nil, errS3
  3860  				}
  3861  
  3862  				return &FileInfoMock{}, nil
  3863  			},
  3864  		})
  3865  
  3866  		err := imgStore.DedupeBlob("repo", digest, "", blobPath)
  3867  		So(err, ShouldBeNil)
  3868  
  3869  		_, _, err = imgStore.CheckBlob("repo2", digest)
  3870  		So(err, ShouldBeNil)
  3871  
  3872  		err = imgStore.DeleteBlob("repo", digest)
  3873  		So(err, ShouldNotBeNil)
  3874  	})
  3875  
  3876  	Convey("Test FullBlobUpload", t, func(c C) {
  3877  		tdir := t.TempDir()
  3878  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3879  			MoveFn: func(ctx context.Context, sourcePath, destPath string) error {
  3880  				return errS3
  3881  			},
  3882  		})
  3883  		d := godigest.FromBytes([]byte(""))
  3884  		_, _, err := imgStore.FullBlobUpload(testImage, io.NopCloser(strings.NewReader("")), d)
  3885  		So(err, ShouldNotBeNil)
  3886  	})
  3887  
  3888  	Convey("Test FinishBlobUpload", t, func(c C) {
  3889  		tdir := t.TempDir()
  3890  		imgStore = createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3891  			MoveFn: func(ctx context.Context, sourcePath, destPath string) error {
  3892  				return errS3
  3893  			},
  3894  		})
  3895  		d := godigest.FromBytes([]byte(""))
  3896  		err := imgStore.FinishBlobUpload(testImage, "uuid", io.NopCloser(strings.NewReader("")), d)
  3897  		So(err, ShouldNotBeNil)
  3898  	})
  3899  }
  3900  
  3901  func TestInjectDedupe(t *testing.T) {
  3902  	tdir := t.TempDir()
  3903  
  3904  	uuid, err := guuid.NewV4()
  3905  	if err != nil {
  3906  		panic(err)
  3907  	}
  3908  
  3909  	testDir := path.Join("/oci-repo-test", uuid.String())
  3910  
  3911  	Convey("Inject errors in DedupeBlob function", t, func() {
  3912  		imgStore := createMockStorage(testDir, tdir, true, &StorageDriverMock{
  3913  			StatFn: func(ctx context.Context, path string) (driver.FileInfo, error) {
  3914  				return &FileInfoMock{}, errS3
  3915  			},
  3916  		})
  3917  		err := imgStore.DedupeBlob("blob", "digest", "", "newblob")
  3918  		So(err, ShouldBeNil)
  3919  
  3920  		injected := inject.InjectFailure(0)
  3921  		err = imgStore.DedupeBlob("blob", "digest", "", "newblob")
  3922  		if injected {
  3923  			So(err, ShouldNotBeNil)
  3924  		} else {
  3925  			So(err, ShouldBeNil)
  3926  		}
  3927  
  3928  		injected = inject.InjectFailure(1)
  3929  		err = imgStore.DedupeBlob("blob", "digest", "", "newblob")
  3930  		if injected {
  3931  			So(err, ShouldNotBeNil)
  3932  		} else {
  3933  			So(err, ShouldBeNil)
  3934  		}
  3935  	})
  3936  }