storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/fs-v1-multipart_test.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016, 2017 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"os"
    23  	"path/filepath"
    24  	"runtime"
    25  	"sync"
    26  	"testing"
    27  	"time"
    28  )
    29  
    30  // Tests cleanup multipart uploads for filesystem backend.
    31  func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
    32  	// Prepare for tests
    33  	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
    34  	defer os.RemoveAll(disk)
    35  
    36  	obj := initFSObjects(disk, t)
    37  	fs := obj.(*FSObjects)
    38  
    39  	bucketName := "bucket"
    40  	objectName := "object"
    41  
    42  	// Create a context we can cancel.
    43  	ctx, cancel := context.WithCancel(GlobalContext)
    44  	obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{})
    45  
    46  	uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{})
    47  	if err != nil {
    48  		t.Fatal("Unexpected err: ", err)
    49  	}
    50  
    51  	var cleanupWg sync.WaitGroup
    52  	cleanupWg.Add(1)
    53  	go func() {
    54  		defer cleanupWg.Done()
    55  		fs.cleanupStaleUploads(ctx, time.Millisecond, 0)
    56  	}()
    57  
    58  	// Wait for 100ms such that - we have given enough time for
    59  	// cleanup routine to kick in. Flaky on slow systems...
    60  	time.Sleep(100 * time.Millisecond)
    61  	cancel()
    62  	cleanupWg.Wait()
    63  
    64  	// Check if upload id was already purged.
    65  	if err = obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, ObjectOptions{}); err != nil {
    66  		if _, ok := err.(InvalidUploadID); !ok {
    67  			t.Fatal("Unexpected err: ", err)
    68  		}
    69  	} else {
    70  		t.Error("Item was not cleaned up.")
    71  	}
    72  }
    73  
    74  // TestNewMultipartUploadFaultyDisk - test NewMultipartUpload with faulty disks
    75  func TestNewMultipartUploadFaultyDisk(t *testing.T) {
    76  	// Prepare for tests
    77  	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
    78  	defer os.RemoveAll(disk)
    79  	obj := initFSObjects(disk, t)
    80  
    81  	fs := obj.(*FSObjects)
    82  	bucketName := "bucket"
    83  	objectName := "object"
    84  
    85  	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
    86  		t.Fatal("Cannot create bucket, err: ", err)
    87  	}
    88  
    89  	// Test with disk removed.
    90  	os.RemoveAll(disk)
    91  	if _, err := fs.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}); err != nil {
    92  		if !isSameType(err, BucketNotFound{}) {
    93  			t.Fatal("Unexpected error ", err)
    94  		}
    95  	}
    96  }
    97  
    98  // TestPutObjectPartFaultyDisk - test PutObjectPart with faulty disks
    99  func TestPutObjectPartFaultyDisk(t *testing.T) {
   100  	// Prepare for tests
   101  	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   102  	defer os.RemoveAll(disk)
   103  	obj := initFSObjects(disk, t)
   104  
   105  	bucketName := "bucket"
   106  	objectName := "object"
   107  	data := []byte("12345")
   108  	dataLen := int64(len(data))
   109  
   110  	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
   111  		t.Fatal("Cannot create bucket, err: ", err)
   112  	}
   113  
   114  	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
   115  	if err != nil {
   116  		t.Fatal("Unexpected error ", err)
   117  	}
   118  
   119  	md5Hex := getMD5Hash(data)
   120  	sha256sum := ""
   121  
   122  	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   123  	defer os.RemoveAll(newDisk)
   124  	obj = initFSObjects(newDisk, t)
   125  	if _, err = obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{}); err != nil {
   126  		if !isSameType(err, BucketNotFound{}) {
   127  			t.Fatal("Unexpected error ", err)
   128  		}
   129  	}
   130  }
   131  
   132  // TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks
   133  func TestCompleteMultipartUploadFaultyDisk(t *testing.T) {
   134  	// Prepare for tests
   135  	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   136  	defer os.RemoveAll(disk)
   137  	obj := initFSObjects(disk, t)
   138  
   139  	bucketName := "bucket"
   140  	objectName := "object"
   141  	data := []byte("12345")
   142  
   143  	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
   144  		t.Fatal("Cannot create bucket, err: ", err)
   145  	}
   146  
   147  	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
   148  	if err != nil {
   149  		t.Fatal("Unexpected error ", err)
   150  	}
   151  
   152  	md5Hex := getMD5Hash(data)
   153  
   154  	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
   155  	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   156  	defer os.RemoveAll(newDisk)
   157  	obj = initFSObjects(newDisk, t)
   158  	if _, err := obj.CompleteMultipartUpload(GlobalContext, bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
   159  		if !isSameType(err, BucketNotFound{}) {
   160  			t.Fatal("Unexpected error ", err)
   161  		}
   162  	}
   163  }
   164  
   165  // TestCompleteMultipartUpload - test CompleteMultipartUpload
   166  func TestCompleteMultipartUpload(t *testing.T) {
   167  	// Prepare for tests
   168  	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   169  	defer os.RemoveAll(disk)
   170  	obj := initFSObjects(disk, t)
   171  
   172  	bucketName := "bucket"
   173  	objectName := "object"
   174  	data := []byte("12345")
   175  
   176  	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
   177  		t.Fatal("Cannot create bucket, err: ", err)
   178  	}
   179  
   180  	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
   181  	if err != nil {
   182  		t.Fatal("Unexpected error ", err)
   183  	}
   184  
   185  	md5Hex := getMD5Hash(data)
   186  
   187  	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil {
   188  		t.Fatal("Unexpected error ", err)
   189  	}
   190  
   191  	parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}}
   192  	if _, err := obj.CompleteMultipartUpload(GlobalContext, bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil {
   193  		t.Fatal("Unexpected error ", err)
   194  	}
   195  }
   196  
   197  // TestCompleteMultipartUpload - test CompleteMultipartUpload
   198  func TestAbortMultipartUpload(t *testing.T) {
   199  	if runtime.GOOS == globalWindowsOSName {
   200  		// Concurrent AbortMultipartUpload() fails on windows
   201  		t.Skip()
   202  	}
   203  
   204  	// Prepare for tests
   205  	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   206  	defer os.RemoveAll(disk)
   207  	obj := initFSObjects(disk, t)
   208  
   209  	bucketName := "bucket"
   210  	objectName := "object"
   211  	data := []byte("12345")
   212  
   213  	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
   214  		t.Fatal("Cannot create bucket, err: ", err)
   215  	}
   216  
   217  	uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
   218  	if err != nil {
   219  		t.Fatal("Unexpected error ", err)
   220  	}
   221  
   222  	md5Hex := getMD5Hash(data)
   223  
   224  	opts := ObjectOptions{}
   225  	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), opts); err != nil {
   226  		t.Fatal("Unexpected error ", err)
   227  	}
   228  	if err := obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, opts); err != nil {
   229  		t.Fatal("Unexpected error ", err)
   230  	}
   231  }
   232  
   233  // TestListMultipartUploadsFaultyDisk - test ListMultipartUploads with faulty disks
   234  func TestListMultipartUploadsFaultyDisk(t *testing.T) {
   235  	// Prepare for tests
   236  	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   237  	defer os.RemoveAll(disk)
   238  
   239  	obj := initFSObjects(disk, t)
   240  
   241  	bucketName := "bucket"
   242  	objectName := "object"
   243  
   244  	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil {
   245  		t.Fatal("Cannot create bucket, err: ", err)
   246  	}
   247  
   248  	_, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}})
   249  	if err != nil {
   250  		t.Fatal("Unexpected error ", err)
   251  	}
   252  
   253  	newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
   254  	defer os.RemoveAll(newDisk)
   255  	obj = initFSObjects(newDisk, t)
   256  	if _, err := obj.ListMultipartUploads(GlobalContext, bucketName, objectName, "", "", "", 1000); err != nil {
   257  		if !isSameType(err, BucketNotFound{}) {
   258  			t.Fatal("Unexpected error ", err)
   259  		}
   260  	}
   261  }