storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/fs-v1-multipart_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016, 2017 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "context" 22 "os" 23 "path/filepath" 24 "runtime" 25 "sync" 26 "testing" 27 "time" 28 ) 29 30 // Tests cleanup multipart uploads for filesystem backend. 31 func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) { 32 // Prepare for tests 33 disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 34 defer os.RemoveAll(disk) 35 36 obj := initFSObjects(disk, t) 37 fs := obj.(*FSObjects) 38 39 bucketName := "bucket" 40 objectName := "object" 41 42 // Create a context we can cancel. 43 ctx, cancel := context.WithCancel(GlobalContext) 44 obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{}) 45 46 uploadID, err := obj.NewMultipartUpload(ctx, bucketName, objectName, ObjectOptions{}) 47 if err != nil { 48 t.Fatal("Unexpected err: ", err) 49 } 50 51 var cleanupWg sync.WaitGroup 52 cleanupWg.Add(1) 53 go func() { 54 defer cleanupWg.Done() 55 fs.cleanupStaleUploads(ctx, time.Millisecond, 0) 56 }() 57 58 // Wait for 100ms such that - we have given enough time for 59 // cleanup routine to kick in. Flaky on slow systems... 60 time.Sleep(100 * time.Millisecond) 61 cancel() 62 cleanupWg.Wait() 63 64 // Check if upload id was already purged. 65 if err = obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, ObjectOptions{}); err != nil { 66 if _, ok := err.(InvalidUploadID); !ok { 67 t.Fatal("Unexpected err: ", err) 68 } 69 } else { 70 t.Error("Item was not cleaned up.") 71 } 72 } 73 74 // TestNewMultipartUploadFaultyDisk - test NewMultipartUpload with faulty disks 75 func TestNewMultipartUploadFaultyDisk(t *testing.T) { 76 // Prepare for tests 77 disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 78 defer os.RemoveAll(disk) 79 obj := initFSObjects(disk, t) 80 81 fs := obj.(*FSObjects) 82 bucketName := "bucket" 83 objectName := "object" 84 85 if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil { 86 t.Fatal("Cannot create bucket, err: ", err) 87 } 88 89 // Test with disk removed. 90 os.RemoveAll(disk) 91 if _, err := fs.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}); err != nil { 92 if !isSameType(err, BucketNotFound{}) { 93 t.Fatal("Unexpected error ", err) 94 } 95 } 96 } 97 98 // TestPutObjectPartFaultyDisk - test PutObjectPart with faulty disks 99 func TestPutObjectPartFaultyDisk(t *testing.T) { 100 // Prepare for tests 101 disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 102 defer os.RemoveAll(disk) 103 obj := initFSObjects(disk, t) 104 105 bucketName := "bucket" 106 objectName := "object" 107 data := []byte("12345") 108 dataLen := int64(len(data)) 109 110 if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil { 111 t.Fatal("Cannot create bucket, err: ", err) 112 } 113 114 uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}) 115 if err != nil { 116 t.Fatal("Unexpected error ", err) 117 } 118 119 md5Hex := getMD5Hash(data) 120 sha256sum := "" 121 122 newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 123 defer os.RemoveAll(newDisk) 124 obj = initFSObjects(newDisk, t) 125 if _, err = obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), dataLen, md5Hex, sha256sum), ObjectOptions{}); err != nil { 126 if !isSameType(err, BucketNotFound{}) { 127 t.Fatal("Unexpected error ", err) 128 } 129 } 130 } 131 132 // TestCompleteMultipartUploadFaultyDisk - test CompleteMultipartUpload with faulty disks 133 func TestCompleteMultipartUploadFaultyDisk(t *testing.T) { 134 // Prepare for tests 135 disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 136 defer os.RemoveAll(disk) 137 obj := initFSObjects(disk, t) 138 139 bucketName := "bucket" 140 objectName := "object" 141 data := []byte("12345") 142 143 if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil { 144 t.Fatal("Cannot create bucket, err: ", err) 145 } 146 147 uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}) 148 if err != nil { 149 t.Fatal("Unexpected error ", err) 150 } 151 152 md5Hex := getMD5Hash(data) 153 154 parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}} 155 newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 156 defer os.RemoveAll(newDisk) 157 obj = initFSObjects(newDisk, t) 158 if _, err := obj.CompleteMultipartUpload(GlobalContext, bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil { 159 if !isSameType(err, BucketNotFound{}) { 160 t.Fatal("Unexpected error ", err) 161 } 162 } 163 } 164 165 // TestCompleteMultipartUpload - test CompleteMultipartUpload 166 func TestCompleteMultipartUpload(t *testing.T) { 167 // Prepare for tests 168 disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 169 defer os.RemoveAll(disk) 170 obj := initFSObjects(disk, t) 171 172 bucketName := "bucket" 173 objectName := "object" 174 data := []byte("12345") 175 176 if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil { 177 t.Fatal("Cannot create bucket, err: ", err) 178 } 179 180 uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}) 181 if err != nil { 182 t.Fatal("Unexpected error ", err) 183 } 184 185 md5Hex := getMD5Hash(data) 186 187 if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), ObjectOptions{}); err != nil { 188 t.Fatal("Unexpected error ", err) 189 } 190 191 parts := []CompletePart{{PartNumber: 1, ETag: md5Hex}} 192 if _, err := obj.CompleteMultipartUpload(GlobalContext, bucketName, objectName, uploadID, parts, ObjectOptions{}); err != nil { 193 t.Fatal("Unexpected error ", err) 194 } 195 } 196 197 // TestCompleteMultipartUpload - test CompleteMultipartUpload 198 func TestAbortMultipartUpload(t *testing.T) { 199 if runtime.GOOS == globalWindowsOSName { 200 // Concurrent AbortMultipartUpload() fails on windows 201 t.Skip() 202 } 203 204 // Prepare for tests 205 disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 206 defer os.RemoveAll(disk) 207 obj := initFSObjects(disk, t) 208 209 bucketName := "bucket" 210 objectName := "object" 211 data := []byte("12345") 212 213 if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil { 214 t.Fatal("Cannot create bucket, err: ", err) 215 } 216 217 uploadID, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}) 218 if err != nil { 219 t.Fatal("Unexpected error ", err) 220 } 221 222 md5Hex := getMD5Hash(data) 223 224 opts := ObjectOptions{} 225 if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), opts); err != nil { 226 t.Fatal("Unexpected error ", err) 227 } 228 if err := obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, opts); err != nil { 229 t.Fatal("Unexpected error ", err) 230 } 231 } 232 233 // TestListMultipartUploadsFaultyDisk - test ListMultipartUploads with faulty disks 234 func TestListMultipartUploadsFaultyDisk(t *testing.T) { 235 // Prepare for tests 236 disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 237 defer os.RemoveAll(disk) 238 239 obj := initFSObjects(disk, t) 240 241 bucketName := "bucket" 242 objectName := "object" 243 244 if err := obj.MakeBucketWithLocation(GlobalContext, bucketName, BucketOptions{}); err != nil { 245 t.Fatal("Cannot create bucket, err: ", err) 246 } 247 248 _, err := obj.NewMultipartUpload(GlobalContext, bucketName, objectName, ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-xid": "3f"}}) 249 if err != nil { 250 t.Fatal("Unexpected error ", err) 251 } 252 253 newDisk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) 254 defer os.RemoveAll(newDisk) 255 obj = initFSObjects(newDisk, t) 256 if _, err := obj.ListMultipartUploads(GlobalContext, bucketName, objectName, "", "", "", 1000); err != nil { 257 if !isSameType(err, BucketNotFound{}) { 258 t.Fatal("Unexpected error ", err) 259 } 260 } 261 }