storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/erasure-healing_test.go (about) 1 /* 2 * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package cmd 18 19 import ( 20 "bytes" 21 "context" 22 "crypto/rand" 23 "os" 24 "path" 25 "reflect" 26 "testing" 27 "time" 28 29 "github.com/dustin/go-humanize" 30 31 "storj.io/minio/pkg/madmin" 32 ) 33 34 // Tests both object and bucket healing. 35 func TestHealing(t *testing.T) { 36 ctx, cancel := context.WithCancel(context.Background()) 37 defer cancel() 38 39 obj, fsDirs, err := prepareErasure16(ctx) 40 if err != nil { 41 t.Fatal(err) 42 } 43 defer obj.Shutdown(context.Background()) 44 defer removeRoots(fsDirs) 45 46 z := obj.(*erasureServerPools) 47 er := z.serverPools[0].sets[0] 48 49 // Create "bucket" 50 err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{}) 51 if err != nil { 52 t.Fatal(err) 53 } 54 55 bucket := "bucket" 56 object := "object" 57 58 data := make([]byte, 1*humanize.MiByte) 59 length := int64(len(data)) 60 _, err = rand.Read(data) 61 if err != nil { 62 t.Fatal(err) 63 } 64 65 _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), length, "", ""), ObjectOptions{}) 66 if err != nil { 67 t.Fatal(err) 68 } 69 70 disk := er.getDisks()[0] 71 fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false) 72 if err != nil { 73 t.Fatal(err) 74 } 75 76 // Remove the object - to simulate the case where the disk was down when the object 77 // was created. 78 err = removeAll(pathJoin(disk.String(), bucket, object)) 79 if err != nil { 80 t.Fatal(err) 81 } 82 83 _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) 84 if err != nil { 85 t.Fatal(err) 86 } 87 88 fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false) 89 if err != nil { 90 t.Fatal(err) 91 } 92 93 // After heal the meta file should be as expected. 94 if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) { 95 t.Fatal("HealObject failed") 96 } 97 98 err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "er.meta")) 99 if err != nil { 100 t.Fatal(err) 101 } 102 103 // Write er.meta with different modtime to simulate the case where a disk had 104 // gone down when an object was replaced by a new object. 105 fileInfoOutDated := fileInfoPreHeal 106 fileInfoOutDated.ModTime = time.Now() 107 err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated) 108 if err != nil { 109 t.Fatal(err) 110 } 111 112 _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan}) 113 if err != nil { 114 t.Fatal(err) 115 } 116 117 fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false) 118 if err != nil { 119 t.Fatal(err) 120 } 121 122 // After heal the meta file should be as expected. 123 if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) { 124 t.Fatal("HealObject failed") 125 } 126 127 // Remove the bucket - to simulate the case where bucket was 128 // created when the disk was down. 129 err = os.RemoveAll(path.Join(fsDirs[0], bucket)) 130 if err != nil { 131 t.Fatal(err) 132 } 133 // This would create the bucket. 134 _, err = er.HealBucket(ctx, bucket, madmin.HealOpts{ 135 DryRun: false, 136 Remove: false, 137 }) 138 if err != nil { 139 t.Fatal(err) 140 } 141 // Stat the bucket to make sure that it was created. 142 _, err = er.getDisks()[0].StatVol(context.Background(), bucket) 143 if err != nil { 144 t.Fatal(err) 145 } 146 } 147 148 func TestHealObjectCorrupted(t *testing.T) { 149 ctx, cancel := context.WithCancel(context.Background()) 150 defer cancel() 151 152 resetGlobalHealState() 153 defer resetGlobalHealState() 154 155 nDisks := 16 156 fsDirs, err := getRandomDisks(nDisks) 157 if err != nil { 158 t.Fatal(err) 159 } 160 161 defer removeRoots(fsDirs) 162 163 // Everything is fine, should return nil 164 objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...)) 165 if err != nil { 166 t.Fatal(err) 167 } 168 169 bucket := "bucket" 170 object := "object" 171 data := bytes.Repeat([]byte("a"), 5*1024*1024) 172 var opts ObjectOptions 173 174 err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{}) 175 if err != nil { 176 t.Fatalf("Failed to make a bucket - %v", err) 177 } 178 179 // Create an object with multiple parts uploaded in decreasing 180 // part number. 181 uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts) 182 if err != nil { 183 t.Fatalf("Failed to create a multipart upload - %v", err) 184 } 185 186 var uploadedParts []CompletePart 187 for _, partID := range []int{2, 1} { 188 pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) 189 if err1 != nil { 190 t.Fatalf("Failed to upload a part - %v", err1) 191 } 192 uploadedParts = append(uploadedParts, CompletePart{ 193 PartNumber: pInfo.PartNumber, 194 ETag: pInfo.ETag, 195 }) 196 } 197 198 _, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{}) 199 if err != nil { 200 t.Fatalf("Failed to complete multipart upload - %v", err) 201 } 202 203 // Test 1: Remove the object backend files from the first disk. 204 z := objLayer.(*erasureServerPools) 205 er := z.serverPools[0].sets[0] 206 erasureDisks := er.getDisks() 207 firstDisk := erasureDisks[0] 208 err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false) 209 if err != nil { 210 t.Fatalf("Failed to delete a file - %v", err) 211 } 212 213 _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) 214 if err != nil { 215 t.Fatalf("Failed to heal object - %v", err) 216 } 217 218 fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false) 219 fi, err := getLatestFileInfo(ctx, fileInfos, errs) 220 if err != nil { 221 t.Fatalf("Failed to getLatestFileInfo - %v", err) 222 } 223 224 if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil { 225 t.Errorf("Expected er.meta file to be present but stat failed - %v", err) 226 } 227 228 err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false) 229 if err != nil { 230 t.Errorf("Failure during deleting part.1 - %v", err) 231 } 232 233 err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{}) 234 if err != nil { 235 t.Errorf("Failure during creating part.1 - %v", err) 236 } 237 238 _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan}) 239 if err != nil { 240 t.Errorf("Expected nil but received %v", err) 241 } 242 243 fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false) 244 nfi, err := getLatestFileInfo(ctx, fileInfos, errs) 245 if err != nil { 246 t.Fatalf("Failed to getLatestFileInfo - %v", err) 247 } 248 249 if !reflect.DeepEqual(fi, nfi) { 250 t.Fatalf("FileInfo not equal after healing") 251 } 252 253 err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false) 254 if err != nil { 255 t.Errorf("Failure during deleting part.1 - %v", err) 256 } 257 258 bdata := bytes.Repeat([]byte("b"), int(nfi.Size)) 259 err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata) 260 if err != nil { 261 t.Errorf("Failure during creating part.1 - %v", err) 262 } 263 264 _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan}) 265 if err != nil { 266 t.Errorf("Expected nil but received %v", err) 267 } 268 269 fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false) 270 nfi, err = getLatestFileInfo(ctx, fileInfos, errs) 271 if err != nil { 272 t.Fatalf("Failed to getLatestFileInfo - %v", err) 273 } 274 275 if !reflect.DeepEqual(fi, nfi) { 276 t.Fatalf("FileInfo not equal after healing") 277 } 278 279 // Test 4: checks if HealObject returns an error when xl.meta is not found 280 // in more than read quorum number of disks, to create a corrupted situation. 281 for i := 0; i <= nfi.Erasure.DataBlocks; i++ { 282 erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false) 283 } 284 285 // Try healing now, expect to receive errFileNotFound. 286 _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan}) 287 if err != nil { 288 if _, ok := err.(ObjectNotFound); !ok { 289 t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err) 290 } 291 } 292 293 // since majority of xl.meta's are not available, object should be successfully deleted. 294 _, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) 295 if _, ok := err.(ObjectNotFound); !ok { 296 t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err) 297 } 298 } 299 300 // Tests healing of object. 301 func TestHealObjectErasure(t *testing.T) { 302 ctx, cancel := context.WithCancel(context.Background()) 303 defer cancel() 304 305 nDisks := 16 306 fsDirs, err := getRandomDisks(nDisks) 307 if err != nil { 308 t.Fatal(err) 309 } 310 311 defer removeRoots(fsDirs) 312 313 // Everything is fine, should return nil 314 obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...)) 315 if err != nil { 316 t.Fatal(err) 317 } 318 319 bucket := "bucket" 320 object := "object" 321 data := bytes.Repeat([]byte("a"), 5*1024*1024) 322 var opts ObjectOptions 323 324 err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{}) 325 if err != nil { 326 t.Fatalf("Failed to make a bucket - %v", err) 327 } 328 329 // Create an object with multiple parts uploaded in decreasing 330 // part number. 331 uploadID, err := obj.NewMultipartUpload(ctx, bucket, object, opts) 332 if err != nil { 333 t.Fatalf("Failed to create a multipart upload - %v", err) 334 } 335 336 var uploadedParts []CompletePart 337 for _, partID := range []int{2, 1} { 338 pInfo, err1 := obj.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) 339 if err1 != nil { 340 t.Fatalf("Failed to upload a part - %v", err1) 341 } 342 uploadedParts = append(uploadedParts, CompletePart{ 343 PartNumber: pInfo.PartNumber, 344 ETag: pInfo.ETag, 345 }) 346 } 347 348 // Remove the object backend files from the first disk. 349 z := obj.(*erasureServerPools) 350 er := z.serverPools[0].sets[0] 351 firstDisk := er.getDisks()[0] 352 353 _, err = obj.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{}) 354 if err != nil { 355 t.Fatalf("Failed to complete multipart upload - %v", err) 356 } 357 358 err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false) 359 if err != nil { 360 t.Fatalf("Failed to delete a file - %v", err) 361 } 362 363 _, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) 364 if err != nil { 365 t.Fatalf("Failed to heal object - %v", err) 366 } 367 368 if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil { 369 t.Errorf("Expected er.meta file to be present but stat failed - %v", err) 370 } 371 372 erasureDisks := er.getDisks() 373 z.serverPools[0].erasureDisksMu.Lock() 374 er.getDisks = func() []StorageAPI { 375 // Nil more than half the disks, to remove write quorum. 376 for i := 0; i <= len(erasureDisks)/2; i++ { 377 erasureDisks[i] = nil 378 } 379 return erasureDisks 380 } 381 z.serverPools[0].erasureDisksMu.Unlock() 382 383 // Try healing now, expect to receive errDiskNotFound. 384 _, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan}) 385 // since majority of er.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum 386 if _, ok := err.(InsufficientReadQuorum); !ok { 387 t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err) 388 } 389 } 390 391 // Tests healing of empty directories 392 func TestHealEmptyDirectoryErasure(t *testing.T) { 393 ctx, cancel := context.WithCancel(context.Background()) 394 defer cancel() 395 396 nDisks := 16 397 fsDirs, err := getRandomDisks(nDisks) 398 if err != nil { 399 t.Fatal(err) 400 } 401 defer removeRoots(fsDirs) 402 403 // Everything is fine, should return nil 404 obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...)) 405 if err != nil { 406 t.Fatal(err) 407 } 408 409 bucket := "bucket" 410 object := "empty-dir/" 411 var opts ObjectOptions 412 413 err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{}) 414 if err != nil { 415 t.Fatalf("Failed to make a bucket - %v", err) 416 } 417 418 // Upload an empty directory 419 _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, 420 bytes.NewReader([]byte{}), 0, "", ""), opts) 421 if err != nil { 422 t.Fatal(err) 423 } 424 425 // Remove the object backend files from the first disk. 426 z := obj.(*erasureServerPools) 427 er := z.serverPools[0].sets[0] 428 firstDisk := er.getDisks()[0] 429 err = firstDisk.DeleteVol(context.Background(), pathJoin(bucket, encodeDirObject(object)), true) 430 if err != nil { 431 t.Fatalf("Failed to delete a file - %v", err) 432 } 433 434 // Heal the object 435 hr, err := obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) 436 if err != nil { 437 t.Fatalf("Failed to heal object - %v", err) 438 } 439 440 // Check if the empty directory is restored in the first disk 441 _, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, encodeDirObject(object))) 442 if err != nil { 443 t.Fatalf("Expected object to be present but stat failed - %v", err) 444 } 445 446 // Check the state of the object in the first disk (should be missing) 447 if hr.Before.Drives[0].State != madmin.DriveStateMissing { 448 t.Fatalf("Unexpected drive state: %v", hr.Before.Drives[0].State) 449 } 450 451 // Check the state of all other disks (should be ok) 452 for i, h := range append(hr.Before.Drives[1:], hr.After.Drives...) { 453 if h.State != madmin.DriveStateOk { 454 t.Fatalf("Unexpected drive state (%d): %v", i+1, h.State) 455 } 456 } 457 458 // Heal the same object again 459 hr, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) 460 if err != nil { 461 t.Fatalf("Failed to heal object - %v", err) 462 } 463 464 // Check that Before & After states are all okay 465 for i, h := range append(hr.Before.Drives, hr.After.Drives...) { 466 if h.State != madmin.DriveStateOk { 467 t.Fatalf("Unexpected drive state (%d): %v", i+1, h.State) 468 } 469 } 470 }