storj.io/minio@v0.0.0-20230509071714-0cbc90f649b1/cmd/erasure-healing_test.go (about)

     1  /*
     2   * MinIO Cloud Storage, (C) 2016-2020 MinIO, Inc.
     3   *
     4   * Licensed under the Apache License, Version 2.0 (the "License");
     5   * you may not use this file except in compliance with the License.
     6   * You may obtain a copy of the License at
     7   *
     8   *     http://www.apache.org/licenses/LICENSE-2.0
     9   *
    10   * Unless required by applicable law or agreed to in writing, software
    11   * distributed under the License is distributed on an "AS IS" BASIS,
    12   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13   * See the License for the specific language governing permissions and
    14   * limitations under the License.
    15   */
    16  
    17  package cmd
    18  
    19  import (
    20  	"bytes"
    21  	"context"
    22  	"crypto/rand"
    23  	"os"
    24  	"path"
    25  	"reflect"
    26  	"testing"
    27  	"time"
    28  
    29  	"github.com/dustin/go-humanize"
    30  
    31  	"storj.io/minio/pkg/madmin"
    32  )
    33  
    34  // Tests both object and bucket healing.
    35  func TestHealing(t *testing.T) {
    36  	ctx, cancel := context.WithCancel(context.Background())
    37  	defer cancel()
    38  
    39  	obj, fsDirs, err := prepareErasure16(ctx)
    40  	if err != nil {
    41  		t.Fatal(err)
    42  	}
    43  	defer obj.Shutdown(context.Background())
    44  	defer removeRoots(fsDirs)
    45  
    46  	z := obj.(*erasureServerPools)
    47  	er := z.serverPools[0].sets[0]
    48  
    49  	// Create "bucket"
    50  	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
    51  	if err != nil {
    52  		t.Fatal(err)
    53  	}
    54  
    55  	bucket := "bucket"
    56  	object := "object"
    57  
    58  	data := make([]byte, 1*humanize.MiByte)
    59  	length := int64(len(data))
    60  	_, err = rand.Read(data)
    61  	if err != nil {
    62  		t.Fatal(err)
    63  	}
    64  
    65  	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), length, "", ""), ObjectOptions{})
    66  	if err != nil {
    67  		t.Fatal(err)
    68  	}
    69  
    70  	disk := er.getDisks()[0]
    71  	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false)
    72  	if err != nil {
    73  		t.Fatal(err)
    74  	}
    75  
    76  	// Remove the object - to simulate the case where the disk was down when the object
    77  	// was created.
    78  	err = removeAll(pathJoin(disk.String(), bucket, object))
    79  	if err != nil {
    80  		t.Fatal(err)
    81  	}
    82  
    83  	_, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
    84  	if err != nil {
    85  		t.Fatal(err)
    86  	}
    87  
    88  	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false)
    89  	if err != nil {
    90  		t.Fatal(err)
    91  	}
    92  
    93  	// After heal the meta file should be as expected.
    94  	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
    95  		t.Fatal("HealObject failed")
    96  	}
    97  
    98  	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "er.meta"))
    99  	if err != nil {
   100  		t.Fatal(err)
   101  	}
   102  
   103  	// Write er.meta with different modtime to simulate the case where a disk had
   104  	// gone down when an object was replaced by a new object.
   105  	fileInfoOutDated := fileInfoPreHeal
   106  	fileInfoOutDated.ModTime = time.Now()
   107  	err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
   108  	if err != nil {
   109  		t.Fatal(err)
   110  	}
   111  
   112  	_, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
   113  	if err != nil {
   114  		t.Fatal(err)
   115  	}
   116  
   117  	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
   118  	if err != nil {
   119  		t.Fatal(err)
   120  	}
   121  
   122  	// After heal the meta file should be as expected.
   123  	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
   124  		t.Fatal("HealObject failed")
   125  	}
   126  
   127  	// Remove the bucket - to simulate the case where bucket was
   128  	// created when the disk was down.
   129  	err = os.RemoveAll(path.Join(fsDirs[0], bucket))
   130  	if err != nil {
   131  		t.Fatal(err)
   132  	}
   133  	// This would create the bucket.
   134  	_, err = er.HealBucket(ctx, bucket, madmin.HealOpts{
   135  		DryRun: false,
   136  		Remove: false,
   137  	})
   138  	if err != nil {
   139  		t.Fatal(err)
   140  	}
   141  	// Stat the bucket to make sure that it was created.
   142  	_, err = er.getDisks()[0].StatVol(context.Background(), bucket)
   143  	if err != nil {
   144  		t.Fatal(err)
   145  	}
   146  }
   147  
   148  func TestHealObjectCorrupted(t *testing.T) {
   149  	ctx, cancel := context.WithCancel(context.Background())
   150  	defer cancel()
   151  
   152  	resetGlobalHealState()
   153  	defer resetGlobalHealState()
   154  
   155  	nDisks := 16
   156  	fsDirs, err := getRandomDisks(nDisks)
   157  	if err != nil {
   158  		t.Fatal(err)
   159  	}
   160  
   161  	defer removeRoots(fsDirs)
   162  
   163  	// Everything is fine, should return nil
   164  	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
   165  	if err != nil {
   166  		t.Fatal(err)
   167  	}
   168  
   169  	bucket := "bucket"
   170  	object := "object"
   171  	data := bytes.Repeat([]byte("a"), 5*1024*1024)
   172  	var opts ObjectOptions
   173  
   174  	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
   175  	if err != nil {
   176  		t.Fatalf("Failed to make a bucket - %v", err)
   177  	}
   178  
   179  	// Create an object with multiple parts uploaded in decreasing
   180  	// part number.
   181  	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
   182  	if err != nil {
   183  		t.Fatalf("Failed to create a multipart upload - %v", err)
   184  	}
   185  
   186  	var uploadedParts []CompletePart
   187  	for _, partID := range []int{2, 1} {
   188  		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
   189  		if err1 != nil {
   190  			t.Fatalf("Failed to upload a part - %v", err1)
   191  		}
   192  		uploadedParts = append(uploadedParts, CompletePart{
   193  			PartNumber: pInfo.PartNumber,
   194  			ETag:       pInfo.ETag,
   195  		})
   196  	}
   197  
   198  	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
   199  	if err != nil {
   200  		t.Fatalf("Failed to complete multipart upload - %v", err)
   201  	}
   202  
   203  	// Test 1: Remove the object backend files from the first disk.
   204  	z := objLayer.(*erasureServerPools)
   205  	er := z.serverPools[0].sets[0]
   206  	erasureDisks := er.getDisks()
   207  	firstDisk := erasureDisks[0]
   208  	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
   209  	if err != nil {
   210  		t.Fatalf("Failed to delete a file - %v", err)
   211  	}
   212  
   213  	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
   214  	if err != nil {
   215  		t.Fatalf("Failed to heal object - %v", err)
   216  	}
   217  
   218  	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
   219  	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
   220  	if err != nil {
   221  		t.Fatalf("Failed to getLatestFileInfo - %v", err)
   222  	}
   223  
   224  	if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
   225  		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
   226  	}
   227  
   228  	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
   229  	if err != nil {
   230  		t.Errorf("Failure during deleting part.1 - %v", err)
   231  	}
   232  
   233  	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
   234  	if err != nil {
   235  		t.Errorf("Failure during creating part.1 - %v", err)
   236  	}
   237  
   238  	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
   239  	if err != nil {
   240  		t.Errorf("Expected nil but received %v", err)
   241  	}
   242  
   243  	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
   244  	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
   245  	if err != nil {
   246  		t.Fatalf("Failed to getLatestFileInfo - %v", err)
   247  	}
   248  
   249  	if !reflect.DeepEqual(fi, nfi) {
   250  		t.Fatalf("FileInfo not equal after healing")
   251  	}
   252  
   253  	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
   254  	if err != nil {
   255  		t.Errorf("Failure during deleting part.1 - %v", err)
   256  	}
   257  
   258  	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
   259  	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
   260  	if err != nil {
   261  		t.Errorf("Failure during creating part.1 - %v", err)
   262  	}
   263  
   264  	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
   265  	if err != nil {
   266  		t.Errorf("Expected nil but received %v", err)
   267  	}
   268  
   269  	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
   270  	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
   271  	if err != nil {
   272  		t.Fatalf("Failed to getLatestFileInfo - %v", err)
   273  	}
   274  
   275  	if !reflect.DeepEqual(fi, nfi) {
   276  		t.Fatalf("FileInfo not equal after healing")
   277  	}
   278  
   279  	// Test 4: checks if HealObject returns an error when xl.meta is not found
   280  	// in more than read quorum number of disks, to create a corrupted situation.
   281  	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
   282  		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
   283  	}
   284  
   285  	// Try healing now, expect to receive errFileNotFound.
   286  	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
   287  	if err != nil {
   288  		if _, ok := err.(ObjectNotFound); !ok {
   289  			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
   290  		}
   291  	}
   292  
   293  	// since majority of xl.meta's are not available, object should be successfully deleted.
   294  	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
   295  	if _, ok := err.(ObjectNotFound); !ok {
   296  		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
   297  	}
   298  }
   299  
   300  // Tests healing of object.
   301  func TestHealObjectErasure(t *testing.T) {
   302  	ctx, cancel := context.WithCancel(context.Background())
   303  	defer cancel()
   304  
   305  	nDisks := 16
   306  	fsDirs, err := getRandomDisks(nDisks)
   307  	if err != nil {
   308  		t.Fatal(err)
   309  	}
   310  
   311  	defer removeRoots(fsDirs)
   312  
   313  	// Everything is fine, should return nil
   314  	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
   315  	if err != nil {
   316  		t.Fatal(err)
   317  	}
   318  
   319  	bucket := "bucket"
   320  	object := "object"
   321  	data := bytes.Repeat([]byte("a"), 5*1024*1024)
   322  	var opts ObjectOptions
   323  
   324  	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
   325  	if err != nil {
   326  		t.Fatalf("Failed to make a bucket - %v", err)
   327  	}
   328  
   329  	// Create an object with multiple parts uploaded in decreasing
   330  	// part number.
   331  	uploadID, err := obj.NewMultipartUpload(ctx, bucket, object, opts)
   332  	if err != nil {
   333  		t.Fatalf("Failed to create a multipart upload - %v", err)
   334  	}
   335  
   336  	var uploadedParts []CompletePart
   337  	for _, partID := range []int{2, 1} {
   338  		pInfo, err1 := obj.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
   339  		if err1 != nil {
   340  			t.Fatalf("Failed to upload a part - %v", err1)
   341  		}
   342  		uploadedParts = append(uploadedParts, CompletePart{
   343  			PartNumber: pInfo.PartNumber,
   344  			ETag:       pInfo.ETag,
   345  		})
   346  	}
   347  
   348  	// Remove the object backend files from the first disk.
   349  	z := obj.(*erasureServerPools)
   350  	er := z.serverPools[0].sets[0]
   351  	firstDisk := er.getDisks()[0]
   352  
   353  	_, err = obj.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
   354  	if err != nil {
   355  		t.Fatalf("Failed to complete multipart upload - %v", err)
   356  	}
   357  
   358  	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
   359  	if err != nil {
   360  		t.Fatalf("Failed to delete a file - %v", err)
   361  	}
   362  
   363  	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
   364  	if err != nil {
   365  		t.Fatalf("Failed to heal object - %v", err)
   366  	}
   367  
   368  	if err = firstDisk.CheckFile(context.Background(), bucket, object); err != nil {
   369  		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
   370  	}
   371  
   372  	erasureDisks := er.getDisks()
   373  	z.serverPools[0].erasureDisksMu.Lock()
   374  	er.getDisks = func() []StorageAPI {
   375  		// Nil more than half the disks, to remove write quorum.
   376  		for i := 0; i <= len(erasureDisks)/2; i++ {
   377  			erasureDisks[i] = nil
   378  		}
   379  		return erasureDisks
   380  	}
   381  	z.serverPools[0].erasureDisksMu.Unlock()
   382  
   383  	// Try healing now, expect to receive errDiskNotFound.
   384  	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
   385  	// since majority of er.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
   386  	if _, ok := err.(InsufficientReadQuorum); !ok {
   387  		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)
   388  	}
   389  }
   390  
   391  // Tests healing of empty directories
   392  func TestHealEmptyDirectoryErasure(t *testing.T) {
   393  	ctx, cancel := context.WithCancel(context.Background())
   394  	defer cancel()
   395  
   396  	nDisks := 16
   397  	fsDirs, err := getRandomDisks(nDisks)
   398  	if err != nil {
   399  		t.Fatal(err)
   400  	}
   401  	defer removeRoots(fsDirs)
   402  
   403  	// Everything is fine, should return nil
   404  	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
   405  	if err != nil {
   406  		t.Fatal(err)
   407  	}
   408  
   409  	bucket := "bucket"
   410  	object := "empty-dir/"
   411  	var opts ObjectOptions
   412  
   413  	err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
   414  	if err != nil {
   415  		t.Fatalf("Failed to make a bucket - %v", err)
   416  	}
   417  
   418  	// Upload an empty directory
   419  	_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t,
   420  		bytes.NewReader([]byte{}), 0, "", ""), opts)
   421  	if err != nil {
   422  		t.Fatal(err)
   423  	}
   424  
   425  	// Remove the object backend files from the first disk.
   426  	z := obj.(*erasureServerPools)
   427  	er := z.serverPools[0].sets[0]
   428  	firstDisk := er.getDisks()[0]
   429  	err = firstDisk.DeleteVol(context.Background(), pathJoin(bucket, encodeDirObject(object)), true)
   430  	if err != nil {
   431  		t.Fatalf("Failed to delete a file - %v", err)
   432  	}
   433  
   434  	// Heal the object
   435  	hr, err := obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
   436  	if err != nil {
   437  		t.Fatalf("Failed to heal object - %v", err)
   438  	}
   439  
   440  	// Check if the empty directory is restored in the first disk
   441  	_, err = firstDisk.StatVol(context.Background(), pathJoin(bucket, encodeDirObject(object)))
   442  	if err != nil {
   443  		t.Fatalf("Expected object to be present but stat failed - %v", err)
   444  	}
   445  
   446  	// Check the state of the object in the first disk (should be missing)
   447  	if hr.Before.Drives[0].State != madmin.DriveStateMissing {
   448  		t.Fatalf("Unexpected drive state: %v", hr.Before.Drives[0].State)
   449  	}
   450  
   451  	// Check the state of all other disks (should be ok)
   452  	for i, h := range append(hr.Before.Drives[1:], hr.After.Drives...) {
   453  		if h.State != madmin.DriveStateOk {
   454  			t.Fatalf("Unexpected drive state (%d): %v", i+1, h.State)
   455  		}
   456  	}
   457  
   458  	// Heal the same object again
   459  	hr, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
   460  	if err != nil {
   461  		t.Fatalf("Failed to heal object - %v", err)
   462  	}
   463  
   464  	// Check that Before & After states are all okay
   465  	for i, h := range append(hr.Before.Drives, hr.After.Drives...) {
   466  		if h.State != madmin.DriveStateOk {
   467  			t.Fatalf("Unexpected drive state (%d): %v", i+1, h.State)
   468  		}
   469  	}
   470  }