gitlab.com/SiaPrime/SiaPrime@v1.4.1/modules/renter/siafile/acid_test.go (about)

     1  package siafile
     2  
     3  import (
     4  	"math"
     5  	"testing"
     6  	"time"
     7  
     8  	"gitlab.com/NebulousLabs/errors"
     9  	"gitlab.com/NebulousLabs/fastrand"
    10  	"gitlab.com/SiaPrime/SiaPrime/build"
    11  	"gitlab.com/SiaPrime/SiaPrime/crypto"
    12  	"gitlab.com/SiaPrime/SiaPrime/types"
    13  	"gitlab.com/SiaPrime/writeaheadlog"
    14  )
    15  
    16  // TestSiaFileFaultyDisk simulates interacting with a SiaFile on a faulty disk.
    17  func TestSiaFileFaultyDisk(t *testing.T) {
    18  	if testing.Short() {
    19  		t.SkipNow()
    20  	}
    21  	t.Parallel()
    22  
    23  	// Determine a reasonable timeout for the test.
    24  	var testTimeout time.Duration
    25  	if testing.Short() {
    26  		t.SkipNow()
    27  	} else if build.VLONG {
    28  		testTimeout = time.Minute
    29  	} else {
    30  		testTimeout = 10 * time.Second
    31  	}
    32  
    33  	// Create the dependency.
    34  	fdd := newFaultyDiskDependency(10000) // Fails after 10000 writes.
    35  	fdd.disable()
    36  
    37  	// Create a new blank siafile.
    38  	siafile, wal, walPath := newBlankTestFileAndWAL(1)
    39  	siafile.deps = fdd
    40  
    41  	// Wrap it in a file set entry.
    42  	sf := dummyEntry(siafile)
    43  	if err := setCombinedChunkOfTestFile(sf.SiaFile); err != nil {
    44  		t.Fatal(err)
    45  	}
    46  
    47  	// Create 50 hostkeys from which to choose from.
    48  	hostkeys := make([]types.SiaPublicKey, 0, 50)
    49  	for i := 0; i < 50; i++ {
    50  		spk := types.SiaPublicKey{}
    51  		fastrand.Read(spk.Key)
    52  		hostkeys = append(hostkeys, types.SiaPublicKey{})
    53  	}
    54  
    55  	// The outer loop is responsible for simulating a restart of siad by
    56  	// reloading the wal, applying transactions and loading the sf from disk
    57  	// again.
    58  	fdd.enable()
    59  	testDone := time.After(testTimeout)
    60  	numRecoveries := 0
    61  	numSuccessfulIterations := 0
    62  OUTER:
    63  	for {
    64  		select {
    65  		case <-testDone:
    66  			break OUTER
    67  		default:
    68  		}
    69  
    70  		// The inner loop applies a random number of operations on the file.
    71  		for {
    72  			select {
    73  			case <-testDone:
    74  				break OUTER
    75  			default:
    76  			}
    77  			// 5% chance to break out of inner loop.
    78  			if fastrand.Intn(100) < 5 {
    79  				break
    80  			}
    81  			// 80% chance to add a piece.
    82  			if fastrand.Intn(100) < 80 {
    83  				spk := hostkeys[fastrand.Intn(len(hostkeys))]
    84  				offset := uint64(fastrand.Intn(int(sf.staticMetadata.FileSize)))
    85  				snap, err := sf.Snapshot()
    86  				if err != nil {
    87  					if errors.Contains(err, errDiskFault) {
    88  						numRecoveries++
    89  						break
    90  					}
    91  					// If the error wasn't caused by the dependency, the test
    92  					// fails.
    93  					t.Fatal(err)
    94  				}
    95  				chunkIndex, _ := snap.ChunkIndexByOffset(offset)
    96  				pieceIndex := uint64(fastrand.Intn(sf.staticMetadata.staticErasureCode.NumPieces()))
    97  				if err := sf.AddPiece(spk, chunkIndex, pieceIndex, crypto.Hash{}); err != nil {
    98  					if errors.Contains(err, errDiskFault) {
    99  						numRecoveries++
   100  						break
   101  					}
   102  					// If the error wasn't caused by the dependency, the test
   103  					// fails.
   104  					t.Fatal(err)
   105  				}
   106  			}
   107  			numSuccessfulIterations++
   108  		}
   109  
   110  		// 20% chance that drive is repaired.
   111  		if fastrand.Intn(100) < 20 {
   112  			fdd.reset()
   113  		}
   114  
   115  		// Try to reload the file. This simulates failures during recovery.
   116  	LOAD:
   117  		for tries := 0; ; tries++ {
   118  			// If we have already tried for 10 times, we reset the dependency
   119  			// to avoid getting stuck here.
   120  			if tries%10 == 0 {
   121  				fdd.reset()
   122  			}
   123  			// Close existing wal.
   124  			_, err := wal.CloseIncomplete()
   125  			if err != nil {
   126  				t.Fatal(err)
   127  			}
   128  			// Reopen wal.
   129  			var txns []*writeaheadlog.Transaction
   130  			txns, wal, err = writeaheadlog.New(walPath)
   131  			if err != nil {
   132  				t.Fatal(err)
   133  			}
   134  			// Apply unfinished txns.
   135  			for _, txn := range txns {
   136  				if err := applyUpdates(fdd, txn.Updates...); err != nil {
   137  					if errors.Contains(err, errDiskFault) {
   138  						numRecoveries++
   139  						continue LOAD // try again
   140  					} else {
   141  						t.Fatal(err)
   142  					}
   143  				}
   144  				if err := txn.SignalUpdatesApplied(); err != nil {
   145  					t.Fatal(err)
   146  				}
   147  			}
   148  			// Load file again.
   149  			partialsSiaFile, err := loadSiaFile(sf.partialsSiaFile.siaFilePath, wal, fdd)
   150  			if err != nil {
   151  				if errors.Contains(err, errDiskFault) {
   152  					numRecoveries++
   153  					continue // try again
   154  				} else {
   155  					t.Fatal(err)
   156  				}
   157  			}
   158  			siafile, err = loadSiaFile(sf.siaFilePath, wal, fdd)
   159  			if err != nil {
   160  				if errors.Contains(err, errDiskFault) {
   161  					numRecoveries++
   162  					continue // try again
   163  				} else {
   164  					t.Fatal(err)
   165  				}
   166  			}
   167  			partialsEntry := &SiaFileSetEntry{
   168  				dummyEntry(partialsSiaFile),
   169  				uint64(fastrand.Intn(math.MaxInt32)),
   170  			}
   171  			siafile.deps = fdd
   172  			sf = dummyEntry(siafile)
   173  			sf.SetPartialsSiaFile(partialsEntry)
   174  			break
   175  		}
   176  
   177  	}
   178  	t.Logf("Recovered from %v disk failures", numRecoveries)
   179  	t.Logf("Inner loop %v iterations without failures", numSuccessfulIterations)
   180  }