gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/filesystem/siafile/acid_test.go (about)

     1  package siafile
     2  
     3  import (
     4  	"testing"
     5  	"time"
     6  
     7  	"gitlab.com/SkynetLabs/skyd/siatest/dependencies"
     8  
     9  	"gitlab.com/NebulousLabs/errors"
    10  	"gitlab.com/NebulousLabs/fastrand"
    11  	"gitlab.com/NebulousLabs/writeaheadlog"
    12  
    13  	"gitlab.com/SkynetLabs/skyd/build"
    14  	"gitlab.com/SkynetLabs/skyd/skymodules"
    15  	"go.sia.tech/siad/crypto"
    16  	"go.sia.tech/siad/types"
    17  )
    18  
    19  // TestSiaFileFaultyDisk simulates interacting with a SiaFile on a faulty disk.
    20  func TestSiaFileFaultyDisk(t *testing.T) {
    21  	if testing.Short() {
    22  		t.SkipNow()
    23  	}
    24  	t.Parallel()
    25  
    26  	// Determine a reasonable timeout for the test.
    27  	var testTimeout time.Duration
    28  	if testing.Short() {
    29  		t.SkipNow()
    30  	} else if build.VLONG {
    31  		testTimeout = time.Minute
    32  	} else {
    33  		testTimeout = 10 * time.Second
    34  	}
    35  
    36  	// Create the dependency.
    37  	fdd := dependencies.NewFaultyDiskDependency(10000) // Fails after 10000 writes.
    38  	fdd.Disable()
    39  
    40  	// Create a new blank siafile.
    41  	sf, wal, walPath := newBlankTestFileAndWAL(1)
    42  	sf.deps = fdd
    43  
    44  	// Create 50 hostkeys from which to choose from.
    45  	hostkeys := make([]types.SiaPublicKey, 0, 50)
    46  	for i := 0; i < 50; i++ {
    47  		spk := types.SiaPublicKey{}
    48  		fastrand.Read(spk.Key)
    49  		hostkeys = append(hostkeys, types.SiaPublicKey{})
    50  	}
    51  
    52  	// The outer loop is responsible for simulating a restart of siad by
    53  	// reloading the wal, applying transactions and loading the sf from disk
    54  	// again.
    55  	fdd.Enable()
    56  	testDone := time.After(testTimeout)
    57  	numRecoveries := 0
    58  	numSuccessfulIterations := 0
    59  	filePath := sf.siaFilePath
    60  OUTER:
    61  	for {
    62  		select {
    63  		case <-testDone:
    64  			break OUTER
    65  		default:
    66  		}
    67  
    68  		// The inner loop applies a random number of operations on the file.
    69  		for {
    70  			select {
    71  			case <-testDone:
    72  				break OUTER
    73  			default:
    74  			}
    75  			// 5% chance to break out of inner loop.
    76  			if fastrand.Intn(100) < 5 {
    77  				break
    78  			}
    79  			// 80% chance to add a piece.
    80  			if fastrand.Intn(100) < 80 {
    81  				spk := hostkeys[fastrand.Intn(len(hostkeys))]
    82  				offset := uint64(fastrand.Intn(int(sf.staticMetadata.FileSize)))
    83  				snap, err := sf.Snapshot(skymodules.RandomSiaPath())
    84  				if err != nil {
    85  					if errors.Contains(err, dependencies.ErrDiskFault) {
    86  						numRecoveries++
    87  						break
    88  					}
    89  					// If the error wasn't caused by the dependency, the test
    90  					// fails.
    91  					t.Fatal(err)
    92  				}
    93  				chunkIndex, _ := snap.ChunkIndexByOffset(offset)
    94  				pieceIndex := uint64(fastrand.Intn(sf.staticMetadata.staticErasureCode.NumPieces()))
    95  				if err := sf.AddPiece(spk, chunkIndex, pieceIndex, crypto.Hash{}); err != nil {
    96  					if errors.Contains(err, dependencies.ErrDiskFault) {
    97  						numRecoveries++
    98  						break
    99  					}
   100  					// If the error wasn't caused by the dependency, the test
   101  					// fails.
   102  					t.Fatal(err)
   103  				}
   104  			}
   105  			numSuccessfulIterations++
   106  		}
   107  
   108  		// 20% chance that drive is repaired.
   109  		if fastrand.Intn(100) < 20 {
   110  			fdd.Reset()
   111  		}
   112  
   113  		// Try to reload the file. This simulates failures during recovery.
   114  	LOAD:
   115  		for tries := 0; ; tries++ {
   116  			// If we have already tried for 10 times, we reset the dependency
   117  			// to avoid getting stuck here.
   118  			if tries%10 == 0 {
   119  				fdd.Reset()
   120  			}
   121  			// Close existing wal.
   122  			_, err := wal.CloseIncomplete()
   123  			if err != nil {
   124  				t.Fatal(err)
   125  			}
   126  			// Reopen wal.
   127  			var txns []*writeaheadlog.Transaction
   128  			txns, wal, err = writeaheadlog.New(walPath)
   129  			if err != nil {
   130  				t.Fatal(err)
   131  			}
   132  			// Apply unfinished txns.
   133  			for _, txn := range txns {
   134  				if err := applyUpdates(fdd, txn.Updates...); err != nil {
   135  					if errors.Contains(err, dependencies.ErrDiskFault) {
   136  						numRecoveries++
   137  						continue LOAD // try again
   138  					} else {
   139  						t.Fatal(err)
   140  					}
   141  				}
   142  				if err := txn.SignalUpdatesApplied(); err != nil {
   143  					t.Fatal(err)
   144  				}
   145  			}
   146  			// Load file again.
   147  			sf, err = loadSiaFile(filePath, wal, fdd)
   148  			if err != nil && errors.Contains(err, dependencies.ErrDiskFault) {
   149  				numRecoveries++
   150  				continue // try again
   151  			} else if err != nil {
   152  				t.Fatal(err)
   153  			}
   154  			sf.deps = fdd
   155  			break
   156  		}
   157  	}
   158  	t.Logf("Recovered from %v disk failures", numRecoveries)
   159  	t.Logf("Inner loop %v iterations without failures", numSuccessfulIterations)
   160  }