gitlab.com/SiaPrime/SiaPrime@v1.4.1/modules/renter/siafile/siafileset_test.go (about)

     1  package siafile
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/hex"
     6  	"io/ioutil"
     7  	"os"
     8  	"path/filepath"
     9  	"reflect"
    10  	"strings"
    11  	"sync"
    12  	"testing"
    13  	"time"
    14  
    15  	"gitlab.com/NebulousLabs/errors"
    16  	"gitlab.com/NebulousLabs/fastrand"
    17  
    18  	"gitlab.com/SiaPrime/SiaPrime/modules"
    19  	"gitlab.com/SiaPrime/SiaPrime/modules/renter/siadir"
    20  )
    21  
    22  // newTestSiaFileSetWithFile creates a new SiaFileSet and SiaFile and makes sure
    23  // that they are linked
    24  func newTestSiaFileSetWithFile() (*SiaFileSetEntry, *SiaFileSet, error) {
    25  	// Create new SiaFile params
    26  	_, siaPath, source, rc, sk, fileSize, _, fileMode := newTestFileParams(1, true)
    27  	dir := filepath.Join(os.TempDir(), "siafiles", hex.EncodeToString(fastrand.Bytes(16)))
    28  	// Create SiaFileSet
    29  	wal, _ := newTestWAL()
    30  	sfs := NewSiaFileSet(dir, wal)
    31  	// Create SiaFile
    32  	up := modules.FileUploadParams{
    33  		Source:      source,
    34  		SiaPath:     siaPath,
    35  		ErasureCode: rc,
    36  	}
    37  	entry, err := sfs.NewSiaFile(up, sk, fileSize, fileMode)
    38  	if err != nil {
    39  		return nil, nil, err
    40  	}
    41  	return entry, sfs, nil
    42  }
    43  
    44  // TestAddExistingSiafile tests the AddExistingSiaFile method's behavior.
    45  func TestAddExistingSiafile(t *testing.T) {
    46  	if testing.Short() {
    47  		t.SkipNow()
    48  	}
    49  	t.Parallel()
    50  	// Create a fileset with file.
    51  	sf, sfs, err := newTestSiaFileSetWithFile()
    52  	if err != nil {
    53  		t.Fatal(err)
    54  	}
    55  	// Add the existing file to the set again this shouldn't do anything.
    56  	if err := sfs.AddExistingSiaFile(sf.SiaFile, []chunk{}); err != nil {
    57  		t.Fatal(err)
    58  	}
    59  	numSiaFiles := 0
    60  	err = filepath.Walk(sfs.staticSiaFileDir, func(path string, info os.FileInfo, err error) error {
    61  		if filepath.Ext(path) == modules.SiaFileExtension {
    62  			numSiaFiles++
    63  		}
    64  		return nil
    65  	})
    66  	if err != nil {
    67  		t.Fatal(err)
    68  	}
    69  	// There should be 1 siafile.
    70  	if numSiaFiles != 1 {
    71  		t.Fatalf("Found %v siafiles but expected %v", numSiaFiles, 1)
    72  	}
    73  	// Load the same siafile again, but change the UID.
    74  	b, err := ioutil.ReadFile(sf.siaFilePath)
    75  	if err != nil {
    76  		t.Fatal(err)
    77  	}
    78  	reader := bytes.NewReader(b)
    79  	newSF, newChunks, err := LoadSiaFileFromReaderWithChunks(reader, sf.SiaFilePath(), sf.wal)
    80  	if err != nil {
    81  		t.Fatal(err)
    82  	}
    83  	// Grab the pre-import UID after changing it.
    84  	newSF.UpdateUniqueID()
    85  	preImportUID := newSF.UID()
    86  	// Import the file. This should work because the files no longer share the same
    87  	// UID.
    88  	if err := sfs.AddExistingSiaFile(newSF, newChunks); err != nil {
    89  		t.Fatal(err)
    90  	}
    91  	// sf and newSF should have the same pieces.
    92  	for chunkIndex := uint64(0); chunkIndex < sf.NumChunks(); chunkIndex++ {
    93  		piecesOld, err1 := sf.Pieces(chunkIndex)
    94  		piecesNew, err2 := newSF.Pieces(chunkIndex)
    95  		if err := errors.Compose(err1, err2); err != nil {
    96  			t.Fatal(err)
    97  		}
    98  		if !reflect.DeepEqual(piecesOld, piecesNew) {
    99  			t.Log("piecesOld: ", piecesOld)
   100  			t.Log("piecesNew: ", piecesNew)
   101  			t.Fatal("old pieces don't match new pieces")
   102  		}
   103  	}
   104  	numSiaFiles = 0
   105  	err = filepath.Walk(sfs.staticSiaFileDir, func(path string, info os.FileInfo, err error) error {
   106  		if filepath.Ext(path) == modules.SiaFileExtension {
   107  			numSiaFiles++
   108  		}
   109  		return nil
   110  	})
   111  	if err != nil {
   112  		t.Fatal(err)
   113  	}
   114  	// There should be 2 siafiles.
   115  	if numSiaFiles != 2 {
   116  		t.Fatalf("Found %v siafiles but expected %v", numSiaFiles, 2)
   117  	}
   118  	// The UID should have changed.
   119  	if newSF.UID() == preImportUID {
   120  		t.Fatal("newSF UID should have changed after importing the file")
   121  	}
   122  	if !strings.HasSuffix(newSF.SiaFilePath(), "_1"+modules.SiaFileExtension) {
   123  		t.Fatal("SiaFile should have a suffix but didn't")
   124  	}
   125  	// Should be able to open the new file from disk.
   126  	if _, err := os.Stat(newSF.SiaFilePath()); err != nil {
   127  		t.Fatal(err)
   128  	}
   129  }
   130  
   131  // TestSiaFileSetDeleteOpen checks that deleting an entry from the set followed
   132  // by creating a Siafile with the same name without closing the deleted entry
   133  // works as expected.
   134  func TestSiaFileSetDeleteOpen(t *testing.T) {
   135  	if testing.Short() {
   136  		t.SkipNow()
   137  	}
   138  	t.Parallel()
   139  
   140  	// Create new SiaFile params
   141  	_, siaPath, source, rc, sk, fileSize, _, fileMode := newTestFileParams(1, true)
   142  	// Create SiaFileSet
   143  	wal, _ := newTestWAL()
   144  	dir := filepath.Join(os.TempDir(), "siafiles")
   145  	sfs := NewSiaFileSet(dir, wal)
   146  
   147  	// Repeatedly create a SiaFile and delete it while still keeping the entry
   148  	// around. That should only be possible without errors if the correctly
   149  	// delete the entry from the set.
   150  	var entries []*SiaFileSetEntry
   151  	for i := 0; i < 10; i++ {
   152  		// Create SiaFile
   153  		up := modules.FileUploadParams{
   154  			Source:      source,
   155  			SiaPath:     siaPath,
   156  			ErasureCode: rc,
   157  		}
   158  		entry, err := sfs.NewSiaFile(up, sk, fileSize, fileMode)
   159  		if err != nil {
   160  			t.Fatal(err)
   161  		}
   162  		// Delete SiaFile
   163  		if err := sfs.Delete(sfs.SiaPath(entry)); err != nil {
   164  			t.Fatal(err)
   165  		}
   166  		// The set should be empty except for the partials file.
   167  		if len(sfs.siaFileMap) != 1 {
   168  			t.Fatal("SiaFileMap should have 1 file")
   169  		}
   170  		// Append the entry to make sure we can close it later.
   171  		entries = append(entries, entry)
   172  	}
   173  	// The SiaFile shouldn't exist anymore.
   174  	exists := sfs.Exists(siaPath)
   175  	if exists {
   176  		t.Fatal("SiaFile shouldn't exist anymore")
   177  	}
   178  	// Close the entries.
   179  	for _, entry := range entries {
   180  		if err := entry.Close(); err != nil {
   181  			t.Fatal(err)
   182  		}
   183  	}
   184  }
   185  
   186  // TestSiaFileSetOpenClose tests that the threadCount of the siafile is
   187  // incremented and decremented properly when Open() and Close() are called
   188  func TestSiaFileSetOpenClose(t *testing.T) {
   189  	if testing.Short() {
   190  		t.SkipNow()
   191  	}
   192  	t.Parallel()
   193  
   194  	// Create SiaFileSet with SiaFile
   195  	entry, sfs, err := newTestSiaFileSetWithFile()
   196  	if err != nil {
   197  		t.Fatal(err)
   198  	}
   199  	siaPath := sfs.SiaPath(entry)
   200  	exists := sfs.Exists(siaPath)
   201  	if !exists {
   202  		t.Fatal("No SiaFileSetEntry found")
   203  	}
   204  	if err != nil {
   205  		t.Fatal(err)
   206  	}
   207  
   208  	// Confirm 2 files are in memory
   209  	if len(sfs.siaFileMap) != 2 {
   210  		t.Fatalf("Expected SiaFileSet map to be of length 2, instead is length %v", len(sfs.siaFileMap))
   211  	}
   212  
   213  	// Confirm threadCount is incremented properly
   214  	if len(entry.threadMap) != 1 {
   215  		t.Fatalf("Expected threadMap to be of length 1, got %v", len(entry.threadMap))
   216  	}
   217  
   218  	// Close SiaFileSetEntry
   219  	entry.Close()
   220  
   221  	// Confirm that threadCount was decremented
   222  	if len(entry.threadMap) != 0 {
   223  		t.Fatalf("Expected threadCount to be 0, got %v", len(entry.threadMap))
   224  	}
   225  
   226  	// Confirm file and partialsSiaFile were removed from memory
   227  	if len(sfs.siaFileMap) != 0 {
   228  		t.Fatalf("Expected SiaFileSet map to contain 0 files, instead is length %v", len(sfs.siaFileMap))
   229  	}
   230  
   231  	// Open siafile again and confirm threadCount was incremented
   232  	entry, err = sfs.Open(siaPath)
   233  	if err != nil {
   234  		t.Fatal(err)
   235  	}
   236  	if len(entry.threadMap) != 1 {
   237  		t.Fatalf("Expected threadCount to be 1, got %v", len(entry.threadMap))
   238  	}
   239  }
   240  
   241  // TestFilesInMemory confirms that files are added and removed from memory
   242  // as expected when files are in use and not in use
   243  func TestFilesInMemory(t *testing.T) {
   244  	if testing.Short() {
   245  		t.SkipNow()
   246  	}
   247  	t.Parallel()
   248  
   249  	// Create SiaFileSet with SiaFile
   250  	entry, sfs, err := newTestSiaFileSetWithFile()
   251  	if err != nil {
   252  		t.Fatal(err)
   253  	}
   254  	siaPath := sfs.SiaPath(entry)
   255  	exists := sfs.Exists(siaPath)
   256  	if !exists {
   257  		t.Fatal("No SiaFileSetEntry found")
   258  	}
   259  	if err != nil {
   260  		t.Fatal(err)
   261  	}
   262  	// Confirm there are 2 files in memory. The partialsSiafile and the regular
   263  	// file.
   264  	if len(sfs.siaFileMap) != 2 {
   265  		t.Fatal("Expected 2 files in memory, got:", len(sfs.siaFileMap))
   266  	}
   267  	// Close File
   268  	err = entry.Close()
   269  	if err != nil {
   270  		t.Fatal(err)
   271  	}
   272  	// Confirm there are no files in memory
   273  	if len(sfs.siaFileMap) != 0 {
   274  		t.Fatal("Expected 0 files in memory, got:", len(sfs.siaFileMap))
   275  	}
   276  
   277  	// Test accessing the same file from two separate threads
   278  	//
   279  	// Open file
   280  	entry1, err := sfs.Open(siaPath)
   281  	if err != nil {
   282  		t.Fatal(err)
   283  	}
   284  	// Confirm there is 2 file in memory
   285  	if len(sfs.siaFileMap) != 2 {
   286  		t.Fatal("Expected 2 files in memory, got:", len(sfs.siaFileMap))
   287  	}
   288  	// Access the file again
   289  	entry2, err := sfs.Open(siaPath)
   290  	if err != nil {
   291  		t.Fatal(err)
   292  	}
   293  	// Confirm there is still only has 2 files in memory
   294  	if len(sfs.siaFileMap) != 2 {
   295  		t.Fatal("Expected 2 files in memory, got:", len(sfs.siaFileMap))
   296  	}
   297  	// Close one of the file instances
   298  	err = entry1.Close()
   299  	if err != nil {
   300  		t.Fatal(err)
   301  	}
   302  	// Confirm there is still only has 2 files in memory
   303  	if len(sfs.siaFileMap) != 2 {
   304  		t.Fatal("Expected 2 files in memory, got:", len(sfs.siaFileMap))
   305  	}
   306  
   307  	// Confirm closing out remaining files removes all files from memory
   308  	//
   309  	// Close last instance of the first file
   310  	err = entry2.Close()
   311  	if err != nil {
   312  		t.Fatal(err)
   313  	}
   314  	// Confirm there is one file in memory
   315  	if len(sfs.siaFileMap) != 1 {
   316  		t.Fatal("Expected 1 file in memory, got:", len(sfs.siaFileMap))
   317  	}
   318  }
   319  
   320  // TestRenameFileInMemory confirms that threads that have access to a file
   321  // will continue to have access to the file even it another thread renames it
   322  func TestRenameFileInMemory(t *testing.T) {
   323  	if testing.Short() {
   324  		t.SkipNow()
   325  	}
   326  	t.Parallel()
   327  
   328  	// Create SiaFileSet with SiaFile and corresponding combined siafile.
   329  	entry, sfs, err := newTestSiaFileSetWithFile()
   330  	if err != nil {
   331  		t.Fatal(err)
   332  	}
   333  	siaPath := sfs.SiaPath(entry)
   334  	exists := sfs.Exists(siaPath)
   335  	if !exists {
   336  		t.Fatal("No SiaFileSetEntry found")
   337  	}
   338  	if err != nil {
   339  		t.Fatal(err)
   340  	}
   341  
   342  	// Confirm there are 2 files in memory
   343  	if len(sfs.siaFileMap) != 2 {
   344  		t.Fatal("Expected  file in memory, got:", len(sfs.siaFileMap))
   345  	}
   346  
   347  	// Test renaming an instance of a file
   348  	//
   349  	// Access file with another instance
   350  	entry2, err := sfs.Open(siaPath)
   351  	if err != nil {
   352  		t.Fatal(err)
   353  	}
   354  	// Confirm that renter still only has 2 files in memory
   355  	if len(sfs.siaFileMap) != 2 {
   356  		t.Fatal("Expected 2 file in memory, got:", len(sfs.siaFileMap))
   357  	}
   358  	_, err = os.Stat(entry.SiaFilePath())
   359  	if err != nil {
   360  		println("err2", err.Error())
   361  	}
   362  	// Rename second instance
   363  	newSiaPath := modules.RandomSiaPath()
   364  	err = sfs.Rename(siaPath, newSiaPath)
   365  	if err != nil {
   366  		t.Fatal(err)
   367  	}
   368  	// Confirm there are still only 2 files in memory as renaming doesn't add
   369  	// the new name to memory
   370  	if len(sfs.siaFileMap) != 2 {
   371  		t.Fatal("Expected 2 files in memory, got:", len(sfs.siaFileMap))
   372  	}
   373  	// Close instance of renamed file
   374  	err = entry2.Close()
   375  	if err != nil {
   376  		t.Fatal(err)
   377  	}
   378  	// Confirm there are still only 2 files in memory
   379  	if len(sfs.siaFileMap) != 2 {
   380  		t.Fatal("Expected 2 files in memory, got:", len(sfs.siaFileMap))
   381  	}
   382  	// Close other instance of second file
   383  	err = entry.Close()
   384  	if err != nil {
   385  		t.Fatal(err)
   386  	}
   387  	// Confirm there is only 1 file in memory
   388  	if len(sfs.siaFileMap) != 1 {
   389  		t.Fatal("Expected 1 file in memory, got:", len(sfs.siaFileMap))
   390  	}
   391  }
   392  
   393  // TestDeleteFileInMemory confirms that threads that have access to a file
   394  // will continue to have access to the file even it another thread deletes it
   395  func TestDeleteFileInMemory(t *testing.T) {
   396  	if testing.Short() {
   397  		t.SkipNow()
   398  	}
   399  	t.Parallel()
   400  
   401  	// Create SiaFileSet with SiaFile
   402  	entry, sfs, err := newTestSiaFileSetWithFile()
   403  	if err != nil {
   404  		t.Fatal(err)
   405  	}
   406  	siaPath := sfs.SiaPath(entry)
   407  	exists := sfs.Exists(siaPath)
   408  	if !exists {
   409  		t.Fatal("No SiaFileSetEntry found")
   410  	}
   411  	if err != nil {
   412  		t.Fatal(err)
   413  	}
   414  
   415  	// Confirm there are 2 files in memory
   416  	if len(sfs.siaFileMap) != 2 {
   417  		t.Fatal("Expected 1 file in memory, got:", len(sfs.siaFileMap))
   418  	}
   419  
   420  	// Test deleting an instance of a file
   421  	//
   422  	// Access the file again
   423  	entry2, err := sfs.Open(siaPath)
   424  	if err != nil {
   425  		t.Fatal(err)
   426  	}
   427  	// Confirm there is still only has 2 files in memory
   428  	if len(sfs.siaFileMap) != 2 {
   429  		t.Fatal("Expected 2 files in memory, got:", len(sfs.siaFileMap))
   430  	}
   431  	// delete and close instance of file
   432  	if err := sfs.Delete(siaPath); err != nil {
   433  		t.Fatal(err)
   434  	}
   435  	err = entry2.Close()
   436  	if err != nil {
   437  		t.Fatal(err)
   438  	}
   439  	// There should be one file in the set after deleting it.
   440  	if len(sfs.siaFileMap) != 1 {
   441  		t.Fatal("Expected 1 file in memory, got:", len(sfs.siaFileMap))
   442  	}
   443  	// confirm other instance is still in memory by calling methods on it
   444  	if !entry.Deleted() {
   445  		t.Fatal("Expected file to be deleted")
   446  	}
   447  
   448  	// Confirm closing out remaining files removes all files from memory
   449  	//
   450  	// Close last instance of the first file
   451  	err = entry.Close()
   452  	if err != nil {
   453  		t.Fatal(err)
   454  	}
   455  	// Confirm renter has one file in memory
   456  	if len(sfs.siaFileMap) != 1 {
   457  		t.Fatal("Expected 1 file in memory, got:", len(sfs.siaFileMap))
   458  	}
   459  }
   460  
   461  // TestDeleteCorruptSiaFile confirms that the siafileset will delete a siafile
   462  // even if it cannot be opened
   463  func TestDeleteCorruptSiaFile(t *testing.T) {
   464  	if testing.Short() {
   465  		t.SkipNow()
   466  	}
   467  	t.Parallel()
   468  
   469  	// Create siafileset
   470  	_, sfs, err := newTestSiaFileSetWithFile()
   471  	if err != nil {
   472  		t.Fatal(err)
   473  	}
   474  
   475  	// Create siafile on disk with random bytes
   476  	siaPath, err := modules.NewSiaPath("badFile")
   477  	if err != nil {
   478  		t.Fatal(err)
   479  	}
   480  	siaFilePath := siaPath.SiaFileSysPath(sfs.staticSiaFileDir)
   481  	err = ioutil.WriteFile(siaFilePath, fastrand.Bytes(100), 0666)
   482  	if err != nil {
   483  		t.Fatal(err)
   484  	}
   485  
   486  	// Confirm the siafile cannot be opened
   487  	_, err = sfs.Open(siaPath)
   488  	if err == nil || err == ErrUnknownPath {
   489  		t.Fatal("expected open to fail for read error but instead got:", err)
   490  	}
   491  
   492  	// Delete the siafile
   493  	err = sfs.Delete(siaPath)
   494  	if err != nil {
   495  		t.Fatal(err)
   496  	}
   497  
   498  	// Confirm the file is no longer on disk
   499  	_, err = os.Stat(siaFilePath)
   500  	if !os.IsNotExist(err) {
   501  		t.Fatal("Expected err to be that file does not exists but was:", err)
   502  	}
   503  }
   504  
   505  // TestSiaDirDelete tests the DeleteDir method of the siafileset.
   506  func TestSiaDirDelete(t *testing.T) {
   507  	if testing.Short() {
   508  		t.SkipNow()
   509  	}
   510  	// Prepare a siadirset
   511  	dirRoot := filepath.Join(os.TempDir(), "siadirs", t.Name())
   512  	os.RemoveAll(dirRoot)
   513  	os.RemoveAll(dirRoot)
   514  	wal, _ := newTestWAL()
   515  	sds := siadir.NewSiaDirSet(dirRoot, wal)
   516  	sfs := NewSiaFileSet(dirRoot, wal)
   517  
   518  	// Specify a directory structure for this test.
   519  	var dirStructure = []string{
   520  		"dir1",
   521  		"dir1/subdir1",
   522  		"dir1/subdir1/subsubdir1",
   523  		"dir1/subdir1/subsubdir2",
   524  		"dir1/subdir1/subsubdir3",
   525  		"dir1/subdir2",
   526  		"dir1/subdir2/subsubdir1",
   527  		"dir1/subdir2/subsubdir2",
   528  		"dir1/subdir2/subsubdir3",
   529  		"dir1/subdir3",
   530  		"dir1/subdir3/subsubdir1",
   531  		"dir1/subdir3/subsubdir2",
   532  		"dir1/subdir3/subsubdir3",
   533  	}
   534  	// Specify a function that's executed in parallel which continuously saves a
   535  	// file to disk.
   536  	stop := make(chan struct{})
   537  	wg := new(sync.WaitGroup)
   538  	f := func(entry *SiaFileSetEntry) {
   539  		defer wg.Done()
   540  		defer entry.Close()
   541  		for {
   542  			select {
   543  			case <-stop:
   544  				return
   545  			default:
   546  			}
   547  			err := entry.SaveHeader()
   548  			if err != nil && !strings.Contains(err.Error(), "can't call createAndApplyTransaction on deleted file") {
   549  				t.Fatal(err)
   550  			}
   551  			time.Sleep(50 * time.Millisecond)
   552  		}
   553  	}
   554  	// Create the structure and spawn a goroutine that keeps saving the structure
   555  	// to disk for each directory.
   556  	for _, dir := range dirStructure {
   557  		sp, err := modules.NewSiaPath(dir)
   558  		if err != nil {
   559  			t.Fatal(err)
   560  		}
   561  		entry, err := sds.NewSiaDir(sp)
   562  		if err != nil {
   563  			t.Fatal(err)
   564  		}
   565  		// 50% chance to close the dir.
   566  		if fastrand.Intn(2) == 0 {
   567  			entry.Close()
   568  		}
   569  		// Create a file in the dir.
   570  		fileSP, err := sp.Join(hex.EncodeToString(fastrand.Bytes(16)))
   571  		if err != nil {
   572  			t.Fatal(err)
   573  		}
   574  		_, _, source, rc, sk, fileSize, _, fileMode := newTestFileParams(1, true)
   575  		sf, err := sfs.NewSiaFile(modules.FileUploadParams{Source: source, SiaPath: fileSP, ErasureCode: rc}, sk, fileSize, fileMode)
   576  		if err != nil {
   577  			t.Fatal(err)
   578  		}
   579  		// 50% chance to spawn goroutine. It's not realistic to assume that all dirs
   580  		// are loaded.
   581  		if fastrand.Intn(2) == 0 {
   582  			wg.Add(1)
   583  			go f(sf)
   584  		} else {
   585  			sf.Close()
   586  		}
   587  	}
   588  	// Wait a second for the goroutines to write to disk a few times.
   589  	time.Sleep(time.Second)
   590  	// Delete dir1.
   591  	sp, err := modules.NewSiaPath("dir1")
   592  	if err != nil {
   593  		t.Fatal(err)
   594  	}
   595  	if err := sfs.DeleteDir(sp, sds.Delete); err != nil {
   596  		t.Fatal(err)
   597  	}
   598  
   599  	// Wait another second for more writes to disk after renaming the dir before
   600  	// killing the goroutines.
   601  	time.Sleep(time.Second)
   602  	close(stop)
   603  	wg.Wait()
   604  	time.Sleep(time.Second)
   605  	// The root siafile dir should be empty except for 1 .siadir file and a .csia
   606  	// file.
   607  	files, err := ioutil.ReadDir(sfs.staticSiaFileDir)
   608  	if err != nil {
   609  		t.Fatal(err)
   610  	}
   611  	if len(files) != 2 {
   612  		for _, file := range files {
   613  			t.Log("Found ", file.Name())
   614  		}
   615  		t.Fatalf("There should be %v files/folders in the root dir but found %v\n", 1, len(files))
   616  	}
   617  	for _, file := range files {
   618  		if filepath.Ext(file.Name()) != modules.SiaDirExtension &&
   619  			filepath.Ext(file.Name()) != modules.PartialsSiaFileExtension {
   620  			t.Fatal("Encountered unexpected file:", file.Name())
   621  		}
   622  	}
   623  }
   624  
   625  // TestSiaDirRename tests the RenameDir method of the siafileset.
   626  func TestSiaDirRename(t *testing.T) {
   627  	if testing.Short() {
   628  		t.SkipNow()
   629  	}
   630  	// Prepare a siadirset
   631  	dirRoot := filepath.Join(os.TempDir(), "siadirs", t.Name())
   632  	os.RemoveAll(dirRoot)
   633  	os.RemoveAll(dirRoot)
   634  	wal, _ := newTestWAL()
   635  	sds := siadir.NewSiaDirSet(dirRoot, wal)
   636  	sfs := NewSiaFileSet(dirRoot, wal)
   637  
   638  	// Specify a directory structure for this test.
   639  	var dirStructure = []string{
   640  		"dir1",
   641  		"dir1/subdir1",
   642  		"dir1/subdir1/subsubdir1",
   643  		"dir1/subdir1/subsubdir2",
   644  		"dir1/subdir1/subsubdir3",
   645  		"dir1/subdir2",
   646  		"dir1/subdir2/subsubdir1",
   647  		"dir1/subdir2/subsubdir2",
   648  		"dir1/subdir2/subsubdir3",
   649  		"dir1/subdir3",
   650  		"dir1/subdir3/subsubdir1",
   651  		"dir1/subdir3/subsubdir2",
   652  		"dir1/subdir3/subsubdir3",
   653  	}
   654  	// Specify a function that's executed in parallel which continuously saves a
   655  	// file to disk.
   656  	stop := make(chan struct{})
   657  	wg := new(sync.WaitGroup)
   658  	f := func(entry *SiaFileSetEntry) {
   659  		defer wg.Done()
   660  		defer entry.Close()
   661  		for {
   662  			select {
   663  			case <-stop:
   664  				return
   665  			default:
   666  			}
   667  			err := entry.SaveHeader()
   668  			if err != nil {
   669  				t.Fatal(err)
   670  			}
   671  			time.Sleep(50 * time.Millisecond)
   672  		}
   673  	}
   674  	// Create the structure and spawn a goroutine that keeps saving the structure
   675  	// to disk for each directory.
   676  	for _, dir := range dirStructure {
   677  		sp, err := modules.NewSiaPath(dir)
   678  		if err != nil {
   679  			t.Fatal(err)
   680  		}
   681  		entry, err := sds.NewSiaDir(sp)
   682  		if err != nil {
   683  			t.Fatal(err)
   684  		}
   685  		// 50% chance to close the dir.
   686  		if fastrand.Intn(2) == 0 {
   687  			entry.Close()
   688  		}
   689  		// Create a file in the dir.
   690  		fileSP, err := sp.Join(hex.EncodeToString(fastrand.Bytes(16)))
   691  		if err != nil {
   692  			t.Fatal(err)
   693  		}
   694  		_, _, source, rc, sk, fileSize, _, fileMode := newTestFileParams(1, true)
   695  		sf, err := sfs.NewSiaFile(modules.FileUploadParams{Source: source, SiaPath: fileSP, ErasureCode: rc}, sk, fileSize, fileMode)
   696  		if err != nil {
   697  			t.Fatal(err)
   698  		}
   699  		// 50% chance to spawn goroutine. It's not realistic to assume that all dirs
   700  		// are loaded.
   701  		if fastrand.Intn(2) == 0 {
   702  			wg.Add(1)
   703  			go f(sf)
   704  		} else {
   705  			sf.Close()
   706  		}
   707  	}
   708  	// Wait a second for the goroutines to write to disk a few times.
   709  	time.Sleep(time.Second)
   710  	// Rename dir1 to dir2.
   711  	oldPath, err1 := modules.NewSiaPath(dirStructure[0])
   712  	newPath, err2 := modules.NewSiaPath("dir2")
   713  	if err := errors.Compose(err1, err2); err != nil {
   714  		t.Fatal(err)
   715  	}
   716  	if err := sfs.RenameDir(oldPath, newPath, sds.Rename); err != nil {
   717  		t.Fatal(err)
   718  	}
   719  	// Wait another second for more writes to disk after renaming the dir before
   720  	// killing the goroutines.
   721  	time.Sleep(time.Second)
   722  	close(stop)
   723  	wg.Wait()
   724  	time.Sleep(time.Second)
   725  	// Make sure we can't open any of the old folders/files on disk but we can open
   726  	// the new ones.
   727  	for _, dir := range dirStructure {
   728  		oldDir, err1 := modules.NewSiaPath(dir)
   729  		newDir, err2 := oldDir.Rebase(oldPath, newPath)
   730  		if err := errors.Compose(err1, err2); err != nil {
   731  			t.Fatal(err)
   732  		}
   733  		// Open entry with old dir. Shouldn't work.
   734  		_, err := sds.Open(oldDir)
   735  		if err != siadir.ErrUnknownPath {
   736  			t.Fatal("shouldn't be able to open old path", oldDir.String(), err)
   737  		}
   738  		// Old dir shouldn't exist.
   739  		if _, err = os.Stat(oldDir.SiaDirSysPath(dirRoot)); !os.IsNotExist(err) {
   740  			t.Fatal(err)
   741  		}
   742  		// Open entry with new dir. Should succeed.
   743  		entry, err := sds.Open(newDir)
   744  		if err != nil {
   745  			t.Fatal(err)
   746  		}
   747  		defer entry.Close()
   748  		// New dir should contain 1 siafile.
   749  		fis, err := ioutil.ReadDir(newDir.SiaDirSysPath(dirRoot))
   750  		if err != nil {
   751  			t.Fatal(err)
   752  		}
   753  		numFiles := 0
   754  		for _, fi := range fis {
   755  			if !fi.IsDir() && filepath.Ext(fi.Name()) == modules.SiaFileExtension {
   756  				numFiles++
   757  			}
   758  		}
   759  		if numFiles != 1 {
   760  			t.Fatalf("there should be 1 file in the new dir not %v", numFiles)
   761  		}
   762  		// New entry should have a file.
   763  		// Check siapath of entry.
   764  		if entry.SiaPath() != newDir {
   765  			t.Fatalf("entry should have siapath '%v' but was '%v'", newDir, entry.SiaPath())
   766  		}
   767  	}
   768  }