gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/filesystem/siafile/persist_compat.go (about)

     1  package siafile
     2  
     3  import (
     4  	"os"
     5  	"time"
     6  
     7  	"gitlab.com/NebulousLabs/errors"
     8  	"gitlab.com/NebulousLabs/writeaheadlog"
     9  
    10  	"gitlab.com/SkynetLabs/skyd/build"
    11  	"gitlab.com/SkynetLabs/skyd/skymodules"
    12  	"go.sia.tech/siad/crypto"
    13  	"go.sia.tech/siad/modules"
    14  )
    15  
    16  var (
    17  	// errWrongMetadataVersion is the error returned when the metadata
    18  	// version is wrong
    19  	errWrongMetadataVersion = errors.New("wrong metadata version")
    20  
    21  	// metadataVersion is the current version of the siafile Metadata
    22  	metadataVersion = metadataVersion3
    23  
    24  	// metadataVersion3 is the third version of the siafile Metadata
    25  	metadataVersion3 = [16]byte{3}
    26  
    27  	// metadataVersion2 is the second version of the siafile Metadata
    28  	metadataVersion2 = [16]byte{2}
    29  
    30  	// metadataVersion1 is the first version of the siafile Metadata
    31  	metadataVersion1 = [16]byte{1}
    32  
    33  	// nilMetadataVesion is a helper for identifying an uninitialized
    34  	// metadata version
    35  	nilMetadataVesion = [16]byte{}
    36  )
    37  
    38  type (
    39  	// FileData is a helper struct that contains all the relevant information
    40  	// of a file. It simplifies passing the necessary data between modules and
    41  	// keeps the interface clean.
    42  	FileData struct {
    43  		Name        string
    44  		FileSize    uint64
    45  		MasterKey   [crypto.EntropySize]byte
    46  		ErasureCode skymodules.ErasureCoder
    47  		RepairPath  string
    48  		PieceSize   uint64
    49  		Mode        os.FileMode
    50  		Deleted     bool
    51  		UID         SiafileUID
    52  		Chunks      []FileChunk
    53  	}
    54  	// FileChunk is a helper struct that contains data about a chunk.
    55  	FileChunk struct {
    56  		Pieces [][]Piece
    57  	}
    58  )
    59  
    60  // NewFromLegacyData creates a new SiaFile from data that was previously loaded
    61  // from a legacy file.
    62  func NewFromLegacyData(fd FileData, siaFilePath string, wal *writeaheadlog.WAL) (*SiaFile, error) {
    63  	// Legacy master keys are always twofish keys.
    64  	mk, err := crypto.NewSiaKey(crypto.TypeTwofish, fd.MasterKey[:])
    65  	if err != nil {
    66  		return nil, errors.AddContext(err, "failed to restore master key")
    67  	}
    68  	currentTime := time.Now()
    69  	ecType, ecParams := marshalErasureCoder(fd.ErasureCode)
    70  	zeroHealth := float64(1 + fd.ErasureCode.MinPieces()/(fd.ErasureCode.NumPieces()-fd.ErasureCode.MinPieces()))
    71  	file := &SiaFile{
    72  		staticMetadata: Metadata{
    73  			AccessTime:              currentTime,
    74  			ChunkOffset:             defaultReservedMDPages * pageSize,
    75  			ChangeTime:              currentTime,
    76  			CreateTime:              currentTime,
    77  			CachedHealth:            zeroHealth,
    78  			CachedStuckHealth:       0,
    79  			CachedRedundancy:        0,
    80  			CachedUserRedundancy:    0,
    81  			CachedUploadProgress:    0,
    82  			FileSize:                int64(fd.FileSize),
    83  			LocalPath:               fd.RepairPath,
    84  			StaticMasterKey:         mk.Key(),
    85  			StaticMasterKeyType:     mk.Type(),
    86  			Mode:                    fd.Mode,
    87  			ModTime:                 currentTime,
    88  			staticErasureCode:       fd.ErasureCode,
    89  			StaticErasureCodeType:   ecType,
    90  			StaticErasureCodeParams: ecParams,
    91  			StaticPagesPerChunk:     numChunkPagesRequired(fd.ErasureCode.NumPieces()),
    92  			StaticPieceSize:         fd.PieceSize,
    93  			UniqueID:                SiafileUID(fd.UID),
    94  		},
    95  		deps:        modules.ProdDependencies,
    96  		deleted:     fd.Deleted,
    97  		numChunks:   len(fd.Chunks),
    98  		siaFilePath: siaFilePath,
    99  		wal:         wal,
   100  	}
   101  	// Update cached fields for 0-Byte files.
   102  	if file.staticMetadata.FileSize == 0 {
   103  		file.staticMetadata.CachedHealth = 0
   104  		file.staticMetadata.CachedStuckHealth = 0
   105  		file.staticMetadata.CachedRedundancy = float64(fd.ErasureCode.NumPieces()) / float64(fd.ErasureCode.MinPieces())
   106  		file.staticMetadata.CachedUserRedundancy = file.staticMetadata.CachedRedundancy
   107  		file.staticMetadata.CachedUploadProgress = 100
   108  	}
   109  
   110  	// Create the chunks.
   111  	chunks := make([]chunk, len(fd.Chunks))
   112  	for i := range chunks {
   113  		chunks[i].Pieces = make([][]piece, file.staticMetadata.staticErasureCode.NumPieces())
   114  		chunks[i].Index = i
   115  	}
   116  
   117  	// Populate the pubKeyTable of the file and add the pieces.
   118  	pubKeyMap := make(map[string]uint32)
   119  	for chunkIndex, chunk := range fd.Chunks {
   120  		for pieceIndex, pieceSet := range chunk.Pieces {
   121  			for _, p := range pieceSet {
   122  				// Check if we already added that public key.
   123  				tableOffset, exists := pubKeyMap[string(p.HostPubKey.Key)]
   124  				if !exists {
   125  					tableOffset = uint32(len(file.pubKeyTable))
   126  					pubKeyMap[string(p.HostPubKey.Key)] = tableOffset
   127  					file.pubKeyTable = append(file.pubKeyTable, HostPublicKey{
   128  						PublicKey: p.HostPubKey,
   129  						Used:      true,
   130  					})
   131  				}
   132  				// Add the piece to the SiaFile.
   133  				chunks[chunkIndex].Pieces[pieceIndex] = append(chunks[chunkIndex].Pieces[pieceIndex], piece{
   134  					HostTableOffset: tableOffset,
   135  					MerkleRoot:      p.MerkleRoot,
   136  				})
   137  			}
   138  		}
   139  	}
   140  
   141  	// Save file to disk.
   142  	if err := file.saveFile(chunks); err != nil {
   143  		return nil, errors.AddContext(err, "unable to save file")
   144  	}
   145  
   146  	// Update the cached fields for progress and uploaded bytes.
   147  	_, _, err = file.UploadProgressAndBytes()
   148  	return file, err
   149  }
   150  
   151  // metadataCompatCheck handles the compatibility checks for the metadata based
   152  // on the version
   153  //
   154  // NOTE: there is no need to use the backup and restore method of the metadata
   155  // here because this is called on load. If there is an error if means we are
   156  // unable to load the siafile, and therefore cannot use it which makes restoring
   157  // the metadata pointless.
   158  func (sf *SiaFile) metadataCompatCheck() error {
   159  	// Quit early to avoid unnecessary disk write.
   160  	if sf.staticMetadata.StaticVersion == metadataVersion {
   161  		return nil
   162  	}
   163  
   164  	// Check uninitialized case
   165  	if sf.staticMetadata.StaticVersion == nilMetadataVesion {
   166  		sf.upgradeMetadataFromNilToV1()
   167  	}
   168  
   169  	// Check for version 1 updates.
   170  	if sf.staticMetadata.StaticVersion == metadataVersion1 {
   171  		err := sf.upgradeMetadataFromV1ToV2()
   172  		if err != nil {
   173  			return err
   174  		}
   175  	}
   176  
   177  	// Check for version 2 updates.
   178  	if sf.staticMetadata.StaticVersion == metadataVersion2 {
   179  		err := sf.upgradeMetadataFromV2ToV3()
   180  		if err != nil {
   181  			return err
   182  		}
   183  	}
   184  
   185  	// Check for current version
   186  	if sf.staticMetadata.StaticVersion != metadataVersion {
   187  		return errWrongMetadataVersion
   188  	}
   189  
   190  	// Save Metadata to persist updates
   191  	err := sf.saveMetadata()
   192  	if err != nil {
   193  		return err
   194  	}
   195  
   196  	return nil
   197  }
   198  
   199  // upgradeMetadataFromNilToV1 upgrades an uninitialized metadata version to
   200  // version 1 with the corresponding compat code
   201  func (sf *SiaFile) upgradeMetadataFromNilToV1() {
   202  	// Sanity Check
   203  	if sf.staticMetadata.StaticVersion != nilMetadataVesion {
   204  		build.Critical("upgradeMetadataFromNilToV1 called with non nil metadata")
   205  		return
   206  	}
   207  
   208  	// COMPATv137 legacy files might not have a unique id.
   209  	if sf.staticMetadata.UniqueID == "" {
   210  		sf.staticMetadata.UniqueID = uniqueID()
   211  	}
   212  
   213  	// COMPATv140 legacy 0-byte files might not have correct cached
   214  	// fields since we never update them once they are created.
   215  	if sf.staticMetadata.FileSize == 0 {
   216  		ec := sf.staticMetadata.staticErasureCode
   217  		sf.staticMetadata.CachedHealth = 0
   218  		sf.staticMetadata.CachedStuckHealth = 0
   219  		sf.staticMetadata.CachedRedundancy = float64(ec.NumPieces()) / float64(ec.MinPieces())
   220  		sf.staticMetadata.CachedUserRedundancy = sf.staticMetadata.CachedRedundancy
   221  		sf.staticMetadata.CachedUploadProgress = 100
   222  	}
   223  
   224  	// Update the version now that we have completed the compat updates
   225  	sf.staticMetadata.StaticVersion = metadataVersion1
   226  }
   227  
   228  // upgradeMetadataFromV1ToV2 upgrades a version 1 metadata to a version 2 with
   229  // the corresponding compat code
   230  func (sf *SiaFile) upgradeMetadataFromV1ToV2() error {
   231  	// Sanity Check
   232  	if sf.staticMetadata.StaticVersion != metadataVersion1 {
   233  		err := errors.New("upgradeMetadataFromV1ToV2 called with non version 1 metadata")
   234  		build.Critical(err)
   235  		return err
   236  	}
   237  
   238  	// Stuck vs Unfinished files compatibility check.
   239  	//
   240  	// Before unfinished files were introduced a file might have been marked
   241  	// as stuck if the upload failed. In this case, we don't expect the file
   242  	// to ever be recoverable since the upload failed. Therefore, we don't
   243  	// want it marked as stuck, we just want to ignore it and let the
   244  	// unfinished files code eventually prune it.
   245  
   246  	// Get the file's stuck status
   247  	stuck := sf.numStuckChunks() > 0
   248  
   249  	// Get the file's unique uploaded bytes to compare against the file size
   250  	_, unique, err := sf.uploadedBytes()
   251  	if err != nil {
   252  		return err
   253  	}
   254  	size := uint64(sf.staticMetadata.FileSize)
   255  
   256  	// Determine if the file is finished based on if it ever finished
   257  	// uploading or has a localpath defined.
   258  	sf.staticMetadata.Finished = unique >= size || sf.onDisk()
   259  
   260  	// If the File is not finished, and stuck, reset the stuck status
   261  	if !sf.staticMetadata.Finished && stuck {
   262  		err = sf.setAllStuck(false)
   263  		if err != nil {
   264  			return errors.AddContext(err, "unable to mark unfinished file as unstuck")
   265  		}
   266  	}
   267  
   268  	// Update the version now that we have completed the compat updates
   269  	sf.staticMetadata.StaticVersion = metadataVersion2
   270  	return nil
   271  }
   272  
   273  // upgradeMetadataFromV2ToV3 upgrades a version 2 metadata to a version 3 with
   274  // the corresponding compat code
   275  func (sf *SiaFile) upgradeMetadataFromV2ToV3() error {
   276  	// Sanity Check
   277  	if sf.staticMetadata.StaticVersion != metadataVersion2 {
   278  		err := errors.New("upgradeMetadataFromV2ToV3 called with non version 2 metadata")
   279  		build.Critical(err)
   280  		return err
   281  	}
   282  
   283  	// Unfinished files update.
   284  	//
   285  	// The original intent of instituting the unfinished files was to do a
   286  	// full reset of the filesystem and let the health loop work through
   287  	// updating the state of the files.
   288  
   289  	// Reset the Finished state.
   290  	sf.staticMetadata.Finished = false
   291  
   292  	// Get the file's stuck status
   293  	stuck := sf.numStuckChunks() > 0
   294  
   295  	// If the File is stuck, reset the stuck status
   296  	if stuck {
   297  		err := sf.setAllStuck(false)
   298  		if err != nil {
   299  			return errors.AddContext(err, "unable to reset stuck status")
   300  		}
   301  	}
   302  
   303  	// Update the version now that we have completed the compat updates
   304  	sf.staticMetadata.StaticVersion = metadataVersion3
   305  	return nil
   306  }