gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/filesystem/siafile/persist_compat.go (about) 1 package siafile 2 3 import ( 4 "os" 5 "time" 6 7 "gitlab.com/NebulousLabs/errors" 8 "gitlab.com/NebulousLabs/writeaheadlog" 9 10 "gitlab.com/SkynetLabs/skyd/build" 11 "gitlab.com/SkynetLabs/skyd/skymodules" 12 "go.sia.tech/siad/crypto" 13 "go.sia.tech/siad/modules" 14 ) 15 16 var ( 17 // errWrongMetadataVersion is the error returned when the metadata 18 // version is wrong 19 errWrongMetadataVersion = errors.New("wrong metadata version") 20 21 // metadataVersion is the current version of the siafile Metadata 22 metadataVersion = metadataVersion3 23 24 // metadataVersion3 is the third version of the siafile Metadata 25 metadataVersion3 = [16]byte{3} 26 27 // metadataVersion2 is the second version of the siafile Metadata 28 metadataVersion2 = [16]byte{2} 29 30 // metadataVersion1 is the first version of the siafile Metadata 31 metadataVersion1 = [16]byte{1} 32 33 // nilMetadataVesion is a helper for identifying an uninitialized 34 // metadata version 35 nilMetadataVesion = [16]byte{} 36 ) 37 38 type ( 39 // FileData is a helper struct that contains all the relevant information 40 // of a file. It simplifies passing the necessary data between modules and 41 // keeps the interface clean. 42 FileData struct { 43 Name string 44 FileSize uint64 45 MasterKey [crypto.EntropySize]byte 46 ErasureCode skymodules.ErasureCoder 47 RepairPath string 48 PieceSize uint64 49 Mode os.FileMode 50 Deleted bool 51 UID SiafileUID 52 Chunks []FileChunk 53 } 54 // FileChunk is a helper struct that contains data about a chunk. 55 FileChunk struct { 56 Pieces [][]Piece 57 } 58 ) 59 60 // NewFromLegacyData creates a new SiaFile from data that was previously loaded 61 // from a legacy file. 62 func NewFromLegacyData(fd FileData, siaFilePath string, wal *writeaheadlog.WAL) (*SiaFile, error) { 63 // Legacy master keys are always twofish keys. 64 mk, err := crypto.NewSiaKey(crypto.TypeTwofish, fd.MasterKey[:]) 65 if err != nil { 66 return nil, errors.AddContext(err, "failed to restore master key") 67 } 68 currentTime := time.Now() 69 ecType, ecParams := marshalErasureCoder(fd.ErasureCode) 70 zeroHealth := float64(1 + fd.ErasureCode.MinPieces()/(fd.ErasureCode.NumPieces()-fd.ErasureCode.MinPieces())) 71 file := &SiaFile{ 72 staticMetadata: Metadata{ 73 AccessTime: currentTime, 74 ChunkOffset: defaultReservedMDPages * pageSize, 75 ChangeTime: currentTime, 76 CreateTime: currentTime, 77 CachedHealth: zeroHealth, 78 CachedStuckHealth: 0, 79 CachedRedundancy: 0, 80 CachedUserRedundancy: 0, 81 CachedUploadProgress: 0, 82 FileSize: int64(fd.FileSize), 83 LocalPath: fd.RepairPath, 84 StaticMasterKey: mk.Key(), 85 StaticMasterKeyType: mk.Type(), 86 Mode: fd.Mode, 87 ModTime: currentTime, 88 staticErasureCode: fd.ErasureCode, 89 StaticErasureCodeType: ecType, 90 StaticErasureCodeParams: ecParams, 91 StaticPagesPerChunk: numChunkPagesRequired(fd.ErasureCode.NumPieces()), 92 StaticPieceSize: fd.PieceSize, 93 UniqueID: SiafileUID(fd.UID), 94 }, 95 deps: modules.ProdDependencies, 96 deleted: fd.Deleted, 97 numChunks: len(fd.Chunks), 98 siaFilePath: siaFilePath, 99 wal: wal, 100 } 101 // Update cached fields for 0-Byte files. 102 if file.staticMetadata.FileSize == 0 { 103 file.staticMetadata.CachedHealth = 0 104 file.staticMetadata.CachedStuckHealth = 0 105 file.staticMetadata.CachedRedundancy = float64(fd.ErasureCode.NumPieces()) / float64(fd.ErasureCode.MinPieces()) 106 file.staticMetadata.CachedUserRedundancy = file.staticMetadata.CachedRedundancy 107 file.staticMetadata.CachedUploadProgress = 100 108 } 109 110 // Create the chunks. 111 chunks := make([]chunk, len(fd.Chunks)) 112 for i := range chunks { 113 chunks[i].Pieces = make([][]piece, file.staticMetadata.staticErasureCode.NumPieces()) 114 chunks[i].Index = i 115 } 116 117 // Populate the pubKeyTable of the file and add the pieces. 118 pubKeyMap := make(map[string]uint32) 119 for chunkIndex, chunk := range fd.Chunks { 120 for pieceIndex, pieceSet := range chunk.Pieces { 121 for _, p := range pieceSet { 122 // Check if we already added that public key. 123 tableOffset, exists := pubKeyMap[string(p.HostPubKey.Key)] 124 if !exists { 125 tableOffset = uint32(len(file.pubKeyTable)) 126 pubKeyMap[string(p.HostPubKey.Key)] = tableOffset 127 file.pubKeyTable = append(file.pubKeyTable, HostPublicKey{ 128 PublicKey: p.HostPubKey, 129 Used: true, 130 }) 131 } 132 // Add the piece to the SiaFile. 133 chunks[chunkIndex].Pieces[pieceIndex] = append(chunks[chunkIndex].Pieces[pieceIndex], piece{ 134 HostTableOffset: tableOffset, 135 MerkleRoot: p.MerkleRoot, 136 }) 137 } 138 } 139 } 140 141 // Save file to disk. 142 if err := file.saveFile(chunks); err != nil { 143 return nil, errors.AddContext(err, "unable to save file") 144 } 145 146 // Update the cached fields for progress and uploaded bytes. 147 _, _, err = file.UploadProgressAndBytes() 148 return file, err 149 } 150 151 // metadataCompatCheck handles the compatibility checks for the metadata based 152 // on the version 153 // 154 // NOTE: there is no need to use the backup and restore method of the metadata 155 // here because this is called on load. If there is an error if means we are 156 // unable to load the siafile, and therefore cannot use it which makes restoring 157 // the metadata pointless. 158 func (sf *SiaFile) metadataCompatCheck() error { 159 // Quit early to avoid unnecessary disk write. 160 if sf.staticMetadata.StaticVersion == metadataVersion { 161 return nil 162 } 163 164 // Check uninitialized case 165 if sf.staticMetadata.StaticVersion == nilMetadataVesion { 166 sf.upgradeMetadataFromNilToV1() 167 } 168 169 // Check for version 1 updates. 170 if sf.staticMetadata.StaticVersion == metadataVersion1 { 171 err := sf.upgradeMetadataFromV1ToV2() 172 if err != nil { 173 return err 174 } 175 } 176 177 // Check for version 2 updates. 178 if sf.staticMetadata.StaticVersion == metadataVersion2 { 179 err := sf.upgradeMetadataFromV2ToV3() 180 if err != nil { 181 return err 182 } 183 } 184 185 // Check for current version 186 if sf.staticMetadata.StaticVersion != metadataVersion { 187 return errWrongMetadataVersion 188 } 189 190 // Save Metadata to persist updates 191 err := sf.saveMetadata() 192 if err != nil { 193 return err 194 } 195 196 return nil 197 } 198 199 // upgradeMetadataFromNilToV1 upgrades an uninitialized metadata version to 200 // version 1 with the corresponding compat code 201 func (sf *SiaFile) upgradeMetadataFromNilToV1() { 202 // Sanity Check 203 if sf.staticMetadata.StaticVersion != nilMetadataVesion { 204 build.Critical("upgradeMetadataFromNilToV1 called with non nil metadata") 205 return 206 } 207 208 // COMPATv137 legacy files might not have a unique id. 209 if sf.staticMetadata.UniqueID == "" { 210 sf.staticMetadata.UniqueID = uniqueID() 211 } 212 213 // COMPATv140 legacy 0-byte files might not have correct cached 214 // fields since we never update them once they are created. 215 if sf.staticMetadata.FileSize == 0 { 216 ec := sf.staticMetadata.staticErasureCode 217 sf.staticMetadata.CachedHealth = 0 218 sf.staticMetadata.CachedStuckHealth = 0 219 sf.staticMetadata.CachedRedundancy = float64(ec.NumPieces()) / float64(ec.MinPieces()) 220 sf.staticMetadata.CachedUserRedundancy = sf.staticMetadata.CachedRedundancy 221 sf.staticMetadata.CachedUploadProgress = 100 222 } 223 224 // Update the version now that we have completed the compat updates 225 sf.staticMetadata.StaticVersion = metadataVersion1 226 } 227 228 // upgradeMetadataFromV1ToV2 upgrades a version 1 metadata to a version 2 with 229 // the corresponding compat code 230 func (sf *SiaFile) upgradeMetadataFromV1ToV2() error { 231 // Sanity Check 232 if sf.staticMetadata.StaticVersion != metadataVersion1 { 233 err := errors.New("upgradeMetadataFromV1ToV2 called with non version 1 metadata") 234 build.Critical(err) 235 return err 236 } 237 238 // Stuck vs Unfinished files compatibility check. 239 // 240 // Before unfinished files were introduced a file might have been marked 241 // as stuck if the upload failed. In this case, we don't expect the file 242 // to ever be recoverable since the upload failed. Therefore, we don't 243 // want it marked as stuck, we just want to ignore it and let the 244 // unfinished files code eventually prune it. 245 246 // Get the file's stuck status 247 stuck := sf.numStuckChunks() > 0 248 249 // Get the file's unique uploaded bytes to compare against the file size 250 _, unique, err := sf.uploadedBytes() 251 if err != nil { 252 return err 253 } 254 size := uint64(sf.staticMetadata.FileSize) 255 256 // Determine if the file is finished based on if it ever finished 257 // uploading or has a localpath defined. 258 sf.staticMetadata.Finished = unique >= size || sf.onDisk() 259 260 // If the File is not finished, and stuck, reset the stuck status 261 if !sf.staticMetadata.Finished && stuck { 262 err = sf.setAllStuck(false) 263 if err != nil { 264 return errors.AddContext(err, "unable to mark unfinished file as unstuck") 265 } 266 } 267 268 // Update the version now that we have completed the compat updates 269 sf.staticMetadata.StaticVersion = metadataVersion2 270 return nil 271 } 272 273 // upgradeMetadataFromV2ToV3 upgrades a version 2 metadata to a version 3 with 274 // the corresponding compat code 275 func (sf *SiaFile) upgradeMetadataFromV2ToV3() error { 276 // Sanity Check 277 if sf.staticMetadata.StaticVersion != metadataVersion2 { 278 err := errors.New("upgradeMetadataFromV2ToV3 called with non version 2 metadata") 279 build.Critical(err) 280 return err 281 } 282 283 // Unfinished files update. 284 // 285 // The original intent of instituting the unfinished files was to do a 286 // full reset of the filesystem and let the health loop work through 287 // updating the state of the files. 288 289 // Reset the Finished state. 290 sf.staticMetadata.Finished = false 291 292 // Get the file's stuck status 293 stuck := sf.numStuckChunks() > 0 294 295 // If the File is stuck, reset the stuck status 296 if stuck { 297 err := sf.setAllStuck(false) 298 if err != nil { 299 return errors.AddContext(err, "unable to reset stuck status") 300 } 301 } 302 303 // Update the version now that we have completed the compat updates 304 sf.staticMetadata.StaticVersion = metadataVersion3 305 return nil 306 }