github.com/nebulouslabs/sia@v1.3.7/modules/host/storageobligations.go (about) 1 package host 2 3 // storageobligations.go is responsible for managing the storage obligations 4 // within the host - making sure that any file contracts, transaction 5 // dependencies, file contract revisions, and storage proofs are making it into 6 // the blockchain in a reasonable time. 7 // 8 // NOTE: Currently, the code partially supports changing the storage proof 9 // window in file contract revisions, however the action item code will not 10 // handle it correctly. Until the action item code is improved (to also handle 11 // byzantine situations where the renter submits prior revisions), the host 12 // should not support changing the storage proof window, especially to further 13 // in the future. 14 15 // TODO: Need to queue the action item for checking on the submission status of 16 // the file contract revision. Also need to make sure that multiple actions are 17 // being taken if needed. 18 19 // TODO: Make sure that the origin tranasction set is not submitted to the 20 // transaction pool before addSO is called - if it is, there will be a 21 // duplicate transaction error, and then the storage obligation will return an 22 // error, which is bad. Well, or perhas we just need to have better logic 23 // handling. 24 25 // TODO: Need to make sure that 'revision confirmed' is actually looking only 26 // at the most recent revision (I think it is...) 27 28 // TODO: Make sure that not too many action items are being created. 29 30 // TODO: The ProofConstructed and NegotiationHeight fields of storageObligation 31 // are not set or used. 32 33 import ( 34 "encoding/binary" 35 "encoding/json" 36 "errors" 37 "strconv" 38 39 "github.com/NebulousLabs/Sia/build" 40 "github.com/NebulousLabs/Sia/crypto" 41 "github.com/NebulousLabs/Sia/encoding" 42 "github.com/NebulousLabs/Sia/modules" 43 "github.com/NebulousLabs/Sia/types" 44 45 "github.com/coreos/bbolt" 46 ) 47 48 const ( 49 obligationUnresolved storageObligationStatus = iota // Indicatees that an unitialized value was used. 50 obligationRejected // Indicates that the obligation never got started, no revenue gained or lost. 51 obligationSucceeded // Indicates that the obligation was completed, revenues were gained. 52 obligationFailed // Indicates that the obligation failed, revenues and collateral were lost. 53 ) 54 55 var ( 56 // errDuplicateStorageObligation is returned when the storage obligation 57 // database already has a storage obligation with the provided file 58 // contract. This error should only happen in the event of a developer 59 // mistake. 60 errDuplicateStorageObligation = errors.New("storage obligation has a file contract which conflicts with an existing storage obligation") 61 62 // errInsaneFileContractOutputCounts is returned when a file contract has 63 // the wrong number of outputs for either the valid or missed payouts. 64 errInsaneFileContractOutputCounts = errors.New("file contract has incorrect number of outputs for the valid or missed payouts") 65 66 // errInsaneFileContractRevisionOutputCounts is returned when a file 67 // contract has the wrong number of outputs for either the valid or missed 68 // payouts. 69 errInsaneFileContractRevisionOutputCounts = errors.New("file contract revision has incorrect number of outputs for the valid or missed payouts") 70 71 // errInsaneOriginSetFileContract is returned is the final transaction of 72 // the origin transaction set of a storage obligation does not have a file 73 // contract in the final transaction - there should be a file contract 74 // associated with every storage obligation. 75 errInsaneOriginSetFileContract = errors.New("origin transaction set of storage obligation should have one file contract in the final transaction") 76 77 // errInsaneOriginSetSize is returned if the origin transaction set of a 78 // storage obligation is empty - there should be a file contract associated 79 // with every storage obligation. 80 errInsaneOriginSetSize = errors.New("origin transaction set of storage obligation is size zero") 81 82 // errInsaneRevisionSetRevisionCount is returned if the final transaction 83 // in the revision transaction set of a storage obligation has more or less 84 // than one file contract revision. 85 errInsaneRevisionSetRevisionCount = errors.New("revision transaction set of storage obligation should have one file contract revision in the final transaction") 86 87 // errInsaneStorageObligationRevision is returned if there is an attempted 88 // storage obligation revision which does not have sensical inputs. 89 errInsaneStorageObligationRevision = errors.New("revision to storage obligation does not make sense") 90 91 // errInsaneStorageObligationRevisionData is returned if there is an 92 // attempted storage obligation revision which does not have sensical 93 // inputs. 94 errInsaneStorageObligationRevisionData = errors.New("revision to storage obligation has insane data") 95 96 // errNoBuffer is returned if there is an attempted storage obligation that 97 // needs to have the storage proof submitted in less than 98 // revisionSubmissionBuffer blocks. 99 errNoBuffer = errors.New("file contract rejected because storage proof window is too close") 100 101 // errNoStorageObligation is returned if the requested storage obligation 102 // is not found in the database. 103 errNoStorageObligation = errors.New("storage obligation not found in database") 104 105 // errObligationUnlocked is returned when a storage obligation is being 106 // removed from lock, but is already unlocked. 107 errObligationUnlocked = errors.New("storage obligation is unlocked, and should not be getting unlocked") 108 ) 109 110 type storageObligationStatus uint64 111 112 // storageObligation contains all of the metadata related to a file contract 113 // and the storage contained by the file contract. 114 type storageObligation struct { 115 // Storage obligations are broken up into ordered atomic sectors that are 116 // exactly 4MiB each. By saving the roots of each sector, storage proofs 117 // and modifications to the data can be made inexpensively by making use of 118 // the merkletree.CachedTree. Sectors can be appended, modified, or deleted 119 // and the host can recompute the Merkle root of the whole file without 120 // much computational or I/O expense. 121 SectorRoots []crypto.Hash 122 123 // Variables about the file contract that enforces the storage obligation. 124 // The origin an revision transaction are stored as a set, where the set 125 // contains potentially unconfirmed transactions. 126 ContractCost types.Currency 127 LockedCollateral types.Currency 128 PotentialDownloadRevenue types.Currency 129 PotentialStorageRevenue types.Currency 130 PotentialUploadRevenue types.Currency 131 RiskedCollateral types.Currency 132 TransactionFeesAdded types.Currency 133 134 // The negotiation height specifies the block height at which the file 135 // contract was negotiated. If the origin transaction set is not accepted 136 // onto the blockchain quickly enough, the contract is pruned from the 137 // host. The origin and revision transaction set contain the contracts + 138 // revisions as well as all parent transactions. The parents are necessary 139 // because after a restart the transaction pool may be emptied out. 140 NegotiationHeight types.BlockHeight 141 OriginTransactionSet []types.Transaction 142 RevisionTransactionSet []types.Transaction 143 144 // Variables indicating whether the critical transactions in a storage 145 // obligation have been confirmed on the blockchain. 146 ObligationStatus storageObligationStatus 147 OriginConfirmed bool 148 ProofConfirmed bool 149 ProofConstructed bool 150 RevisionConfirmed bool 151 RevisionConstructed bool 152 } 153 154 func (i storageObligationStatus) String() string { 155 if i == 0 { 156 return "obligationUnresolved" 157 } 158 if i == 1 { 159 return "obligationRejected" 160 } 161 if i == 2 { 162 return "obligationSucceeded" 163 } 164 if i == 3 { 165 return "obligationFailed" 166 } 167 return "storageObligationStatus(" + strconv.FormatInt(int64(i), 10) + ")" 168 } 169 170 // getStorageObligation fetches a storage obligation from the database tx. 171 func getStorageObligation(tx *bolt.Tx, soid types.FileContractID) (so storageObligation, err error) { 172 soBytes := tx.Bucket(bucketStorageObligations).Get(soid[:]) 173 if soBytes == nil { 174 return storageObligation{}, errNoStorageObligation 175 } 176 err = json.Unmarshal(soBytes, &so) 177 if err != nil { 178 return storageObligation{}, err 179 } 180 return so, nil 181 } 182 183 // putStorageObligation places a storage obligation into the database, 184 // overwriting the existing storage obligation if there is one. 185 func putStorageObligation(tx *bolt.Tx, so storageObligation) error { 186 soBytes, err := json.Marshal(so) 187 if err != nil { 188 return err 189 } 190 soid := so.id() 191 return tx.Bucket(bucketStorageObligations).Put(soid[:], soBytes) 192 } 193 194 // expiration returns the height at which the storage obligation expires. 195 func (so storageObligation) expiration() types.BlockHeight { 196 if len(so.RevisionTransactionSet) > 0 { 197 return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewWindowStart 198 } 199 return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].WindowStart 200 } 201 202 // fileSize returns the size of the data protected by the obligation. 203 func (so storageObligation) fileSize() uint64 { 204 if len(so.RevisionTransactionSet) > 0 { 205 return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewFileSize 206 } 207 return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].FileSize 208 } 209 210 // id returns the id of the storage obligation, which is definied by the file 211 // contract id of the file contract that governs the storage contract. 212 func (so storageObligation) id() types.FileContractID { 213 return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContractID(0) 214 } 215 216 // isSane checks that required assumptions about the storage obligation are 217 // correct. 218 func (so storageObligation) isSane() error { 219 // There should be an origin transaction set. 220 if len(so.OriginTransactionSet) == 0 { 221 build.Critical("origin transaction set is empty") 222 return errInsaneOriginSetSize 223 } 224 225 // The final transaction of the origin transaction set should have one file 226 // contract. 227 final := len(so.OriginTransactionSet) - 1 228 fcCount := len(so.OriginTransactionSet[final].FileContracts) 229 if fcCount != 1 { 230 build.Critical("wrong number of file contracts associated with storage obligation:", fcCount) 231 return errInsaneOriginSetFileContract 232 } 233 234 // The file contract in the final transaction of the origin transaction set 235 // should have two valid proof outputs and two missed proof outputs. 236 lenVPOs := len(so.OriginTransactionSet[final].FileContracts[0].ValidProofOutputs) 237 lenMPOs := len(so.OriginTransactionSet[final].FileContracts[0].MissedProofOutputs) 238 if lenVPOs != 2 || lenMPOs != 2 { 239 build.Critical("file contract has wrong number of VPOs and MPOs, expecting 2 each:", lenVPOs, lenMPOs) 240 return errInsaneFileContractOutputCounts 241 } 242 243 // If there is a revision transaction set, there should be one file 244 // contract revision in the final transaction. 245 if len(so.RevisionTransactionSet) > 0 { 246 final = len(so.OriginTransactionSet) - 1 247 fcrCount := len(so.OriginTransactionSet[final].FileContractRevisions) 248 if fcrCount != 1 { 249 build.Critical("wrong number of file contract revisions in final transaction of revision transaction set:", fcrCount) 250 return errInsaneRevisionSetRevisionCount 251 } 252 253 // The file contract revision in the final transaction of the revision 254 // transaction set should have two valid proof outputs and two missed 255 // proof outputs. 256 lenVPOs = len(so.RevisionTransactionSet[final].FileContractRevisions[0].NewValidProofOutputs) 257 lenMPOs = len(so.RevisionTransactionSet[final].FileContractRevisions[0].NewMissedProofOutputs) 258 if lenVPOs != 2 || lenMPOs != 2 { 259 build.Critical("file contract has wrong number of VPOs and MPOs, expecting 2 each:", lenVPOs, lenMPOs) 260 return errInsaneFileContractRevisionOutputCounts 261 } 262 } 263 return nil 264 } 265 266 // merkleRoot returns the file merkle root of a storage obligation. 267 func (so storageObligation) merkleRoot() crypto.Hash { 268 if len(so.RevisionTransactionSet) > 0 { 269 return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewFileMerkleRoot 270 } 271 return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].FileMerkleRoot 272 } 273 274 // payous returns the set of valid payouts and missed payouts that represent 275 // the latest revision for the storage obligation. 276 func (so storageObligation) payouts() (valid []types.SiacoinOutput, missed []types.SiacoinOutput) { 277 valid = make([]types.SiacoinOutput, 2) 278 missed = make([]types.SiacoinOutput, 2) 279 if len(so.RevisionTransactionSet) > 0 { 280 copy(valid, so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewValidProofOutputs) 281 copy(missed, so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewMissedProofOutputs) 282 return 283 } 284 copy(valid, so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].ValidProofOutputs) 285 copy(missed, so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].MissedProofOutputs) 286 return 287 } 288 289 // proofDeadline returns the height by which the storage proof must be 290 // submitted. 291 func (so storageObligation) proofDeadline() types.BlockHeight { 292 if len(so.RevisionTransactionSet) > 0 { 293 return so.RevisionTransactionSet[len(so.RevisionTransactionSet)-1].FileContractRevisions[0].NewWindowEnd 294 } 295 return so.OriginTransactionSet[len(so.OriginTransactionSet)-1].FileContracts[0].WindowEnd 296 } 297 298 // value returns the value of fulfilling the storage obligation to the host. 299 func (so storageObligation) value() types.Currency { 300 return so.ContractCost.Add(so.PotentialDownloadRevenue).Add(so.PotentialStorageRevenue).Add(so.PotentialUploadRevenue).Add(so.RiskedCollateral) 301 } 302 303 // queueActionItem adds an action item to the host at the input height so that 304 // the host knows to perform maintenance on the associated storage obligation 305 // when that height is reached. 306 func (h *Host) queueActionItem(height types.BlockHeight, id types.FileContractID) error { 307 // Sanity check - action item should be at a higher height than the current 308 // block height. 309 if height <= h.blockHeight { 310 h.log.Println("action item queued improperly") 311 } 312 return h.db.Update(func(tx *bolt.Tx) error { 313 // Translate the height into a byte slice. 314 heightBytes := make([]byte, 8) 315 binary.BigEndian.PutUint64(heightBytes, uint64(height)) 316 317 // Get the list of action items already at this height and extend it. 318 bai := tx.Bucket(bucketActionItems) 319 existingItems := bai.Get(heightBytes) 320 var extendedItems = make([]byte, len(existingItems), len(existingItems)+len(id[:])) 321 copy(extendedItems, existingItems) 322 extendedItems = append(extendedItems, id[:]...) 323 return bai.Put(heightBytes, extendedItems) 324 }) 325 } 326 327 // managedAddStorageObligation adds a storage obligation to the host. Because 328 // this operation can return errors, the transactions should not be submitted to 329 // the blockchain until after this function has indicated success. All of the 330 // sectors that are present in the storage obligation should already be on disk, 331 // which means that addStorageObligation should be exclusively called when 332 // creating a new, empty file contract or when renewing an existing file 333 // contract. 334 func (h *Host) managedAddStorageObligation(so storageObligation) error { 335 var soid types.FileContractID 336 err := func() error { 337 h.mu.Lock() 338 defer h.mu.Unlock() 339 340 // Sanity check - obligation should be under lock while being added. 341 soid = so.id() 342 _, exists := h.lockedStorageObligations[soid] 343 if !exists { 344 h.log.Critical("addStorageObligation called with an obligation that is not locked") 345 } 346 // Sanity check - There needs to be enough time left on the file contract 347 // for the host to safely submit the file contract revision. 348 if h.blockHeight+revisionSubmissionBuffer >= so.expiration() { 349 h.log.Critical("submission window was not verified before trying to submit a storage obligation") 350 return errNoBuffer 351 } 352 // Sanity check - the resubmission timeout needs to be smaller than storage 353 // proof window. 354 if so.expiration()+resubmissionTimeout >= so.proofDeadline() { 355 h.log.Critical("host is misconfigured - the storage proof window needs to be long enough to resubmit if needed") 356 return errors.New("fill me in") 357 } 358 359 // Add the storage obligation information to the database. 360 err := h.db.Update(func(tx *bolt.Tx) error { 361 // Sanity check - a storage obligation using the same file contract id 362 // should not already exist. This situation can happen if the 363 // transaction pool ejects a file contract and then a new one is 364 // created. Though the file contract will have the same terms, some 365 // other conditions might cause problems. The check for duplicate file 366 // contract ids should happen during the negotiation phase, and not 367 // during the 'addStorageObligation' phase. 368 bso := tx.Bucket(bucketStorageObligations) 369 370 // If the storage obligation already has sectors, it means that the 371 // file contract is being renewed, and that the sector should be 372 // re-added with a new expiration height. If there is an error at any 373 // point, all of the sectors should be removed. 374 if len(so.SectorRoots) != 0 { 375 err := h.AddSectorBatch(so.SectorRoots) 376 if err != nil { 377 return err 378 } 379 } 380 381 // Add the storage obligation to the database. 382 soBytes, err := json.Marshal(so) 383 if err != nil { 384 return err 385 } 386 return bso.Put(soid[:], soBytes) 387 }) 388 if err != nil { 389 return err 390 } 391 392 // Update the host financial metrics with regards to this storage 393 // obligation. 394 h.financialMetrics.ContractCount++ 395 h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Add(so.ContractCost) 396 h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Add(so.LockedCollateral) 397 h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Add(so.PotentialStorageRevenue) 398 h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Add(so.PotentialDownloadRevenue) 399 h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Add(so.PotentialUploadRevenue) 400 h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Add(so.RiskedCollateral) 401 h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Add(so.TransactionFeesAdded) 402 return nil 403 }() 404 if err != nil { 405 return err 406 } 407 408 // Check that the transaction is fully valid and submit it to the 409 // transaction pool. 410 err = h.tpool.AcceptTransactionSet(so.OriginTransactionSet) 411 if err != nil { 412 h.log.Println("Failed to add storage obligation, transaction set was not accepted:", err) 413 return err 414 } 415 416 // Queue the action items. 417 h.mu.Lock() 418 defer h.mu.Unlock() 419 420 // The file contract was already submitted to the blockchain, need to check 421 // after the resubmission timeout that it was submitted successfully. 422 err1 := h.queueActionItem(h.blockHeight+resubmissionTimeout, soid) 423 err2 := h.queueActionItem(h.blockHeight+resubmissionTimeout*2, soid) // Paranoia 424 // Queue an action item to submit the file contract revision - if there is 425 // never a file contract revision, the handling of this action item will be 426 // a no-op. 427 err3 := h.queueActionItem(so.expiration()-revisionSubmissionBuffer, soid) 428 err4 := h.queueActionItem(so.expiration()-revisionSubmissionBuffer+resubmissionTimeout, soid) // Paranoia 429 // The storage proof should be submitted 430 err5 := h.queueActionItem(so.expiration()+resubmissionTimeout, soid) 431 err6 := h.queueActionItem(so.expiration()+resubmissionTimeout*2, soid) // Paranoia 432 err = composeErrors(err1, err2, err3, err4, err5, err6) 433 if err != nil { 434 h.log.Println("Error with transaction set, redacting obligation, id", so.id()) 435 return composeErrors(err, h.removeStorageObligation(so, obligationRejected)) 436 } 437 return nil 438 } 439 440 // modifyStorageObligation will take an updated storage obligation along with a 441 // list of sector changes and update the database to account for all of it. The 442 // sector modifications are only used to update the sector database, they will 443 // not be used to modify the storage obligation (most importantly, this means 444 // that sectorRoots needs to be updated by the calling function). Virtual 445 // sectors will be removed the number of times that they are listed, to remove 446 // multiple instances of the same virtual sector, the virtural sector will need 447 // to appear in 'sectorsRemoved' multiple times. Same with 'sectorsGained'. 448 func (h *Host) modifyStorageObligation(so storageObligation, sectorsRemoved []crypto.Hash, sectorsGained []crypto.Hash, gainedSectorData [][]byte) error { 449 // Sanity check - obligation should be under lock while being modified. 450 soid := so.id() 451 _, exists := h.lockedStorageObligations[soid] 452 if !exists { 453 h.log.Critical("modifyStorageObligation called with an obligation that is not locked") 454 } 455 // Sanity check - there needs to be enough time to submit the file contract 456 // revision to the blockchain. 457 if so.expiration()-revisionSubmissionBuffer <= h.blockHeight { 458 return errNoBuffer 459 } 460 // Sanity check - sectorsGained and gainedSectorData need to have the same length. 461 if len(sectorsGained) != len(gainedSectorData) { 462 h.log.Critical("modifying a revision with garbage sector data", len(sectorsGained), len(gainedSectorData)) 463 return errInsaneStorageObligationRevision 464 } 465 // Sanity check - all of the sector data should be modules.SectorSize 466 for _, data := range gainedSectorData { 467 if uint64(len(data)) != modules.SectorSize { 468 h.log.Critical("modifying a revision with garbase sector sizes", len(data)) 469 return errInsaneStorageObligationRevision 470 } 471 } 472 473 // Note, for safe error handling, the operation order should be: add 474 // sectors, update database, remove sectors. If the adding or update fails, 475 // the added sectors should be removed and the storage obligation shoud be 476 // considered invalid. If the removing fails, this is okay, it's ignored 477 // and left to consistency checks and user actions to fix (will reduce host 478 // capacity, but will not inhibit the host's ability to submit storage 479 // proofs) 480 var i int 481 var err error 482 for i = range sectorsGained { 483 err = h.AddSector(sectorsGained[i], gainedSectorData[i]) 484 if err != nil { 485 break 486 } 487 } 488 if err != nil { 489 // Because there was an error, all of the sectors that got added need 490 // to be reverted. 491 for j := 0; j < i; j++ { 492 // Error is not checked because there's nothing useful that can be 493 // done about an error. 494 _ = h.RemoveSector(sectorsGained[j]) 495 } 496 return err 497 } 498 // Update the database to contain the new storage obligation. 499 var oldSO storageObligation 500 err = h.db.Update(func(tx *bolt.Tx) error { 501 // Get the old storage obligation as a reference to know how to upate 502 // the host financial stats. 503 oldSO, err = getStorageObligation(tx, soid) 504 if err != nil { 505 return err 506 } 507 508 // Store the new storage obligation to replace the old one. 509 return putStorageObligation(tx, so) 510 }) 511 if err != nil { 512 // Because there was an error, all of the sectors that got added need 513 // to be reverted. 514 for i := range sectorsGained { 515 // Error is not checked because there's nothing useful that can be 516 // done about an error. 517 _ = h.RemoveSector(sectorsGained[i]) 518 } 519 return err 520 } 521 // Call removeSector for all of the sectors that have been removed. 522 for k := range sectorsRemoved { 523 // Error is not checkeed because there's nothing useful that can be 524 // done about an error. Failing to remove a sector is not a terrible 525 // place to be, especially if the host can run consistency checks. 526 _ = h.RemoveSector(sectorsRemoved[k]) 527 } 528 529 // Update the financial information for the storage obligation - apply the 530 // new values. 531 h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Add(so.ContractCost) 532 h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Add(so.LockedCollateral) 533 h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Add(so.PotentialStorageRevenue) 534 h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Add(so.PotentialDownloadRevenue) 535 h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Add(so.PotentialUploadRevenue) 536 h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Add(so.RiskedCollateral) 537 h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Add(so.TransactionFeesAdded) 538 539 // Update the financial information for the storage obligation - remove the 540 // old values. 541 h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(oldSO.ContractCost) 542 h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(oldSO.LockedCollateral) 543 h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(oldSO.PotentialStorageRevenue) 544 h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(oldSO.PotentialDownloadRevenue) 545 h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(oldSO.PotentialUploadRevenue) 546 h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(oldSO.RiskedCollateral) 547 h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Sub(oldSO.TransactionFeesAdded) 548 return nil 549 } 550 551 // removeStorageObligation will remove a storage obligation from the host, 552 // either due to failure or success. 553 func (h *Host) removeStorageObligation(so storageObligation, sos storageObligationStatus) error { 554 // Error is not checked, we want to call remove on every sector even if 555 // there are problems - disk health information will be updated. 556 _ = h.RemoveSectorBatch(so.SectorRoots) 557 558 // Update the host revenue metrics based on the status of the obligation. 559 if sos == obligationUnresolved { 560 h.log.Critical("storage obligation 'unresolved' during call to removeStorageObligation, id", so.id()) 561 } 562 if sos == obligationRejected { 563 if h.financialMetrics.TransactionFeeExpenses.Cmp(so.TransactionFeesAdded) >= 0 { 564 h.financialMetrics.TransactionFeeExpenses = h.financialMetrics.TransactionFeeExpenses.Sub(so.TransactionFeesAdded) 565 566 // Remove the obligation statistics as potential risk and income. 567 h.log.Printf("Rejecting storage obligation expiring at block %v, current height is %v. Potential revenue is %v.\n", so.expiration(), h.blockHeight, h.financialMetrics.PotentialContractCompensation.Add(h.financialMetrics.PotentialStorageRevenue).Add(h.financialMetrics.PotentialDownloadBandwidthRevenue).Add(h.financialMetrics.PotentialUploadBandwidthRevenue)) 568 h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(so.ContractCost) 569 h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(so.LockedCollateral) 570 h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(so.PotentialStorageRevenue) 571 h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(so.PotentialDownloadRevenue) 572 h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(so.PotentialUploadRevenue) 573 h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(so.RiskedCollateral) 574 } 575 } 576 if sos == obligationSucceeded { 577 // Empty obligations don't submit a storage proof. The revenue for an empty 578 // storage obligation should equal the contract cost of the obligation 579 revenue := so.ContractCost.Add(so.PotentialStorageRevenue).Add(so.PotentialDownloadRevenue).Add(so.PotentialUploadRevenue) 580 if len(so.SectorRoots) == 0 { 581 h.log.Printf("No need to submit a storage proof for empty contract. Revenue is %v.\n", revenue) 582 } else { 583 h.log.Printf("Successfully submitted a storage proof. Revenue is %v.\n", revenue) 584 } 585 586 // Remove the obligation statistics as potential risk and income. 587 h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(so.ContractCost) 588 h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(so.LockedCollateral) 589 h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(so.PotentialStorageRevenue) 590 h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(so.PotentialDownloadRevenue) 591 h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(so.PotentialUploadRevenue) 592 h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(so.RiskedCollateral) 593 594 // Add the obligation statistics as actual income. 595 h.financialMetrics.ContractCompensation = h.financialMetrics.ContractCompensation.Add(so.ContractCost) 596 h.financialMetrics.StorageRevenue = h.financialMetrics.StorageRevenue.Add(so.PotentialStorageRevenue) 597 h.financialMetrics.DownloadBandwidthRevenue = h.financialMetrics.DownloadBandwidthRevenue.Add(so.PotentialDownloadRevenue) 598 h.financialMetrics.UploadBandwidthRevenue = h.financialMetrics.UploadBandwidthRevenue.Add(so.PotentialUploadRevenue) 599 } 600 if sos == obligationFailed { 601 // Remove the obligation statistics as potential risk and income. 602 h.log.Printf("Missed storage proof. Revenue would have been %v.\n", so.ContractCost.Add(so.PotentialStorageRevenue).Add(so.PotentialDownloadRevenue).Add(so.PotentialUploadRevenue)) 603 h.financialMetrics.PotentialContractCompensation = h.financialMetrics.PotentialContractCompensation.Sub(so.ContractCost) 604 h.financialMetrics.LockedStorageCollateral = h.financialMetrics.LockedStorageCollateral.Sub(so.LockedCollateral) 605 h.financialMetrics.PotentialStorageRevenue = h.financialMetrics.PotentialStorageRevenue.Sub(so.PotentialStorageRevenue) 606 h.financialMetrics.PotentialDownloadBandwidthRevenue = h.financialMetrics.PotentialDownloadBandwidthRevenue.Sub(so.PotentialDownloadRevenue) 607 h.financialMetrics.PotentialUploadBandwidthRevenue = h.financialMetrics.PotentialUploadBandwidthRevenue.Sub(so.PotentialUploadRevenue) 608 h.financialMetrics.RiskedStorageCollateral = h.financialMetrics.RiskedStorageCollateral.Sub(so.RiskedCollateral) 609 610 // Add the obligation statistics as loss. 611 h.financialMetrics.LostStorageCollateral = h.financialMetrics.LostStorageCollateral.Add(so.RiskedCollateral) 612 h.financialMetrics.LostRevenue = h.financialMetrics.LostRevenue.Add(so.ContractCost).Add(so.PotentialStorageRevenue).Add(so.PotentialDownloadRevenue).Add(so.PotentialUploadRevenue) 613 } 614 615 // Update the storage obligation to be finalized but still in-database. The 616 // obligation status is updated so that the user can see how the obligation 617 // ended up, and the sector roots are removed because they are large 618 // objects with little purpose once storage proofs are no longer needed. 619 h.financialMetrics.ContractCount-- 620 so.ObligationStatus = sos 621 so.SectorRoots = nil 622 return h.db.Update(func(tx *bolt.Tx) error { 623 return putStorageObligation(tx, so) 624 }) 625 } 626 627 // threadedHandleActionItem will look at a storage obligation and determine 628 // which action is necessary for the storage obligation to succeed. 629 func (h *Host) threadedHandleActionItem(soid types.FileContractID) { 630 err := h.tg.Add() 631 if err != nil { 632 return 633 } 634 defer h.tg.Done() 635 636 // Lock the storage obligation in question. 637 h.managedLockStorageObligation(soid) 638 defer func() { 639 h.managedUnlockStorageObligation(soid) 640 }() 641 642 // Fetch the storage obligation associated with the storage obligation id. 643 var so storageObligation 644 h.mu.RLock() 645 blockHeight := h.blockHeight 646 err = h.db.View(func(tx *bolt.Tx) error { 647 so, err = getStorageObligation(tx, soid) 648 return err 649 }) 650 h.mu.RUnlock() 651 if err != nil { 652 h.log.Println("Could not get storage obligation:", err) 653 return 654 } 655 656 // Check whether the storage obligation has already been completed. 657 if so.ObligationStatus != obligationUnresolved { 658 // Storage obligation has already been completed, skip action item. 659 return 660 } 661 662 // Check whether the file contract has been seen. If not, resubmit and 663 // queue another action item. Check for death. (signature should have a 664 // kill height) 665 if !so.OriginConfirmed { 666 // Submit the transaction set again, try to get the transaction 667 // confirmed. 668 err := h.tpool.AcceptTransactionSet(so.OriginTransactionSet) 669 if err != nil { 670 h.log.Debugln("Could not get origin transaction set accepted", err) 671 672 // Check if the transaction is invalid with the current consensus set. 673 // If so, the transaction is highly unlikely to ever be confirmed, and 674 // the storage obligation should be removed. This check should come 675 // after logging the errror so that the function can quit. 676 // 677 // TODO: If the host or tpool is behind consensus, might be difficult 678 // to have certainty about the issue. If some but not all of the 679 // parents are confirmed, might be some difficulty. 680 _, t := err.(modules.ConsensusConflict) 681 if t { 682 h.log.Println("Consensus conflict on the origin transaction set, id", so.id()) 683 h.mu.Lock() 684 err = h.removeStorageObligation(so, obligationRejected) 685 h.mu.Unlock() 686 if err != nil { 687 h.log.Println("Error removing storage obligation:", err) 688 } 689 return 690 } 691 } 692 693 // Queue another action item to check the status of the transaction. 694 h.mu.Lock() 695 err = h.queueActionItem(h.blockHeight+resubmissionTimeout, so.id()) 696 h.mu.Unlock() 697 if err != nil { 698 h.log.Println("Error queuing action item:", err) 699 } 700 } 701 702 // Check if the file contract revision is ready for submission. Check for death. 703 if !so.RevisionConfirmed && len(so.RevisionTransactionSet) > 0 && blockHeight >= so.expiration()-revisionSubmissionBuffer { 704 // Sanity check - there should be a file contract revision. 705 rtsLen := len(so.RevisionTransactionSet) 706 if rtsLen < 1 || len(so.RevisionTransactionSet[rtsLen-1].FileContractRevisions) != 1 { 707 h.log.Critical("transaction revision marked as unconfirmed, yet there is no transaction revision") 708 return 709 } 710 711 // Check if the revision has failed to submit correctly. 712 if blockHeight > so.expiration() { 713 // TODO: Check this error. 714 // 715 // TODO: this is not quite right, because a previous revision may 716 // be confirmed, and the origin transaction may be confirmed, which 717 // would confuse the revenue stuff a bit. Might happen frequently 718 // due to the dynamic fee pool. 719 h.log.Println("Full time has elapsed, but the revision transaction could not be submitted to consensus, id", so.id()) 720 h.mu.Lock() 721 h.removeStorageObligation(so, obligationRejected) 722 h.mu.Unlock() 723 return 724 } 725 726 // Queue another action item to check the status of the transaction. 727 h.mu.Lock() 728 err := h.queueActionItem(blockHeight+resubmissionTimeout, so.id()) 729 h.mu.Unlock() 730 if err != nil { 731 h.log.Println("Error queuing action item:", err) 732 } 733 734 // Add a miner fee to the transaction and submit it to the blockchain. 735 revisionTxnIndex := len(so.RevisionTransactionSet) - 1 736 revisionParents := so.RevisionTransactionSet[:revisionTxnIndex] 737 revisionTxn := so.RevisionTransactionSet[revisionTxnIndex] 738 builder, err := h.wallet.RegisterTransaction(revisionTxn, revisionParents) 739 if err != nil { 740 h.log.Println("Error registering transaction:", err) 741 return 742 } 743 _, feeRecommendation := h.tpool.FeeEstimation() 744 if so.value().Div64(2).Cmp(feeRecommendation) < 0 { 745 // There's no sense submitting the revision if the fee is more than 746 // half of the anticipated revenue - fee market went up 747 // unexpectedly, and the money that the renter paid to cover the 748 // fees is no longer enough. 749 builder.Drop() 750 return 751 } 752 txnSize := uint64(len(encoding.MarshalAll(so.RevisionTransactionSet)) + 300) 753 requiredFee := feeRecommendation.Mul64(txnSize) 754 err = builder.FundSiacoins(requiredFee) 755 if err != nil { 756 h.log.Println("Error funding transaction fees", err) 757 builder.Drop() 758 } 759 builder.AddMinerFee(requiredFee) 760 if err != nil { 761 h.log.Println("Error adding miner fees", err) 762 builder.Drop() 763 } 764 feeAddedRevisionTransactionSet, err := builder.Sign(true) 765 if err != nil { 766 h.log.Println("Error signing transaction", err) 767 builder.Drop() 768 } 769 err = h.tpool.AcceptTransactionSet(feeAddedRevisionTransactionSet) 770 if err != nil { 771 h.log.Println("Error submitting transaction to transaction pool", err) 772 builder.Drop() 773 } 774 so.TransactionFeesAdded = so.TransactionFeesAdded.Add(requiredFee) 775 // return 776 } 777 778 // Check whether a storage proof is ready to be provided, and whether it 779 // has been accepted. Check for death. 780 if !so.ProofConfirmed && blockHeight >= so.expiration()+resubmissionTimeout { 781 h.log.Debugln("Host is attempting a storage proof for", so.id()) 782 783 // If the obligation has no sector roots, we can remove the obligation and not 784 // submit a storage proof. The host payout for a failed empty contract 785 // includes the contract cost and locked collateral. 786 if len(so.SectorRoots) == 0 { 787 h.log.Debugln("storage proof not submitted for empty contract, id", so.id()) 788 h.mu.Lock() 789 err := h.removeStorageObligation(so, obligationSucceeded) 790 h.mu.Unlock() 791 if err != nil { 792 h.log.Println("Error removing storage obligation:", err) 793 } 794 return 795 } 796 // If the window has closed, the host has failed and the obligation can 797 // be removed. 798 if so.proofDeadline() < blockHeight { 799 h.log.Debugln("storage proof not confirmed by deadline, id", so.id()) 800 h.mu.Lock() 801 err := h.removeStorageObligation(so, obligationFailed) 802 h.mu.Unlock() 803 if err != nil { 804 h.log.Println("Error removing storage obligation:", err) 805 } 806 return 807 } 808 // Get the index of the segment, and the index of the sector containing 809 // the segment. 810 segmentIndex, err := h.cs.StorageProofSegment(so.id()) 811 if err != nil { 812 h.log.Debugln("Host got an error when fetching a storage proof segment:", err) 813 return 814 } 815 sectorIndex := segmentIndex / (modules.SectorSize / crypto.SegmentSize) 816 // Pull the corresponding sector into memory. 817 sectorRoot := so.SectorRoots[sectorIndex] 818 sectorBytes, err := h.ReadSector(sectorRoot) 819 if err != nil { 820 h.log.Debugln(err) 821 return 822 } 823 824 // Build the storage proof for just the sector. 825 sectorSegment := segmentIndex % (modules.SectorSize / crypto.SegmentSize) 826 base, cachedHashSet := crypto.MerkleProof(sectorBytes, sectorSegment) 827 828 // Using the sector, build a cached root. 829 log2SectorSize := uint64(0) 830 for 1<<log2SectorSize < (modules.SectorSize / crypto.SegmentSize) { 831 log2SectorSize++ 832 } 833 ct := crypto.NewCachedTree(log2SectorSize) 834 ct.SetIndex(segmentIndex) 835 for _, root := range so.SectorRoots { 836 ct.Push(root) 837 } 838 hashSet := ct.Prove(base, cachedHashSet) 839 sp := types.StorageProof{ 840 ParentID: so.id(), 841 HashSet: hashSet, 842 } 843 copy(sp.Segment[:], base) 844 845 // Create and build the transaction with the storage proof. 846 builder, err := h.wallet.StartTransaction() 847 if err != nil { 848 h.log.Println("Failed to start transaction:", err) 849 return 850 } 851 _, feeRecommendation := h.tpool.FeeEstimation() 852 if so.value().Cmp(feeRecommendation) < 0 { 853 // There's no sense submitting the storage proof if the fee is more 854 // than the anticipated revenue. 855 h.log.Debugln("Host not submitting storage proof due to a value that does not sufficiently exceed the fee cost") 856 builder.Drop() 857 return 858 } 859 txnSize := uint64(len(encoding.Marshal(sp)) + 300) 860 requiredFee := feeRecommendation.Mul64(txnSize) 861 err = builder.FundSiacoins(requiredFee) 862 if err != nil { 863 h.log.Println("Host error when funding a storage proof transaction fee:", err) 864 builder.Drop() 865 return 866 } 867 builder.AddMinerFee(requiredFee) 868 builder.AddStorageProof(sp) 869 storageProofSet, err := builder.Sign(true) 870 if err != nil { 871 h.log.Println("Host error when signing the storage proof transaction:", err) 872 builder.Drop() 873 return 874 } 875 err = h.tpool.AcceptTransactionSet(storageProofSet) 876 if err != nil { 877 h.log.Println("Host unable to submit storage proof transaction to transaction pool:", err) 878 builder.Drop() 879 return 880 } 881 so.TransactionFeesAdded = so.TransactionFeesAdded.Add(requiredFee) 882 883 // Queue another action item to check whether the storage proof 884 // got confirmed. 885 h.mu.Lock() 886 err = h.queueActionItem(so.proofDeadline(), so.id()) 887 h.mu.Unlock() 888 if err != nil { 889 h.log.Println("Error queuing action item:", err) 890 } 891 } 892 893 // Save the storage obligation to account for any fee changes. 894 err = h.db.Update(func(tx *bolt.Tx) error { 895 soBytes, err := json.Marshal(so) 896 if err != nil { 897 return err 898 } 899 return tx.Bucket(bucketStorageObligations).Put(soid[:], soBytes) 900 }) 901 if err != nil { 902 h.log.Println("Error updating the storage obligations", err) 903 } 904 905 // Check if all items have succeeded with the required confirmations. Report 906 // success, delete the obligation. 907 if so.ProofConfirmed && blockHeight >= so.proofDeadline() { 908 h.log.Println("file contract complete, id", so.id()) 909 h.mu.Lock() 910 h.removeStorageObligation(so, obligationSucceeded) 911 h.mu.Unlock() 912 } 913 } 914 915 // StorageObligations fetches the set of storage obligations in the host and 916 // returns metadata on them. 917 func (h *Host) StorageObligations() (sos []modules.StorageObligation) { 918 h.mu.RLock() 919 defer h.mu.RUnlock() 920 921 err := h.db.View(func(tx *bolt.Tx) error { 922 b := tx.Bucket(bucketStorageObligations) 923 err := b.ForEach(func(idBytes, soBytes []byte) error { 924 var so storageObligation 925 err := json.Unmarshal(soBytes, &so) 926 if err != nil { 927 return build.ExtendErr("unable to unmarshal storage obligation:", err) 928 } 929 mso := modules.StorageObligation{ 930 ContractCost: so.ContractCost, 931 DataSize: so.fileSize(), 932 LockedCollateral: so.LockedCollateral, 933 ObligationId: so.id(), 934 PotentialDownloadRevenue: so.PotentialDownloadRevenue, 935 PotentialStorageRevenue: so.PotentialStorageRevenue, 936 PotentialUploadRevenue: so.PotentialUploadRevenue, 937 RiskedCollateral: so.RiskedCollateral, 938 SectorRootsCount: uint64(len(so.SectorRoots)), 939 TransactionFeesAdded: so.TransactionFeesAdded, 940 941 ExpirationHeight: so.expiration(), 942 NegotiationHeight: so.NegotiationHeight, 943 ProofDeadLine: so.proofDeadline(), 944 945 ObligationStatus: so.ObligationStatus.String(), 946 OriginConfirmed: so.OriginConfirmed, 947 ProofConfirmed: so.ProofConfirmed, 948 ProofConstructed: so.ProofConstructed, 949 RevisionConfirmed: so.RevisionConfirmed, 950 RevisionConstructed: so.RevisionConstructed, 951 } 952 sos = append(sos, mso) 953 return nil 954 }) 955 if err != nil { 956 return build.ExtendErr("ForEach failed to get next storage obligation:", err) 957 } 958 return nil 959 }) 960 if err != nil { 961 h.log.Println(build.ExtendErr("database failed to provide storage obligations:", err)) 962 } 963 964 return sos 965 }