gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/workerjobuploadsnapshot.go (about) 1 package renter 2 3 import ( 4 "bytes" 5 "context" 6 "fmt" 7 "sort" 8 "time" 9 10 "gitlab.com/NebulousLabs/encoding" 11 "gitlab.com/SkynetLabs/skyd/build" 12 "gitlab.com/SkynetLabs/skyd/skymodules" 13 "gitlab.com/SkynetLabs/skyd/skymodules/renter/contractor" 14 "go.sia.tech/siad/crypto" 15 "go.sia.tech/siad/modules" 16 17 "gitlab.com/NebulousLabs/errors" 18 "gitlab.com/NebulousLabs/fastrand" 19 ) 20 21 const ( 22 // snapshotUploadGougingFractionDenom sets the fraction to 1/100 because 23 // uploading backups is important, so there is less sensitivity to gouging. 24 // Also, this is a rare operation. 25 snapshotUploadGougingFractionDenom = 100 26 ) 27 28 type ( 29 // jobUploadSnapshot is a job for the worker to upload a snapshot to its 30 // respective host. 31 jobUploadSnapshot struct { 32 staticSiaFileData []byte 33 34 staticResponseChan chan *jobUploadSnapshotResponse 35 36 jobGeneric 37 } 38 39 // jobUploadSnapshotQueue contains the set of snapshots that need to be 40 // uploaded. 41 jobUploadSnapshotQueue struct { 42 *jobGenericQueue 43 } 44 45 // jobUploadSnapshotResponse contains the response to an upload snapshot 46 // job. 47 jobUploadSnapshotResponse struct { 48 staticErr error 49 } 50 ) 51 52 // checkUploadSnapshotGouging looks at the current renter allowance and the 53 // active settings for a host and determines whether a snapshot upload should be 54 // halted due to price gouging. 55 func checkUploadSnapshotGouging(allowance skymodules.Allowance, hostSettings modules.HostExternalSettings) error { 56 // Check whether the base RPC price is too high. 57 if !allowance.MaxRPCPrice.IsZero() && allowance.MaxRPCPrice.Cmp(hostSettings.BaseRPCPrice) < 0 { 58 errStr := fmt.Sprintf("rpc price of host is %v, which is above the maximum allowed by the allowance: %v", hostSettings.BaseRPCPrice, allowance.MaxRPCPrice) 59 return errors.New(errStr) 60 } 61 // Check whether the upload bandwidth price is too high. 62 if !allowance.MaxUploadBandwidthPrice.IsZero() && allowance.MaxUploadBandwidthPrice.Cmp(hostSettings.UploadBandwidthPrice) < 0 { 63 errStr := fmt.Sprintf("upload bandwidth price of host is %v, which is above the maximum allowed by the allowance: %v", hostSettings.UploadBandwidthPrice, allowance.MaxUploadBandwidthPrice) 64 return errors.New(errStr) 65 } 66 // Check whether the storage price is too high. 67 if !allowance.MaxStoragePrice.IsZero() && allowance.MaxStoragePrice.Cmp(hostSettings.StoragePrice) < 0 { 68 errStr := fmt.Sprintf("storage price of host is %v, which is above the maximum allowed by the allowance: %v", hostSettings.StoragePrice, allowance.MaxStoragePrice) 69 return errors.New(errStr) 70 } 71 72 // If there is no allowance, general price gouging checks have to be 73 // disabled, because there is no baseline for understanding what might count 74 // as price gouging. 75 if allowance.Funds.IsZero() { 76 return nil 77 } 78 79 // Check that the combined prices make sense in the context of the overall 80 // allowance. The general idea is to compute the total cost of performing 81 // the same action repeatedly until a fraction of the desired total resource 82 // consumption established by the allowance has been reached. The fraction 83 // is determined on a case-by-case basis. If the host is too expensive to 84 // even satisfy a faction of the user's total desired resource consumption, 85 // the action will be blocked for price gouging. 86 singleUploadCost := hostSettings.BaseRPCPrice.Add(hostSettings.UploadBandwidthPrice.Mul64(skymodules.StreamDownloadSize)).Add(hostSettings.StoragePrice.Mul64(uint64(allowance.Period)).Mul64(modules.SectorSize)) 87 fullCostPerByte := singleUploadCost.Div64(modules.SectorSize) 88 allowanceStorageCost := fullCostPerByte.Mul64(allowance.ExpectedStorage) 89 reducedCost := allowanceStorageCost.Div64(snapshotUploadGougingFractionDenom) 90 if reducedCost.Cmp(allowance.Funds) > 0 { 91 errStr := fmt.Sprintf("combined fetch backups pricing of host yields %v, which is more than the renter is willing to pay for storage: %v - price gouging protection enabled", reducedCost, allowance.Funds) 92 return errors.New(errStr) 93 } 94 95 return nil 96 } 97 98 // callDiscard will discard this job, sending an error down the response 99 // channel. 100 func (j *jobUploadSnapshot) callDiscard(err error) { 101 resp := &jobUploadSnapshotResponse{ 102 staticErr: err, 103 } 104 w := j.staticQueue.staticWorker() 105 errLaunch := w.staticTG.Launch(func() { 106 select { 107 case j.staticResponseChan <- resp: 108 case <-j.staticCtx.Done(): 109 case <-w.staticTG.StopChan(): 110 } 111 }) 112 if errLaunch != nil { 113 w.staticRenter.staticLog.Print("callDiscard: launch failed", errLaunch) 114 } 115 } 116 117 // callExecute will perform an upload snapshot job for the worker. 118 func (j *jobUploadSnapshot) callExecute() (err error) { 119 w := j.staticQueue.staticWorker() 120 121 // Set the execute time 122 j.externExecuteTime = time.Now() 123 124 // Defer a function to send the result down a channel. 125 defer func() { 126 // Return the error to the caller, error may be nil. 127 resp := &jobUploadSnapshotResponse{ 128 staticErr: err, 129 } 130 errLaunch := w.staticTG.Launch(func() { 131 select { 132 case j.staticResponseChan <- resp: 133 case <-j.staticCtx.Done(): 134 case <-w.staticTG.StopChan(): 135 } 136 }) 137 if errLaunch != nil { 138 w.staticRenter.staticLog.Print("callExecute: launch failed", errLaunch) 139 } 140 141 // Report a failure to the queue if this job had an error. 142 if err != nil { 143 j.staticQueue.callReportFailure(err, j.externExecuteTime, time.Now()) 144 } else { 145 j.staticQueue.callReportSuccess() 146 } 147 }() 148 149 // Check that the worker is good for upload. 150 if !w.staticCache().staticContractUtility.GoodForUpload { 151 err = errors.New("snapshot was not uploaded because the worker is not good for upload") 152 return 153 } 154 155 // Perform the actual upload. 156 var sess contractor.Session 157 sess, err = w.staticRenter.staticHostContractor.Session(w.staticHostPubKey, w.staticTG.StopChan()) 158 if err != nil { 159 w.staticRenter.staticLog.Debugln("unable to grab a session to perform an upload snapshot job:", err) 160 err = errors.AddContext(err, "unable to get host session") 161 return 162 } 163 defer func() { 164 closeErr := sess.Close() 165 if closeErr != nil { 166 w.staticRenter.staticLog.Println("error while closing session:", closeErr) 167 } 168 err = errors.Compose(err, closeErr) 169 }() 170 171 allowance := w.staticRenter.staticHostContractor.Allowance() 172 hostSettings := sess.HostSettings() 173 err = checkUploadSnapshotGouging(allowance, hostSettings) 174 if err != nil { 175 err = errors.AddContext(err, "snapshot upload blocked because potential price gouging was detected") 176 return 177 } 178 179 // Safe cast the metadata to the expected type 180 meta, ok := j.staticMetadata.(skymodules.UploadedBackup) 181 if !ok { 182 build.Critical("unable to cast job metadata") // sanity check 183 return 184 } 185 186 // Upload the snapshot to the host. 187 err = w.staticRenter.managedUploadSnapshotHost(meta, j.staticSiaFileData, sess, w) 188 if err != nil { 189 w.staticRenter.staticLog.Debugln("uploading a snapshot to a host failed:", err) 190 err = errors.AddContext(err, "uploading a snapshot to a host failed") 191 return 192 } 193 194 return 195 } 196 197 // callExpectedBandwidth returns the amount of bandwidth this job is expected to 198 // consume. 199 func (j *jobUploadSnapshot) callExpectedBandwidth() (ul, dl uint64) { 200 // Estimate 50kb in overhead for upload and download, and then 4 MiB 201 // necessary to send the actual full sector payload. 202 return 50e3 + 1<<22, 50e3 203 } 204 205 // initJobUploadSnapshotQueue will initialize the upload snapshot job queue for 206 // the worker. 207 func (w *worker) initJobUploadSnapshotQueue() { 208 if w.staticJobUploadSnapshotQueue != nil { 209 w.staticRenter.staticLog.Critical("should not be double initializng the upload snapshot queue") 210 return 211 } 212 213 w.staticJobUploadSnapshotQueue = &jobUploadSnapshotQueue{ 214 jobGenericQueue: newJobGenericQueue(w), 215 } 216 } 217 218 // managedUploadSnapshotHost uploads a snapshot to a single host. 219 func (r *Renter) managedUploadSnapshotHost(meta skymodules.UploadedBackup, dotSia []byte, host contractor.Session, w *worker) error { 220 // Get the wallet seed. 221 ws, _, err := r.staticWallet.PrimarySeed() 222 if err != nil { 223 return errors.AddContext(err, "failed to get wallet's primary seed") 224 } 225 // Derive the renter seed and wipe the memory once we are done using it. 226 rs := skymodules.DeriveRenterSeed(ws) 227 defer fastrand.Read(rs[:]) 228 // Derive the secret and wipe it afterwards. 229 secret := crypto.HashAll(rs, snapshotKeySpecifier) 230 defer fastrand.Read(secret[:]) 231 232 // split the snapshot .sia file into sectors 233 var sectors [][]byte 234 for buf := bytes.NewBuffer(dotSia); buf.Len() > 0; { 235 sector := make([]byte, modules.SectorSize) 236 copy(sector, buf.Next(len(sector))) 237 sectors = append(sectors, sector) 238 } 239 if len(sectors) > 4 { 240 return errors.New("snapshot is too large") 241 } 242 243 // download the snapshot table 244 entryTable, err := r.managedDownloadSnapshotTable(w) 245 if err != nil && !errors.Contains(err, errEmptyContract) { 246 return errors.AddContext(err, "could not download the snapshot table") 247 } 248 249 // check if the table already contains the entry. 250 for _, existingEntry := range entryTable { 251 if existingEntry.UID == meta.UID { 252 return nil // host already contains entry 253 } 254 } 255 256 // upload the siafile, creating a snapshotEntry 257 var name [96]byte 258 copy(name[:], meta.Name) 259 entry := snapshotEntry{ 260 Name: name, 261 UID: meta.UID, 262 CreationDate: meta.CreationDate, 263 Size: meta.Size, 264 } 265 for j, piece := range sectors { 266 root, err := host.Upload(piece) 267 if err != nil { 268 return errors.AddContext(err, "could not perform host upload") 269 } 270 entry.DataSectors[j] = root 271 } 272 273 shouldOverwrite := len(entryTable) != 0 // only overwrite if the sector already contained an entryTable 274 entryTable = append(entryTable, entry) 275 276 // if entryTable is too large to fit in a sector, repeatedly remove the 277 // oldest entry until it fits 278 id := r.mu.Lock() 279 sort.Slice(r.persist.UploadedBackups, func(i, j int) bool { 280 return r.persist.UploadedBackups[i].CreationDate > r.persist.UploadedBackups[j].CreationDate 281 }) 282 r.mu.Unlock(id) 283 c, _ := crypto.NewSiaKey(crypto.TypeThreefish, secret[:]) 284 for len(encoding.Marshal(entryTable)) > int(modules.SectorSize) { 285 entryTable = entryTable[:len(entryTable)-1] 286 } 287 288 // encode and encrypt the table 289 newTable := make([]byte, modules.SectorSize) 290 copy(newTable[:16], snapshotTableSpecifier[:]) 291 copy(newTable[16:], encoding.Marshal(entryTable)) 292 tableSector := c.EncryptBytes(newTable) 293 294 // swap the new entry table into index 0 and delete the old one 295 // (unless it wasn't an entry table) 296 if _, err := host.Replace(tableSector, 0, shouldOverwrite); err != nil { 297 // Sometimes during the siatests, this will fail with 'write to host 298 // failed; connection reset by peer. This error is very consistent in 299 // TestRemoteBackup, but occurs after everything else has succeeded so 300 // the test doesn't fail. 301 return errors.AddContext(err, "could not perform sector replace for the snapshot") 302 } 303 return nil 304 } 305 306 // UploadSnapshot is a helper method to run a UploadSnapshot job on a worker. 307 func (w *worker) UploadSnapshot(ctx context.Context, meta skymodules.UploadedBackup, dotSia []byte) error { 308 uploadSnapshotRespChan := make(chan *jobUploadSnapshotResponse) 309 jus := &jobUploadSnapshot{ 310 staticSiaFileData: dotSia, 311 staticResponseChan: uploadSnapshotRespChan, 312 313 jobGeneric: newJobGeneric(ctx, w.staticJobUploadSnapshotQueue, meta), 314 } 315 316 // Add the job to the queue. 317 if !w.staticJobUploadSnapshotQueue.callAdd(jus) { 318 return errors.New("worker unavailable") 319 } 320 321 // Wait for the response. 322 var resp *jobUploadSnapshotResponse 323 select { 324 case <-ctx.Done(): 325 return errors.New("UploadSnapshot interrupted") 326 case resp = <-uploadSnapshotRespChan: 327 } 328 return resp.staticErr 329 }