github.com/avahowell/sia@v0.5.1-beta.0.20160524050156-83dcc3d37c94/modules/host/storagemanager/sector.go (about) 1 package storagemanager 2 3 import ( 4 "bytes" 5 "encoding/base64" 6 "encoding/hex" 7 "encoding/json" 8 "errors" 9 "io/ioutil" 10 "path/filepath" 11 12 "github.com/NebulousLabs/Sia/crypto" 13 "github.com/NebulousLabs/Sia/modules" 14 "github.com/NebulousLabs/Sia/types" 15 16 "github.com/NebulousLabs/bolt" 17 ) 18 19 // TODO: Write a sector consistency check - every sector in the host database 20 // should be represented by a sector on disk, and vice-versa. This is closer to 21 // a testing check, because the host is tolerant of disk corruption - it is 22 // okay for there to be information in the sector usage struct that cannot be 23 // retrieved from the disk. The consistency check should return information on 24 // how much corruption there is an what shape it takes. If there are files 25 // found on disk that are not represented in the usage struct, those files 26 // should be reported as well. The consistency check should be acoompanied by a 27 // 'purge' mode (perhaps multiple modes) which will delete any files in the 28 // storage folders which are not represented in the sector usage database. 29 // 30 // A simliar check should exist for verifying that the host has the correct 31 // folder structure. All of the standard files, plus all of the storage 32 // folders, nothing more. This check belongs in storagefolders.go 33 // 34 // A final check, the obligations check, should verify that every sector in the 35 // sector usage database is represented correctly by the storage obligations, 36 // and that every sector in the storage obligations is represented by the 37 // sector usage database. 38 // 39 // Disk inconsistencies should be handled by returning errors when trying to 40 // read from the filesystem, which means the problem manifests at the lowest 41 // level, the sector level. Because data is missing, there is no 'repair' 42 // operation that can be supported. The sector usage database should match the 43 // storage obligation database, and should be patched if there's a mismatch. 44 // The storage obligation database gets preference. Any missing sectors will be 45 // treated as if they were filesystem problems. 46 // 47 // The consistency check should be wary of 'SizeRemaining' when it is trying to 48 // do cleanup - if sector removals fail, SizeRemaining should not update as 49 // though the sectors are gone (but should also be correct such that it's the 50 // size of the real sectors + the size of the unremovable files - calculated, 51 // not relative) 52 53 // TODO: Write an RPC that lets the host share which sectors it has lost. 54 55 // TODO: Make sure the host will not stutter if it needs to perform operations 56 // on sectors that have been manually deleted. 57 58 var ( 59 // errDiskTrouble is returned when the host is supposed to have enough 60 // storage to hold a new sector but failures that are likely related to the 61 // disk have prevented the host from successfully adding the sector. 62 errDiskTrouble = errors.New("host unable to add sector despite having the storage capacity to do so") 63 64 // errInsufficientStorageForSector is returned if the host tries to add a 65 // sector when there is not enough storage remaining on the host to accept 66 // the sector. 67 // 68 // Ideally, the host will adjust pricing as the host starts to fill up, so 69 // this error should be pretty rare. Demand should drive the price up 70 // faster than the Host runs out of space, such that the host is always 71 // hovering around 95% capacity and rarely over 98% or under 90% capacity. 72 errInsufficientStorageForSector = errors.New("not enough storage remaining to accept sector") 73 74 // errMaxVirtualSectors is returned when a sector cannot be added because 75 // the maximum number of virtual sectors for that sector id already exist. 76 errMaxVirtualSectors = errors.New("sector collides with a physical sector that already has the maximum allowed number of virtual sectors") 77 78 // errSectorNotFound is returned when a lookup for a sector fails. 79 errSectorNotFound = errors.New("could not find the desired sector") 80 ) 81 82 // sectorUsage indicates how a sector is being used. Each block height 83 // represents a point at which a file contract using the sector expires. File 84 // contracts that use the sector multiple times will have their block height 85 // appear multiple times. This data allows the host to figure out what types of 86 // discounts can be applied to data that is reusing sectors. This is primarily 87 // useful for file contract renewals, and really shouldn't be used otherwise. 88 // 89 // The StorageFolder field indicates which storage folder is housing the 90 // sector. 91 type sectorUsage struct { 92 Corrupted bool // If the corrupted flag is set, it means the sector is permanently unreachable. 93 Expiry []types.BlockHeight 94 StorageFolder []byte 95 } 96 97 // sectorID returns the id that should be used when referring to a sector. 98 // There are lots of sectors, and to minimize their footprint a reduced size 99 // hash is used. Hashes are typically 256bits to provide collision resistance 100 // against an attacker that is able to peform an obscene number of trials per 101 // second on each of an obscene number of machines. Potential collisions for 102 // sectors are limited because hosts have secret data that the attacker does 103 // not know which is used to salt the transformation of a sector hash to a 104 // sectorID. As a result, an attacker is limited in the number of times they 105 // can try to cause a collision - one random shot every time they upload a 106 // sector, and the attacker has limited ability to learn of the success of the 107 // attempt. Uploads are very slow, even on fast machines there will be less 108 // than 1000 per second. It is therefore safe to reduce the security from 109 // 256bits to 96bits, which has a collision resistance of 2^48. A reasonable 110 // upper bound for the number of sectors on a host is 2^32, corresponding with 111 // 16PB of data. 112 // 113 // 12 bytes can be represented as a filepath using 16 base64 characters. This 114 // keeps the filesize small and therefore limits the amount of load placed on 115 // the filesystem when trying to manage hundreds of thousands or even tens of 116 // millions of sectors in a single folder. 117 func (sm *StorageManager) sectorID(sectorRootBytes []byte) []byte { 118 saltedRoot := crypto.HashAll(sectorRootBytes, sm.sectorSalt) 119 id := make([]byte, base64.RawURLEncoding.EncodedLen(12)) 120 base64.RawURLEncoding.Encode(id, saltedRoot[:12]) 121 return id 122 } 123 124 // AddSector will add a data sector to the host, correctly selecting the 125 // storage folder in which the sector belongs. 126 func (sm *StorageManager) AddSector(sectorRoot crypto.Hash, expiryHeight types.BlockHeight, sectorData []byte) error { 127 sm.mu.Lock() 128 defer sm.mu.Unlock() 129 130 // Sanity check - sector should have modules.SectorSize bytes. 131 if uint64(len(sectorData)) != modules.SectorSize { 132 sm.log.Critical("incorrectly sized sector passed to AddSector in the storage manager") 133 return errors.New("incorrectly sized sector passed to AddSector in the storage manager") 134 } 135 136 // Check that there is enough room for the sector in at least one storage 137 // folder - check will also guarantee that there is at least one storage folder. 138 enoughRoom := false 139 for _, sf := range sm.storageFolders { 140 if sf.SizeRemaining >= modules.SectorSize { 141 enoughRoom = true 142 } 143 } 144 if !enoughRoom { 145 return errInsufficientStorageForSector 146 } 147 148 // Determine which storage folder is going to receive the new sector. 149 err := sm.db.Update(func(tx *bolt.Tx) error { 150 // Check whether the sector is a virtual sector. 151 sectorKey := sm.sectorID(sectorRoot[:]) 152 bsu := tx.Bucket(bucketSectorUsage) 153 usageBytes := bsu.Get(sectorKey) 154 var usage sectorUsage 155 if usageBytes != nil { 156 // usageBytes != nil indicates that this sector is already a in the 157 // database, meaning it's a virtual sector. Add the expiration 158 // height to the list of expirations, and then return. 159 err := json.Unmarshal(usageBytes, &usage) 160 if err != nil { 161 return err 162 } 163 // If the sector already has the maximum number of virtual sectors, 164 // return an error. The host handles virtual sectors differently 165 // from physical sectors and therefore needs to limit the number of 166 // times that the same data can be uploaded to the host. For 167 // renters that are properly using encryption and are using 168 // sane/reasonable file contract renewal practices, this limit will 169 // never be reached (sane behavior will cause 3-5 at an absolute 170 // maximum, but the limit is substantially higher). 171 if len(usage.Expiry) >= maximumVirtualSectors { 172 return errMaxVirtualSectors 173 } 174 usage.Expiry = append(usage.Expiry, expiryHeight) 175 usageBytes, err = json.Marshal(usage) 176 if err != nil { 177 return err 178 } 179 return bsu.Put(sm.sectorID(sectorRoot[:]), usageBytes) 180 } 181 182 // Try adding the sector to disk. In the event of a failure, the host 183 // will try the next storage folder until there is either a success or 184 // until all options have been exhausted. 185 potentialFolders := sm.storageFolders 186 emptiestFolder, emptiestIndex := emptiestStorageFolder(potentialFolders) 187 for emptiestFolder != nil { 188 sectorPath := filepath.Join(sm.persistDir, emptiestFolder.uidString(), string(sectorKey)) 189 err := sm.dependencies.writeFile(sectorPath, sectorData, 0700) 190 if err != nil { 191 // Indicate to the user that the storage folder is having write 192 // trouble. 193 emptiestFolder.FailedWrites++ 194 195 // Remove the attempted write - an an incomplete write can 196 // leave a partial file on disk. Error is not checked, we 197 // already know the disk is having trouble. 198 _ = sm.dependencies.removeFile(sectorPath) 199 200 // Remove the failed folder from the list of folders that can 201 // be tried. 202 potentialFolders = append(potentialFolders[0:emptiestIndex], potentialFolders[emptiestIndex+1:]...) 203 204 // Try the next folder. 205 emptiestFolder, emptiestIndex = emptiestStorageFolder(potentialFolders) 206 continue 207 } 208 emptiestFolder.SuccessfulWrites++ 209 210 // File write succeeded - add the sector to the sector usage 211 // database and return. 212 usage := sectorUsage{ 213 Expiry: []types.BlockHeight{expiryHeight}, 214 StorageFolder: emptiestFolder.UID, 215 } 216 emptiestFolder.SizeRemaining -= modules.SectorSize 217 usageBytes, err = json.Marshal(usage) 218 if err != nil { 219 return err 220 } 221 return bsu.Put(sectorKey, usageBytes) 222 } 223 224 // There is at least one disk that has room, but the write operation 225 // has failed. 226 return errDiskTrouble 227 }) 228 if err != nil { 229 return err 230 } 231 return sm.save() 232 } 233 234 // ReadSector will pull a sector from disk into memory. 235 func (sm *StorageManager) ReadSector(sectorRoot crypto.Hash) (sectorBytes []byte, err error) { 236 sm.mu.Lock() 237 defer sm.mu.Unlock() 238 239 err = sm.db.View(func(tx *bolt.Tx) error { 240 bsu := tx.Bucket(bucketSectorUsage) 241 sectorKey := sm.sectorID(sectorRoot[:]) 242 sectorUsageBytes := bsu.Get(sectorKey) 243 if sectorUsageBytes == nil { 244 return errSectorNotFound 245 } 246 var su sectorUsage 247 err = json.Unmarshal(sectorUsageBytes, &su) 248 if err != nil { 249 return err 250 } 251 252 sectorPath := filepath.Join(sm.persistDir, hex.EncodeToString(su.StorageFolder), string(sectorKey)) 253 sectorBytes, err = ioutil.ReadFile(sectorPath) 254 if err != nil { 255 // Mark the read failure in the sector. 256 sf := sm.storageFolder(su.StorageFolder) 257 sf.FailedReads++ 258 return err 259 } 260 return nil 261 }) 262 return 263 } 264 265 // RemoveSector will remove a sector from the host at the given expiry height. 266 // If the provided sector does not have an expiration at the given height, an 267 // error will be thrown. 268 func (sm *StorageManager) RemoveSector(sectorRoot crypto.Hash, expiryHeight types.BlockHeight) error { 269 sm.mu.Lock() 270 defer sm.mu.Unlock() 271 272 return sm.db.Update(func(tx *bolt.Tx) error { 273 // Grab the existing sector usage information from the database. 274 bsu := tx.Bucket(bucketSectorUsage) 275 sectorKey := sm.sectorID(sectorRoot[:]) 276 sectorUsageBytes := bsu.Get(sectorKey) 277 if sectorUsageBytes == nil { 278 return errSectorNotFound 279 } 280 var usage sectorUsage 281 err := json.Unmarshal(sectorUsageBytes, &usage) 282 if err != nil { 283 return err 284 } 285 if len(usage.Expiry) == 0 { 286 sm.log.Critical("sector recorded in database, but has no expirations") 287 return errSectorNotFound 288 } 289 if len(usage.Expiry) == 1 && usage.Expiry[0] != expiryHeight { 290 return errSectorNotFound 291 } 292 293 // If there are multiple entries in the usage expiry, it means that the 294 // physcial data is in use by other sectors ('virtual sectors'). This 295 // sector can be removed from the usage expiry, but the physical data 296 // needs to remain. 297 if len(usage.Expiry) > 1 { 298 // Find any single entry in the usage that's at the expiry height 299 // and remove it. 300 var i int 301 found := false 302 for i = 0; i < len(usage.Expiry); i++ { 303 if usage.Expiry[i] == expiryHeight { 304 found = true 305 break 306 } 307 } 308 if !found { 309 return errSectorNotFound 310 } 311 usage.Expiry = append(usage.Expiry[0:i], usage.Expiry[i+1:]...) 312 313 // Update the database with the new usage expiry. 314 sectorUsageBytes, err = json.Marshal(usage) 315 if err != nil { 316 return err 317 } 318 return bsu.Put(sectorKey, sectorUsageBytes) 319 } 320 321 // Get the storage folder that contains the phsyical sector. 322 var folder *storageFolder 323 for _, sf := range sm.storageFolders { 324 if bytes.Equal(sf.UID, usage.StorageFolder) { 325 folder = sf 326 } 327 } 328 329 // Remove the sector from the physical disk and update the storage 330 // folder metadata. 331 sectorPath := filepath.Join(sm.persistDir, hex.EncodeToString(usage.StorageFolder), string(sectorKey)) 332 err = sm.dependencies.removeFile(sectorPath) 333 if err != nil { 334 // Indicate that the storage folder is having write troubles. 335 folder.FailedWrites++ 336 return err 337 } 338 folder.SizeRemaining += modules.SectorSize 339 folder.SuccessfulWrites++ 340 err = sm.save() 341 if err != nil { 342 return err 343 } 344 345 // Delete the sector from the bucket - there are no more instances of 346 // this sector in the host. 347 return bsu.Delete(sm.sectorID(sectorRoot[:])) 348 }) 349 } 350 351 // DeleteSector deletes a sector from the host explicitly, meaning that the 352 // host will be unable to transfer that sector to a renter, and that the host 353 // will be unable to perform a storage proof on that sector. This function is 354 // not intended to be used, however is available so that hosts can easily 355 // comply if compelled by their government to delete certain data. 356 func (sm *StorageManager) DeleteSector(sectorRoot crypto.Hash) error { 357 sm.mu.Lock() 358 defer sm.mu.Unlock() 359 sm.resourceLock.RLock() 360 defer sm.resourceLock.RUnlock() 361 if sm.closed { 362 return errStorageManagerClosed 363 } 364 365 return sm.db.Update(func(tx *bolt.Tx) error { 366 // Check that the sector exists in the database. 367 bsu := tx.Bucket(bucketSectorUsage) 368 sectorKey := sm.sectorID(sectorRoot[:]) 369 sectorUsageBytes := bsu.Get(sectorKey) 370 if sectorUsageBytes == nil { 371 return errSectorNotFound 372 } 373 var usage sectorUsage 374 err := json.Unmarshal(sectorUsageBytes, &usage) 375 if err != nil { 376 return err 377 } 378 379 // Get the storage folder that contains the phsyical sector. 380 var folder *storageFolder 381 for _, sf := range sm.storageFolders { 382 if bytes.Equal(sf.UID, usage.StorageFolder) { 383 folder = sf 384 } 385 } 386 387 // Remove the sector from the physical disk and update the storage 388 // folder metadata. The file is removed from disk as early as possible 389 // to prevent potential errors from stopping the delete. 390 sectorPath := filepath.Join(sm.persistDir, hex.EncodeToString(usage.StorageFolder), string(sectorKey)) 391 err = sm.dependencies.removeFile(sectorPath) 392 if err != nil { 393 // Indicate that the storage folder is having write troubles. 394 folder.FailedWrites++ 395 return err 396 } 397 folder.SizeRemaining += modules.SectorSize 398 folder.SuccessfulWrites++ 399 err = sm.save() 400 if err != nil { 401 return err 402 } 403 404 // After removing the file from disk, remove the file from the 405 // database. 406 return bsu.Delete(sectorKey) 407 }) 408 }