github.com/fozzysec/SiaPrime@v0.0.0-20190612043147-66c8e8d11fe3/modules/renter/proto/merkleroots.go (about) 1 package proto 2 3 // TODO currently the cached trees are not persisted and we build them at 4 // startup. For petabytes of data this might take a long time. 5 6 import ( 7 "fmt" 8 "io" 9 10 "SiaPrime/build" 11 "SiaPrime/crypto" 12 "gitlab.com/NebulousLabs/errors" 13 ) 14 15 // merkleRootsCacheHeight is the height of the subTrees in cachedSubTrees. A 16 // height of 7 means that 128 sector roots are covered by a single cached 17 // subTree. 18 const merkleRootsCacheHeight = 7 19 20 // merkleRootsPerCache is the number of merkle roots in a cached subTree of 21 // merkleRootsCacheHeight height. 22 const merkleRootsPerCache = 1 << merkleRootsCacheHeight 23 24 type ( 25 // merkleRoots is a helper struct that makes it easier to add/insert/remove 26 // merkleRoots within a SafeContract. 27 // Modifying the merkleRoots is not ACID. This means that the SafeContract 28 // has to make sure it uses the WAL correctly to guarantee ACID updates to 29 // the underlying file. 30 merkleRoots struct { 31 // cachedSubTrees are cached trees that can be used to more efficiently 32 // compute the merkle root of a contract. 33 cachedSubTrees []*cachedSubTree 34 // uncachedRoots contains the sector roots that are not part of a 35 // cached subTree. The uncachedRoots slice should never get longer than 36 // 2^merkleRootsCacheHeight since that would simply result in a new 37 // cached subTree in cachedSubTrees. 38 uncachedRoots []crypto.Hash 39 40 // rootsFile is the rootsFile of the safe contract that contains the roots. 41 rootsFile *fileSection 42 // numMerkleRoots is the number of merkle roots in file. 43 numMerkleRoots int 44 } 45 46 // cachedSubTree is a cached subTree of a merkle tree. A height of 0 means 47 // that the sum is the hash of a leaf. A subTree of height 1 means sum is 48 // the root of 2 leaves. A subTree of height 2 contains the root of 4 49 // leaves and so on. 50 cachedSubTree struct { 51 height int // height of the subTree 52 sum crypto.Hash // root of the subTree 53 } 54 ) 55 56 // parseRootsFromData takes some data and splits it up into sector roots. It will return an error if the size of the data is not a multiple of crypto.HashSize. 57 func parseRootsFromData(b []byte) ([]crypto.Hash, error) { 58 var roots []crypto.Hash 59 if len(b)%crypto.HashSize != 0 { 60 return roots, errors.New("roots have unexpected length and might be corrupted") 61 } 62 63 var root crypto.Hash 64 for len(b) > 0 { 65 copy(root[:], b[:crypto.HashSize]) 66 roots = append(roots, root) 67 b = b[crypto.HashSize:] 68 } 69 return roots, nil 70 } 71 72 // loadExistingMerkleRoots reads creates a merkleRoots object from existing 73 // merkle roots. 74 func loadExistingMerkleRoots(file *fileSection) (*merkleRoots, error) { 75 mr := &merkleRoots{ 76 rootsFile: file, 77 } 78 // Get the number of roots stored in the file. 79 var err error 80 mr.numMerkleRoots, err = mr.lenFromFile() 81 if err != nil { 82 return nil, err 83 } 84 // Read the roots from the file without reading all of them at once. 85 readOff := int64(0) 86 rootsData := make([]byte, rootsDiskLoadBulkSize) 87 for { 88 n, err := file.ReadAt(rootsData, readOff) 89 if err == io.ErrUnexpectedEOF && n == 0 { 90 break 91 } 92 if err == io.EOF && n == 0 { 93 break 94 } 95 if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { 96 return nil, err 97 } 98 roots, err := parseRootsFromData(rootsData[:n]) 99 if err != nil { 100 return nil, err 101 } 102 mr.appendRootMemory(roots...) 103 readOff += int64(n) 104 } 105 return mr, nil 106 } 107 108 // newCachedSubTree creates a cachedSubTree from exactly 109 // 2^merkleRootsCacheHeight roots. 110 func newCachedSubTree(roots []crypto.Hash) *cachedSubTree { 111 // Sanity check the input length. 112 if len(roots) != merkleRootsPerCache { 113 build.Critical("can't create a cached subTree from the provided number of roots") 114 } 115 return &cachedSubTree{ 116 height: int(merkleRootsCacheHeight + sectorHeight), 117 sum: cachedMerkleRoot(roots), 118 } 119 } 120 121 // newMerkleRoots creates a new merkleRoots object. This doesn't load existing 122 // roots from file and will assume that the file doesn't contain any roots. 123 // Don't use this on a file that contains roots. 124 func newMerkleRoots(file *fileSection) *merkleRoots { 125 return &merkleRoots{ 126 rootsFile: file, 127 } 128 } 129 130 // fileOffsetFromRootIndex calculates the offset of the merkle root at index i from 131 // the beginning of the contract file. 132 func fileOffsetFromRootIndex(i int) int64 { 133 return crypto.HashSize * int64(i) 134 } 135 136 // appendRootMemory appends a root to the in-memory structure of the merkleRoots. If 137 // the length of the uncachedRoots grows too large they will be compressed into 138 // a cachedSubTree. 139 func (mr *merkleRoots) appendRootMemory(roots ...crypto.Hash) { 140 for _, root := range roots { 141 mr.uncachedRoots = append(mr.uncachedRoots, root) 142 if len(mr.uncachedRoots) == merkleRootsPerCache { 143 mr.cachedSubTrees = append(mr.cachedSubTrees, newCachedSubTree(mr.uncachedRoots)) 144 mr.uncachedRoots = mr.uncachedRoots[:0] 145 } 146 } 147 } 148 149 // delete deletes the sector root at a certain index by replacing it with the 150 // last root and truncates the file to truncateSize after that. This ensures 151 // that the operation is indempotent. 152 func (mr *merkleRoots) delete(i int, lastRoot crypto.Hash, truncateSize int64) error { 153 // Swap the element at index i with the lastRoot. This might actually 154 // increase mr.numMerkleRoots since there is a chance that i points to an 155 // index after the end of the file. That's why the insert is executed first 156 // before truncating the file or decreasing the numMerkleRoots field. 157 if err := mr.insert(i, lastRoot); err != nil { 158 return errors.AddContext(err, "failed to swap deleted root with newRoot") 159 } 160 // Truncate the file to truncateSize. 161 if err := mr.rootsFile.Truncate(truncateSize); err != nil { 162 return errors.AddContext(err, "failed to truncate file") 163 } 164 // Adjust the numMerkleRoots field. If the number of roots didn't change we 165 // are done. 166 rootsBefore := mr.numMerkleRoots 167 mr.numMerkleRoots = int(truncateSize / crypto.HashSize) 168 if rootsBefore == mr.numMerkleRoots { 169 return nil 170 } 171 // Sanity check the number of roots. 172 if rootsBefore != mr.numMerkleRoots+1 { 173 build.Critical("a delete should never delete more than one root at once") 174 } 175 // If the last element is uncached we can simply remove it from the slice. 176 if len(mr.uncachedRoots) > 0 { 177 mr.uncachedRoots = mr.uncachedRoots[:len(mr.uncachedRoots)-1] 178 return nil 179 } 180 // If it is not uncached we need to delete the last cached tree and load 181 // its elements into mr.uncachedRoots. This should give us 182 // merkleRootsPerCache-1 uncached roots since we already truncated the file 183 // by 1 root. 184 if err := mr.moveLastCachedSubTreeToUncached(); err != nil { 185 return err 186 } 187 return nil 188 } 189 190 // insert inserts a root by replacing a root at an existing index. 191 func (mr *merkleRoots) insert(index int, root crypto.Hash) error { 192 // If the index does point to an offset beyond the end of the file we fill 193 // in the blanks with empty merkle roots. This usually just means that the 194 // machine crashed during the recovery process and that the next few 195 // updates are probably going to be delete operations that take care of the 196 // blank roots. 197 for index > mr.numMerkleRoots { 198 if err := mr.push(crypto.Hash{}); err != nil { 199 return errors.AddContext(err, "failed to extend roots") 200 } 201 } 202 if index == mr.numMerkleRoots { 203 return mr.push(root) 204 } 205 // Replaced the root on disk. 206 _, err := mr.rootsFile.WriteAt(root[:], fileOffsetFromRootIndex(index)) 207 if err != nil { 208 return errors.AddContext(err, "failed to insert root on disk") 209 } 210 211 // Find out if the root is in mr.cachedSubTree or mr.uncachedRoots. 212 i, cached := mr.isIndexCached(index) 213 // If the root was not cached we can simply replace it in mr.uncachedRoots. 214 if !cached { 215 mr.uncachedRoots[i] = root 216 return nil 217 } 218 // If the root was cached we need to rebuild the cache. 219 if err := mr.rebuildCachedTree(i); err != nil { 220 return errors.AddContext(err, "failed to rebuild cache for inserted root") 221 } 222 return nil 223 } 224 225 // isIndexCached determines if the root at index i is already cached in 226 // mr.cachedSubTree or if it is still in mr.uncachedRoots. It will return true 227 // or false and the index of the root in the corresponding data structure. 228 func (mr *merkleRoots) isIndexCached(i int) (int, bool) { 229 if i/merkleRootsPerCache == len(mr.cachedSubTrees) { 230 // Root is not cached. Return the false and the position in 231 // mr.uncachedRoots 232 return i - len(mr.cachedSubTrees)*merkleRootsPerCache, false 233 } 234 return i / merkleRootsPerCache, true 235 } 236 237 // lenFromFile returns the number of merkle roots by computing it from the 238 // filesize. 239 func (mr *merkleRoots) lenFromFile() (int, error) { 240 size, err := mr.rootsFile.Size() 241 if err != nil { 242 return 0, err 243 } 244 245 // Sanity check contract file length. 246 if size%crypto.HashSize != 0 { 247 return 0, errors.New("contract file has unexpected length and might be corrupted") 248 } 249 return int(size / crypto.HashSize), nil 250 } 251 252 // len returns the number of merkle roots. It should always return the same 253 // number as lenFromFile. 254 func (mr *merkleRoots) len() int { 255 return mr.numMerkleRoots 256 } 257 258 // moveLastCachedSubTreeToUncached deletes the last cached subTree and appends 259 // its elements to the uncached roots. 260 func (mr *merkleRoots) moveLastCachedSubTreeToUncached() error { 261 mr.cachedSubTrees = mr.cachedSubTrees[:len(mr.cachedSubTrees)-1] 262 rootIndex := len(mr.cachedSubTrees) * merkleRootsPerCache 263 roots, err := mr.merkleRootsFromIndexFromDisk(rootIndex, mr.numMerkleRoots) 264 if err != nil { 265 return errors.AddContext(err, "failed to read cached tree's roots") 266 } 267 mr.uncachedRoots = append(mr.uncachedRoots, roots...) 268 return nil 269 } 270 271 // push appends a merkle root to the end of the contract. If the number of 272 // uncached merkle roots grows too big we cache them in a new subTree. 273 func (mr *merkleRoots) push(root crypto.Hash) error { 274 // Sanity check the number of uncached roots before adding a new one. 275 if len(mr.uncachedRoots) == merkleRootsPerCache { 276 build.Critical("the number of uncachedRoots is too big. They should've been cached by now") 277 } 278 // Calculate the root offset within the file and write it to disk. 279 rootOffset := fileOffsetFromRootIndex(mr.len()) 280 if _, err := mr.rootsFile.WriteAt(root[:], rootOffset); err != nil { 281 return err 282 } 283 // Add the root to the unached roots. 284 mr.appendRootMemory(root) 285 286 // Increment the number of roots. 287 mr.numMerkleRoots++ 288 return nil 289 } 290 291 // root returns the root of the merkle roots. 292 func (mr *merkleRoots) root() crypto.Hash { 293 tree := crypto.NewTree() 294 for _, st := range mr.cachedSubTrees { 295 if err := tree.PushSubTree(st.height, st.sum[:]); err != nil { 296 // This should never fail. 297 build.Critical(err) 298 } 299 } 300 for _, root := range mr.uncachedRoots { 301 tree.Push(root[:]) 302 } 303 return tree.Root() 304 } 305 306 // checkNewRoot returns the root of the merkleTree after appending the checkNewRoot 307 // without actually appending it. 308 func (mr *merkleRoots) checkNewRoot(newRoot crypto.Hash) crypto.Hash { 309 tree := crypto.NewCachedTree(sectorHeight) 310 for _, st := range mr.cachedSubTrees { 311 if err := tree.PushSubTree(st.height, st.sum); err != nil { 312 // This should never fail. 313 build.Critical(err) 314 } 315 } 316 for _, root := range mr.uncachedRoots { 317 tree.Push(root) 318 } 319 // Push the new root. 320 tree.Push(newRoot) 321 return tree.Root() 322 } 323 324 // merkleRoots reads all the merkle roots from disk and returns them. 325 func (mr *merkleRoots) merkleRoots() (roots []crypto.Hash, err error) { 326 // Get roots. 327 roots, err = mr.merkleRootsFromIndexFromDisk(0, mr.numMerkleRoots) 328 if err != nil { 329 return nil, err 330 } 331 // Sanity check: should have read exactly numMerkleRoots roots. 332 if len(roots) != mr.numMerkleRoots { 333 build.Critical(fmt.Sprintf("Number of merkle roots on disk (%v) doesn't match numMerkleRoots (%v)", 334 len(roots), mr.numMerkleRoots)) 335 } 336 return 337 } 338 339 // merkleRootsFrom index reads all the merkle roots in range [from;to) 340 func (mr *merkleRoots) merkleRootsFromIndexFromDisk(from, to int) ([]crypto.Hash, error) { 341 merkleRoots := make([]crypto.Hash, 0, to-from) 342 remainingData := fileOffsetFromRootIndex(to) - fileOffsetFromRootIndex(from) 343 readOff := fileOffsetFromRootIndex(from) 344 var rootsData []byte 345 for remainingData > 0 { 346 if remainingData > rootsDiskLoadBulkSize { 347 rootsData = make([]byte, rootsDiskLoadBulkSize) 348 } else { 349 rootsData = make([]byte, remainingData) 350 } 351 n, err := mr.rootsFile.ReadAt(rootsData, readOff) 352 if err == io.ErrUnexpectedEOF || err == io.EOF { 353 return nil, errors.New("merkleRootsFromIndexFromDisk failed: roots have unexpected length") 354 } 355 if err != nil { 356 return nil, err 357 } 358 roots, err := parseRootsFromData(rootsData) 359 if err != nil { 360 return nil, err 361 } 362 merkleRoots = append(merkleRoots, roots...) 363 readOff += int64(n) 364 remainingData -= int64(n) 365 } 366 return merkleRoots, nil 367 } 368 369 // prepareDelete is a helper function that returns the lastRoot and trunateSize 370 // arguments for a certain index to call delete with. 371 func (mr *merkleRoots) prepareDelete(index int) (lastRoot crypto.Hash, truncateSize int64, err error) { 372 roots, err := mr.merkleRootsFromIndexFromDisk(mr.numMerkleRoots-1, mr.numMerkleRoots) 373 if err != nil { 374 return crypto.Hash{}, 0, errors.AddContext(err, "failed to get last root") 375 } 376 if len(roots) != 1 { 377 return crypto.Hash{}, 0, fmt.Errorf("expected exactly 1 root but got %v", len(roots)) 378 } 379 return roots[0], int64((mr.numMerkleRoots - 1) * crypto.HashSize), nil 380 } 381 382 // rebuildCachedTree rebuilds the tree in mr.cachedSubTree at index i. 383 func (mr *merkleRoots) rebuildCachedTree(index int) error { 384 // Find the index of the first root of the cached tree on disk. 385 rootIndex := index * merkleRootsPerCache 386 // Read all the roots necessary for creating the cached tree. 387 roots, err := mr.merkleRootsFromIndexFromDisk(rootIndex, rootIndex+(1<<merkleRootsCacheHeight)) 388 if err != nil { 389 return errors.AddContext(err, "failed to read sectors for rebuilding cached tree") 390 } 391 // Replace the old cached tree. 392 mr.cachedSubTrees[index] = newCachedSubTree(roots) 393 return nil 394 }