github.com/avahowell/sia@v0.5.1-beta.0.20160524050156-83dcc3d37c94/modules/host/storagemanager/storagefolders_smoke_errors_test.go (about) 1 package storagemanager 2 3 import ( 4 "bytes" 5 "io/ioutil" 6 "os" 7 "path/filepath" 8 "strings" 9 "testing" 10 11 "github.com/NebulousLabs/Sia/modules" 12 ) 13 14 // faultyFS is a mocked filesystem which can be configured to fail for certain 15 // files and folders, as indicated by 'brokenSubstrings'. 16 type faultyFS struct { 17 // brokenSubstrings is a list of substrings that, when appearing in a 18 // filepath, will cause the call to fail. 19 brokenSubstrings []string 20 21 productionDependencies 22 } 23 24 // readFile reads a file from the filesystem. The call will fail if reading 25 // from a file that has a substring which matches the ffs list of broken 26 // substrings. 27 func (ffs faultyFS) readFile(s string) ([]byte, error) { 28 for _, bs := range ffs.brokenSubstrings { 29 if strings.Contains(s, bs) { 30 return nil, mockErrReadFile 31 } 32 } 33 return ffs.productionDependencies.readFile(s) 34 } 35 36 // symlink creates a symlink between a source and a destination file, but will 37 // fail if either filename contains a substring found in the set of broken 38 // substrings. 39 func (ffs faultyFS) symlink(s1, s2 string) error { 40 for _, bs := range ffs.brokenSubstrings { 41 if strings.Contains(s1, bs) || strings.Contains(s2, bs) { 42 return mockErrSymlink 43 } 44 } 45 return ffs.productionDependencies.symlink(s1, s2) 46 } 47 48 // writeFile reads a file from the filesystem. The call will fail if reading 49 // from a file that has a substring which matches the ffs list of broken 50 // substrings. 51 func (ffs faultyFS) writeFile(s string, b []byte, fm os.FileMode) error { 52 // The partial write reqires that there be at least a few bytes, so that a 53 // partial write can be properly simulated. 54 if len(b) < 2 { 55 panic("mocked writeFile requires file data that's at least 2 bytes in length") 56 } 57 58 for _, bs := range ffs.brokenSubstrings { 59 if strings.Contains(s, bs) { 60 // Do a partial write, so that garbase is left on the filesystem 61 // that the code should be trying to clean up. 62 err := ioutil.WriteFile(s, b[:len(b)/2], fm) 63 if err != nil { 64 return err 65 } 66 67 // Return a simulated failure, as the full slice was not written. 68 return mockErrWriteFile 69 } 70 } 71 return ioutil.WriteFile(s, b, fm) 72 } 73 74 // faultyRemove is a mocked set of dependencies that operates as normal except 75 // that removeFile will fail. 76 type faultyRemove struct { 77 productionDependencies 78 } 79 80 // removeFile fails to remove a file from the filesystem. 81 func (faultyRemove) removeFile(s string) error { 82 return mockErrRemoveFile 83 } 84 85 // TestStorageFolderTolerance tests the tolerance of storage folders in the 86 // presence of disk failures. Disk failures should be recorded, and the 87 // failures should be handled gracefully - nonfailing disks should not have 88 // problems. 89 func TestStorageFolderTolerance(t *testing.T) { 90 if testing.Short() { 91 t.SkipNow() 92 } 93 t.Parallel() 94 smt, err := newStorageManagerTester("TestStorageFolderTolerance") 95 if err != nil { 96 t.Fatal(err) 97 } 98 defer smt.Close() 99 100 // Replace the storage manager so that it's using faultyOS for its 101 // dependencies. 102 err = smt.sm.Close() 103 if err != nil { 104 t.Fatal(err) 105 } 106 ffs := new(faultyFS) 107 smt.sm, err = newStorageManager(ffs, filepath.Join(smt.persistDir, modules.StorageManagerDir)) 108 if err != nil { 109 t.Fatal(err) 110 } 111 112 // Add a storage folder when the symlinking is failing. 113 storageFolderOne := filepath.Join(smt.persistDir, "driveOne") 114 ffs.brokenSubstrings = []string{storageFolderOne} 115 err = os.Mkdir(storageFolderOne, 0700) 116 if err != nil { 117 t.Fatal(err) 118 } 119 err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) 120 if err != mockErrSymlink { 121 t.Fatal(err) 122 } 123 124 // Add storage folder one without errors, and then add a sector to the 125 // storage folder. 126 ffs.brokenSubstrings = nil 127 err = smt.sm.AddStorageFolder(storageFolderOne, minimumStorageFolderSize) 128 if err != nil { 129 t.Fatal(err) 130 } 131 sectorRoot, sectorData, err := createSector() 132 if err != nil { 133 t.Fatal(err) 134 } 135 err = smt.sm.AddSector(sectorRoot, 10, sectorData) 136 if err != nil { 137 t.Fatal(err) 138 } 139 // Do a probabilistic reset of the storage manager, to verify that the 140 // persistence structures can reboot without causing issues. 141 err = smt.probabilisticReset() 142 if err != nil { 143 t.Fatal(err) 144 } 145 // Check the filesystem - there should be one sector in the storage folder. 146 infos, err := ioutil.ReadDir(storageFolderOne) 147 if err != nil { 148 t.Fatal(err) 149 } 150 if len(infos) != 1 { 151 t.Fatal("expecting at least one sector in storage folder one") 152 } 153 154 // Replace the storage manager dependencies with the faulty remove, and 155 // then try to remove the sector. 156 smt.sm.dependencies = faultyRemove{} 157 err = smt.sm.RemoveSector(sectorRoot, 10) 158 if err != mockErrRemoveFile { 159 t.Fatal(err) 160 } 161 // Check that the failed write count was incremented for the storage 162 // folder. 163 if smt.sm.storageFolders[0].FailedWrites != 1 { 164 t.Fatal("failed writes counter is not incrementing properly") 165 } 166 // Check the filesystem - sector should still be in the storage folder. 167 infos, err = ioutil.ReadDir(storageFolderOne) 168 if err != nil { 169 t.Fatal(err) 170 } 171 if len(infos) != 1 { 172 t.Fatal("expecting at least one sector in storage folder one") 173 } 174 // Put 'ffs' back as the set of dependencies. 175 smt.sm.dependencies = ffs 176 177 // Add a second storage folder, which can receive the sector when the first 178 // storage folder is deleted. 179 storageFolderTwo := filepath.Join(smt.persistDir, "driveTwo") 180 err = os.Mkdir(storageFolderTwo, 0700) 181 if err != nil { 182 t.Fatal(err) 183 } 184 err = smt.sm.AddStorageFolder(storageFolderTwo, minimumStorageFolderSize*3) 185 if err != nil { 186 t.Fatal(err) 187 } 188 // Do a probabilistic reset of the storage manager, to verify that the 189 // persistence structures can reboot without causing issues. 190 err = smt.probabilisticReset() 191 if err != nil { 192 t.Fatal(err) 193 } 194 195 // Trigger read errors in storage folder one, which means the storage 196 // folder is not going to be able to be deleted successfully. 197 ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[0].uidString())} 198 err = smt.sm.RemoveStorageFolder(0, false) 199 if err != errIncompleteOffload { 200 t.Fatal(err) 201 } 202 // Check that the storage folder was not removed. 203 if len(smt.sm.storageFolders) != 2 { 204 t.Fatal("expecting two storage folders after failed remove") 205 } 206 // Check that the read failure was documented. 207 if smt.sm.storageFolders[0].FailedReads != 1 { 208 t.Error("expecting a read failure to be reported:", smt.sm.storageFolders[0].FailedReads) 209 } 210 // Check the filesystem - there should be one sector in the storage folder, 211 // and none in storage folder two. 212 infos, err = ioutil.ReadDir(storageFolderOne) 213 if err != nil { 214 t.Fatal(err) 215 } 216 if len(infos) != 1 { 217 t.Fatal("expecting at least one sector in storage folder one") 218 } 219 infos, err = ioutil.ReadDir(storageFolderTwo) 220 if err != nil { 221 t.Fatal(err) 222 } 223 if len(infos) != 0 { 224 t.Fatal("expecting zero sectors in storage folder two") 225 } 226 227 // Switch the failure from a read error in the source folder to a write 228 // error in the destination folder. 229 ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[1].uidString())} 230 err = smt.sm.RemoveStorageFolder(0, false) 231 if err != errIncompleteOffload { 232 t.Fatal(err) 233 } 234 // Do a probabilistic reset of the storage manager, to verify that the 235 // persistence structures can reboot without causing issues. 236 err = smt.probabilisticReset() 237 if err != nil { 238 t.Fatal(err) 239 } 240 // Check that the storage folder was not removed. 241 if len(smt.sm.storageFolders) != 2 { 242 t.Fatal("expecting two storage folders after failed remove") 243 } 244 // Check that the read failure was documented. 245 if smt.sm.storageFolders[1].FailedWrites != 1 { 246 t.Error("expecting a read failure to be reported:", smt.sm.storageFolders[1].FailedWrites) 247 } 248 // Check the filesystem - there should be one sector in the storage folder, 249 // and none in storage folder two. 250 infos, err = ioutil.ReadDir(storageFolderOne) 251 if err != nil { 252 t.Fatal(err) 253 } 254 if len(infos) != 1 { 255 t.Fatal("expecting at least one sector in storage folder one") 256 } 257 infos, err = ioutil.ReadDir(storageFolderTwo) 258 if err != nil { 259 t.Fatal(err) 260 } 261 if len(infos) != 0 { 262 t.Fatal("expecting zero sectors in storage folder two") 263 } 264 265 // Try to forcibly remove the first storage folder, while in the presence 266 // of read errors. 267 ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[0].uidString())} 268 uid2 := smt.sm.storageFolders[1].UID 269 err = smt.sm.RemoveStorageFolder(0, true) 270 if err != nil { 271 t.Fatal(err) 272 } 273 // Do a probabilistic reset of the storage manager, to verify that the 274 // persistence structures can reboot without causing issues. 275 err = smt.probabilisticReset() 276 if err != nil { 277 t.Fatal(err) 278 } 279 // Check that the storage folder was removed. 280 if len(smt.sm.storageFolders) != 1 { 281 t.Fatal("expecting two storage folders after failed remove") 282 } 283 if !bytes.Equal(uid2, smt.sm.storageFolders[0].UID) { 284 t.Fatal("storage folder was not removed correctly") 285 } 286 // Check the filesystem - there should be no sectors in storage folder two. 287 infos, err = ioutil.ReadDir(storageFolderTwo) 288 if err != nil { 289 t.Fatal(err) 290 } 291 if len(infos) != 0 { 292 t.Fatal("expecting zero sectors in storage folder two") 293 } 294 295 // Add a storage folder with room for sectors. Because storageFolderOne has 296 // leftover sectors that the program was unable to clean up (due to disk 297 // failure), a third storage folder will be created. 298 ffs.brokenSubstrings = nil 299 storageFolderThree := filepath.Join(smt.persistDir, "driveThree") 300 err = os.Mkdir(storageFolderThree, 0700) 301 if err != nil { 302 t.Fatal(err) 303 } 304 err = smt.sm.AddStorageFolder(storageFolderThree, minimumStorageFolderSize+modules.SectorSize) 305 if err != nil { 306 t.Fatal(err) 307 } 308 // Do a probabilistic reset of the storage manager, to verify that the 309 // persistence structures can reboot without causing issues. 310 err = smt.probabilisticReset() 311 if err != nil { 312 t.Fatal(err) 313 } 314 315 // Fill up the second storage folder, so that resizes can be attempted with 316 // failing disks. storageFolderOne has enough space to store the sectors, 317 // but is having disk troubles. 318 ffs.brokenSubstrings = []string{filepath.Join(smt.persistDir, modules.StorageManagerDir, smt.sm.storageFolders[1].uidString())} 319 numSectors := (minimumStorageFolderSize * 3) / modules.SectorSize 320 for i := uint64(0); i < numSectors; i++ { 321 sectorRoot, sectorData, err := createSector() 322 if err != nil { 323 t.Fatal(err) 324 } 325 err = smt.sm.AddSector(sectorRoot, 11, sectorData) 326 if err != nil { 327 t.Fatal(err) 328 } 329 // Do a probabilistic reset of the storage manager, to verify that the 330 // persistence structures can reboot without causing issues. 331 err = smt.probabilisticReset() 332 if err != nil { 333 t.Fatal(err) 334 } 335 } 336 // Check the filesystem - storage folder one is having disk issues and 337 // should have no sectors. Storage folder two should be full. 338 infos, err = ioutil.ReadDir(storageFolderThree) 339 if err != nil { 340 t.Fatal(err) 341 } 342 if len(infos) != 0 { 343 t.Fatal("expecting zero sectors in storage folder one") 344 } 345 infos, err = ioutil.ReadDir(storageFolderTwo) 346 if err != nil { 347 t.Fatal(err) 348 } 349 if len(infos) != int(numSectors) { 350 t.Fatal("expecting", numSectors, "sectors in storage folder two") 351 } 352 // Try adding another sector, there should be an error because the one disk 353 // is full and the other is having disk troubles. 354 sectorRoot, sectorData, err = createSector() 355 if err != nil { 356 t.Fatal(err) 357 } 358 err = smt.sm.AddSector(sectorRoot, 11, sectorData) 359 if err != errDiskTrouble { 360 t.Fatal(err) 361 } 362 // Do a probabilistic reset of the storage manager, to verify that the 363 // persistence structures can reboot without causing issues. 364 err = smt.probabilisticReset() 365 if err != nil { 366 t.Fatal(err) 367 } 368 // Check the filesystem - storage folder one is having disk issues and 369 // should have no sectors. Storage folder two should be full. 370 infos, err = ioutil.ReadDir(storageFolderThree) 371 if err != nil { 372 t.Fatal(err) 373 } 374 if len(infos) != 0 { 375 t.Fatal("expecting zero sectors in storage folder one") 376 } 377 infos, err = ioutil.ReadDir(storageFolderTwo) 378 if err != nil { 379 t.Fatal(err) 380 } 381 if len(infos) != int(numSectors) { 382 t.Fatal("expecting", numSectors, "sectors in storage folder two") 383 } 384 385 // Add a third storage folder. Then try to resize the second storage folder 386 // such that both storageFolderThree and storageFolderFour have room for 387 // the data, but only storageFolderFour is not haivng disk troubles. 388 storageFolderFour := filepath.Join(smt.persistDir, "driveFour") 389 err = os.Mkdir(storageFolderFour, 0700) 390 if err != nil { 391 t.Fatal(err) 392 } 393 err = smt.sm.AddStorageFolder(storageFolderFour, minimumStorageFolderSize) 394 if err != nil { 395 t.Fatal(err) 396 } 397 // Do a probabilistic reset of the storage manager, to verify that the 398 // persistence structures can reboot without causing issues. 399 err = smt.probabilisticReset() 400 if err != nil { 401 t.Fatal(err) 402 } 403 err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize*2) 404 if err != nil { 405 t.Fatal(err) 406 } 407 // Do a probabilistic reset of the storage manager, to verify that the 408 // persistence structures can reboot without causing issues. 409 err = smt.probabilisticReset() 410 if err != nil { 411 t.Fatal(err) 412 } 413 // Check the filesystem - storageFolderTwo should have 414 // minimumStorageFolderSize*2 worth of sectors, and storageFolderFour 415 // should have minimumStorageFolderSize worth of sectors. 416 infos, err = ioutil.ReadDir(storageFolderThree) 417 if err != nil { 418 t.Fatal(err) 419 } 420 if len(infos) != 0 { 421 t.Fatal("expecting zero sectors in storage folder three") 422 } 423 infos, err = ioutil.ReadDir(storageFolderTwo) 424 if err != nil { 425 t.Fatal(err) 426 } 427 if len(infos) != int(numSectors)-int(minimumStorageFolderSize/modules.SectorSize) { 428 t.Fatal("expecting", numSectors, "sectors in storage folder two") 429 } 430 infos, err = ioutil.ReadDir(storageFolderFour) 431 if err != nil { 432 t.Fatal(err) 433 } 434 if len(infos) != int(minimumStorageFolderSize/modules.SectorSize) { 435 t.Fatal("expecting to have 8 sectors in storageFolderFour") 436 } 437 438 // Trigger an incomplete disk transfer by adding room for one more sector 439 // to storageFolderFour, but then trying to remove a bunch of sectors from 440 // storageFolderTwo. There is enough room on storage folder 3 to make the 441 // operation successful, but it is having disk troubles. 442 err = smt.sm.ResizeStorageFolder(2, minimumStorageFolderSize+modules.SectorSize) 443 if err != nil { 444 t.Fatal(err) 445 } 446 err = smt.sm.ResizeStorageFolder(0, minimumStorageFolderSize) 447 if err != errIncompleteOffload { 448 t.Fatal(err) 449 } 450 // Do a probabilistic reset of the storage manager, to verify that the 451 // persistence structures can reboot without causing issues. 452 err = smt.probabilisticReset() 453 if err != nil { 454 t.Fatal(err) 455 } 456 // Check that the sizes of the storage folders have been updated correctly. 457 if smt.sm.storageFolders[0].Size != minimumStorageFolderSize*2-modules.SectorSize { 458 t.Error("storage folder size was not decreased correctly during the shrink operation") 459 } 460 if smt.sm.storageFolders[0].SizeRemaining != 0 { 461 t.Error("storage folder size remaining was not updated correctly after failed shrink operation") 462 } 463 // Check the filesystem - there should be one less sector in 464 // storageFolderTwo from the previous check, and one more sector in 465 // storageFolderFour. 466 infos, err = ioutil.ReadDir(storageFolderThree) 467 if err != nil { 468 t.Fatal(err) 469 } 470 if len(infos) != 0 { 471 t.Fatal("expecting zero sectors in storage folder three") 472 } 473 infos, err = ioutil.ReadDir(storageFolderTwo) 474 if err != nil { 475 t.Fatal(err) 476 } 477 if len(infos) != int(numSectors)-int(minimumStorageFolderSize/modules.SectorSize)-1 { 478 t.Fatal("expecting", numSectors, "sectors in storage folder two") 479 } 480 infos, err = ioutil.ReadDir(storageFolderFour) 481 if err != nil { 482 t.Fatal(err) 483 } 484 if len(infos) != int(minimumStorageFolderSize/modules.SectorSize)+1 { 485 t.Fatal("filesystem consistency error") 486 } 487 }