gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/skynetblocklist/persist_compat_test.go (about) 1 package skynetblocklist 2 3 import ( 4 "bytes" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "path/filepath" 10 "testing" 11 12 "gitlab.com/NebulousLabs/encoding" 13 "gitlab.com/NebulousLabs/errors" 14 "gitlab.com/NebulousLabs/fastrand" 15 "gitlab.com/SkynetLabs/skyd/skymodules" 16 "go.sia.tech/siad/crypto" 17 "go.sia.tech/siad/persist" 18 "go.sia.tech/siad/types" 19 ) 20 21 // TestPersistCompat tests the compat code for the skynet blocklist 22 // persistence. 23 func TestPersistCompat(t *testing.T) { 24 if testing.Short() { 25 t.SkipNow() 26 } 27 t.Parallel() 28 29 // Starting file, v1.4.3 30 t.Run("V143ToV150", testPersistCompatv143Tov150) 31 t.Run("V143ToV151", testPersistCompatv143Tov151) 32 t.Run("V143ToV1510", testPersistCompatv143Tov1510) 33 // Starting file, v1.5.0 34 t.Run("V150ToV151", testPersistCompatv150Tov151) 35 t.Run("V150ToV1510", testPersistCompatv150Tov1510) 36 // Starting file, v1.5.1 37 t.Run("V151ToV1510", testPersistCompatv151Tov1510) 38 39 // Regression Test 40 t.Run("BadCompatTwoFiles", testPersistCompatTwoFiles) 41 } 42 43 // testPersistCompatTwoFiles tests the handling of the persist code when a 44 // blocklist persist file was created without converting the blacklist 45 // persistence 46 // 47 // This occurred when there was a v150 blacklist file and a v151 blocklist file. 48 func testPersistCompatTwoFiles(t *testing.T) { 49 t.Parallel() 50 51 // Create Test directory 52 testdir := testDir(t.Name()) 53 54 // Load a v151 aop to add to the persist file 55 aop, _, err := persist.NewAppendOnlyPersist(testdir, persistFile, metadataHeader, metadataVersionV151) 56 if err != nil { 57 t.Fatal(err) 58 } 59 60 // Add links to it 61 hash1 := crypto.HashObject("link1") 62 hash2 := crypto.HashObject("link2") 63 additions := []crypto.Hash{hash1, hash2} 64 65 // NOTE: can't use UpdateBlocklist method because this is a historical 66 // compat test. So we manually do the marshalling. 67 // 68 // Create buffer for encoder 69 var buf bytes.Buffer 70 // Create and encode the persist links 71 for _, hash := range additions { 72 // Marshal the update 73 pe := persistEntryV151{hash, true} 74 data := encoding.Marshal(pe) 75 _, err := buf.Write(data) 76 if err != nil { 77 t.Fatal(err) 78 } 79 } 80 _, err = aop.Write(buf.Bytes()) 81 if err != nil { 82 t.Fatal(err) 83 } 84 85 // Close 86 err = aop.Close() 87 if err != nil { 88 t.Fatal(err) 89 } 90 91 // Add Blacklist file and load it 92 err = loadCompatPersistFile(testdir, persist.MetadataVersionv150) 93 if err != nil { 94 t.Fatal(err) 95 } 96 oldPersistence, err := loadOldPersistenceV151(testdir, blacklistPersistFile, blacklistMetadataHeader, persist.MetadataVersionv150) 97 if err != nil { 98 t.Fatal(err) 99 } 100 101 // Load SkynetBlocklist. This should pick up the blacklist file and and 102 // remove it. 103 sb, err := New(testdir) 104 if err != nil { 105 t.Fatal(err) 106 } 107 108 // Blacklist persist file should be gone 109 _, err = os.Stat(filepath.Join(testdir, blacklistPersistFile)) 110 if !os.IsNotExist(err) { 111 t.Fatal("blacklist file still exists") 112 } 113 114 // Verify blocklist was not overwritten 115 sb.mu.Lock() 116 defer sb.mu.Unlock() 117 // Old blacklisted links should be in the blocklist 118 for hash := range oldPersistence { 119 _, ok := sb.hashes[hash] 120 if !ok { 121 t.Fatal("old hash not found in new persistence") 122 } 123 } 124 // Newly blocked links should be in the blocklist 125 for _, hash := range additions { 126 _, ok := sb.hashes[hash] 127 if !ok { 128 t.Fatal("added hash not found in new persistence") 129 } 130 } 131 } 132 133 // testPersistCompatv143Tov150 tests converting the skynet blocklist persistence 134 // from v1.4.3 to v1.5.0 135 func testPersistCompatv143Tov150(t *testing.T) { 136 t.Parallel() 137 testdir := testDir(t.Name()) 138 testPersistCompat(t, testdir, blacklistPersistFile, blacklistPersistFile, blacklistMetadataHeader, blacklistMetadataHeader, metadataVersionV143, persist.MetadataVersionv150) 139 } 140 141 // testPersistCompatv143Tov151 tests converting the skynet blacklist persistence 142 // from v1.4.3 to v1.5.1 143 func testPersistCompatv143Tov151(t *testing.T) { 144 t.Parallel() 145 testdir := testDir(t.Name()) 146 testPersistCompat(t, testdir, blacklistPersistFile, persistFile, blacklistMetadataHeader, metadataHeader, metadataVersionV143, metadataVersionV151) 147 } 148 149 // testPersistCompatv143Tov1510 tests converting the skynet blacklist persistence 150 // from v1.4.3 to v1.5.10 151 func testPersistCompatv143Tov1510(t *testing.T) { 152 t.Parallel() 153 testdir := testDir(t.Name()) 154 testPersistCompat(t, testdir, blacklistPersistFile, persistFile, blacklistMetadataHeader, metadataHeader, metadataVersionV143, metadataVersionV1510) 155 } 156 157 // testPersistCompatv150Tov151 tests converting the skynet blacklist persistence 158 // from v1.5.0 to v1.5.1 159 func testPersistCompatv150Tov151(t *testing.T) { 160 t.Parallel() 161 testdir := testDir(t.Name()) 162 testPersistCompat(t, testdir, blacklistPersistFile, persistFile, blacklistMetadataHeader, metadataHeader, persist.MetadataVersionv150, metadataVersionV151) 163 } 164 165 // testPersistCompatv150Tov1510 tests converting the skynet blacklist persistence 166 // from v1.5.0 to v1.5.10 167 func testPersistCompatv150Tov1510(t *testing.T) { 168 t.Parallel() 169 testdir := testDir(t.Name()) 170 testPersistCompat(t, testdir, blacklistPersistFile, persistFile, blacklistMetadataHeader, metadataHeader, persist.MetadataVersionv150, metadataVersionV1510) 171 } 172 173 // testPersistCompatv151Tov1510 tests converting the skynet blocklist persistence 174 // from v1.5.1 to v1.5.10 175 func testPersistCompatv151Tov1510(t *testing.T) { 176 t.Parallel() 177 testdir := testDir(t.Name()) 178 testPersistCompat(t, testdir, persistFile, persistFile, metadataHeader, metadataHeader, metadataVersionV151, metadataVersionV1510) 179 } 180 181 // testPersistCompat tests the persist compat code going between two versions 182 func testPersistCompat(t *testing.T, testdir, oldPersistFile, newPersistFile string, oldHeader, newHeader, oldVersion, newVersion types.Specifier) { 183 t.Run("Clean", func(t *testing.T) { 184 testPersistCompatClean(t, testdir, oldPersistFile, newPersistFile, oldHeader, newHeader, oldVersion, newVersion) 185 }) 186 t.Run("TempFile", func(t *testing.T) { 187 testPersistCompatTempFile(t, testdir, oldPersistFile, newPersistFile, oldHeader, newHeader, oldVersion, newVersion) 188 }) 189 switch oldVersion { 190 case metadataVersionV143, persist.MetadataVersionv150: 191 // This test is broken for older version due to a bug in the 192 // compat code and how the test was written. Previously the 193 // compat code wasn't writing the version in the temp file, 194 // which means there wasn't a way to determine if if was a valid 195 // temp file for the version being converted. This leads to a 196 // valid checksum for an older version that would then corrupt 197 // the data by doing a previous compat conversion again. 198 default: 199 t.Run("ValidChecksum", func(t *testing.T) { 200 testPersistCompatValidCheckSum(t, testdir, oldPersistFile, newPersistFile, oldHeader, newHeader, oldVersion, newVersion) 201 }) 202 } 203 } 204 205 // testPersistCompatClean tests the expected execution of the persist compat 206 // code 207 func testPersistCompatClean(t *testing.T, testdir, oldPersistFile, newPersistFile string, oldHeader, newHeader, oldVersion, newVersion types.Specifier) { 208 // Test 1: Clean conversion 209 210 // Create sub test directory 211 subTestDir := filepath.Join(testdir, "CleanConvert") 212 err := os.MkdirAll(subTestDir, skymodules.DefaultDirPerm) 213 if err != nil { 214 t.Fatal(err) 215 } 216 217 // Initialize the directory with the old version persist file 218 err = loadCompatPersistFile(subTestDir, oldVersion) 219 if err != nil { 220 t.Fatal(err) 221 } 222 223 // Verify the persistence 224 err = loadAndVerifyPersistence(subTestDir, oldPersistFile, newPersistFile, oldHeader, newHeader, oldVersion, newVersion) 225 if err != nil { 226 t.Fatal(err) 227 } 228 229 // Test 2: Clean conversion just calling loadPersist. This always test 230 // to the latest version 231 232 // Create sub test directory 233 subTestDir = filepath.Join(testdir, "CleanConvertB") 234 err = os.MkdirAll(subTestDir, skymodules.DefaultDirPerm) 235 if err != nil { 236 t.Fatal(err) 237 } 238 239 // Initialize the directory with the old version persist file 240 err = loadCompatPersistFile(subTestDir, oldVersion) 241 if err != nil { 242 t.Fatal(err) 243 } 244 245 // Load old persistence for comparison 246 oldPersistence, err := loadOldPersistenceV151(subTestDir, oldPersistFile, oldHeader, oldVersion) 247 if err != nil { 248 t.Fatal(err) 249 } 250 251 // Load the persistence 252 aop, reader, err := loadPersist(subTestDir) 253 if err != nil { 254 t.Fatal(err) 255 } 256 257 // Compare the persistence 258 // NOTE: this is where the latest version is always referenced 259 err = readAndComparePersistence(reader, oldVersion, metadataVersion, oldPersistence) 260 if err != nil { 261 t.Fatal(err) 262 } 263 264 // Close the AOP 265 err = aop.Close() 266 if err != nil { 267 t.Fatal(err) 268 } 269 } 270 271 // testPersistCompatTempFile tests the persist compat code for the case when 272 // there was an unclean shutdown that left an invalid temp file 273 func testPersistCompatTempFile(t *testing.T, testdir, oldPersistFile, newPersistFile string, oldHeader, newHeader, oldVersion, newVersion types.Specifier) { 274 // Test 1: Empty Temp File Exists 275 276 // Create sub test directory 277 subTestDir := filepath.Join(testdir, "EmptyTempFile") 278 err := os.MkdirAll(subTestDir, skymodules.DefaultDirPerm) 279 if err != nil { 280 t.Fatal(err) 281 } 282 283 // Initialize the directory with the old version persist file 284 err = loadCompatPersistFile(subTestDir, oldVersion) 285 if err != nil { 286 t.Fatal(err) 287 } 288 289 // Simulate a crash during the creation a temporary file by creating an empty 290 // temp file 291 f, err := os.Create(filepath.Join(subTestDir, tempPersistFileName(oldPersistFile))) 292 if err != nil { 293 t.Fatal(err) 294 } 295 err = f.Close() 296 if err != nil { 297 t.Fatal(err) 298 } 299 300 // Verify the persistence 301 err = loadAndVerifyPersistence(subTestDir, oldPersistFile, newPersistFile, oldHeader, newHeader, oldVersion, newVersion) 302 if err != nil { 303 t.Fatal(err) 304 } 305 306 // Test 2: Temp File Exists with an invalid checksum 307 308 // Create sub test directory 309 subTestDir = filepath.Join(testdir, "InvalidChecksum") 310 err = os.MkdirAll(subTestDir, skymodules.DefaultDirPerm) 311 if err != nil { 312 t.Fatal(err) 313 } 314 315 // Initialize the directory with the old version persist file 316 err = loadCompatPersistFile(subTestDir, oldVersion) 317 if err != nil { 318 t.Fatal(err) 319 } 320 321 // Simulate a crash during the creation a temporary file by creating a temp 322 // file with random bytes 323 f, err = os.Create(filepath.Join(subTestDir, tempPersistFileName(oldPersistFile))) 324 if err != nil { 325 t.Fatal(err) 326 } 327 _, err = f.Write(fastrand.Bytes(100)) 328 if err != nil { 329 t.Fatal(err) 330 } 331 err = f.Close() 332 if err != nil { 333 t.Fatal(err) 334 } 335 336 // Verify the persistence 337 err = loadAndVerifyPersistence(subTestDir, oldPersistFile, newPersistFile, oldHeader, newHeader, oldVersion, newVersion) 338 if err != nil { 339 t.Fatal(err) 340 } 341 } 342 343 // testPersistCompatValidCheckSum tests the persist compat code for the case 344 // when there was an unclean shutdown that left a temp file with a valid 345 // checksum 346 func testPersistCompatValidCheckSum(t *testing.T, testdir, oldPersistFile, newPersistFile string, oldHeader, newHeader, oldVersion, newVersion types.Specifier) { 347 // Create sub test directory 348 subTestDir := filepath.Join(testdir, "ValidChecksum") 349 err := os.MkdirAll(subTestDir, skymodules.DefaultDirPerm) 350 if err != nil { 351 t.Fatal(err) 352 } 353 354 // Initialize the directory with the old version persist file 355 err = loadCompatPersistFile(subTestDir, oldVersion) 356 if err != nil { 357 t.Fatal(err) 358 } 359 360 // Simulate a crash after creating a temporary file 361 _, err = createTempFileFromPersistFile(subTestDir, oldPersistFile, oldHeader, oldVersion) 362 if err != nil { 363 t.Fatal(err) 364 } 365 366 // Verify the persistence 367 err = loadAndVerifyPersistence(subTestDir, oldPersistFile, newPersistFile, oldHeader, newHeader, oldVersion, newVersion) 368 if err != nil { 369 t.Fatal(err) 370 } 371 } 372 373 // copyFileToTestDir copies the file at fromFilePath and writes it at toFilePath 374 func copyFileToTestDir(fromFilePath, toFilePath string) error { 375 f, err := os.Open(fromFilePath) 376 if err != nil { 377 return err 378 } 379 defer func() { 380 err = errors.Compose(err, f.Close()) 381 }() 382 bytes, err := ioutil.ReadAll(f) 383 if err != nil { 384 return err 385 } 386 pf, err := os.Create(toFilePath) 387 if err != nil { 388 return err 389 } 390 defer func() { 391 err = errors.Compose(err, pf.Close()) 392 }() 393 _, err = pf.Write(bytes) 394 if err != nil { 395 return err 396 } 397 return nil 398 } 399 400 // loadAndVerifyPersistence loads the persistence and verifies that the 401 // conversion updated the persistence as expected 402 func loadAndVerifyPersistence(testDir, oldPersistFile, newPersistFile string, oldHeader, newHeader, oldVersion, newVersion types.Specifier) (err error) { 403 // Load Old Persistence 404 var oldPersistence map[crypto.Hash]struct{} 405 switch oldVersion { 406 case metadataVersionV143, persist.MetadataVersionv150, metadataVersionV151: 407 oldPersistence, err = loadOldPersistenceV151(testDir, oldPersistFile, oldHeader, oldVersion) 408 if err != nil { 409 return errors.AddContext(err, "unable to load old persistence") 410 } 411 default: 412 return fmt.Errorf("%v is not a valid old metadata version, method needs to be updated", oldVersion) 413 } 414 415 // Convert the persistence. 416 switch newVersion { 417 case metadataVersion: 418 errv143 := convertPersistVersionFromv143Tov150(testDir) 419 errv150 := convertPersistVersionFromv150Tov151(testDir) 420 errv151 := convertPersistVersionFromv151Tov1510(testDir) 421 if errv151 != nil { 422 err = errors.Compose(errv143, errv150, errv151) 423 } 424 case metadataVersionV151: 425 errv143 := convertPersistVersionFromv143Tov150(testDir) 426 errv150 := convertPersistVersionFromv150Tov151(testDir) 427 if errv150 != nil { 428 err = errors.Compose(errv143, errv150) 429 } 430 case persist.MetadataVersionv150: 431 errv143 := convertPersistVersionFromv143Tov150(testDir) 432 if errv143 != nil { 433 err = errv143 434 } 435 default: 436 err = fmt.Errorf("%v is now a valid new metadata version", newVersion) 437 } 438 if err != nil { 439 return errors.AddContext(err, "unable to convert persistence") 440 } 441 442 // Load the new persistence 443 aop, reader, err := persist.NewAppendOnlyPersist(testDir, newPersistFile, newHeader, newVersion) 444 if err != nil { 445 return errors.AddContext(err, "unable to open new persistence") 446 } 447 defer func() { 448 err = errors.Compose(err, aop.Close()) 449 }() 450 451 return readAndComparePersistence(reader, oldVersion, newVersion, oldPersistence) 452 } 453 454 // loadCompatPersistFile loads the persist file for the supplied version into 455 // the testDir 456 func loadCompatPersistFile(testDir string, version types.Specifier) error { 457 switch version { 458 case metadataVersionV143: 459 return loadV143CompatPersistFile(testDir) 460 case persist.MetadataVersionv150: 461 return loadV150CompatPersistFile(testDir) 462 case metadataVersionV151: 463 return loadV151CompatPersistFile(testDir) 464 default: 465 } 466 return errors.New("invalid error") 467 } 468 469 // loadOldPersistenceV151 loads the persistence from the old persist file up through compat version v1.5.1 470 func loadOldPersistenceV151(testDir, oldPersistFile string, oldHeader, oldVersion types.Specifier) (_ map[crypto.Hash]struct{}, err error) { 471 // Verify that loading the older persist file works 472 aop, reader, err := persist.NewAppendOnlyPersist(testDir, oldPersistFile, oldHeader, oldVersion) 473 if err != nil { 474 return nil, errors.AddContext(err, "unable to open old persist file") 475 } 476 defer func() { 477 err = errors.Compose(err, aop.Close()) 478 }() 479 480 // Grab the old persistence 481 oldPersistence, err := unmarshalObjectsCompat(reader, oldVersion) 482 if err != nil { 483 return nil, errors.AddContext(err, "unable to unmarshal old persistence") 484 } 485 if len(oldPersistence) == 0 { 486 return nil, errors.New("no data in old version's persist file") 487 } 488 return oldPersistence, nil 489 } 490 491 // loadV143CompatPersistFile loads the v1.4.3 persist file into the testDir 492 func loadV143CompatPersistFile(testDir string) error { 493 v143FileName := filepath.Join("..", "..", "..", "compatibility", blacklistPersistFile+"_v143") 494 return copyFileToTestDir(v143FileName, filepath.Join(testDir, blacklistPersistFile)) 495 } 496 497 // loadV150CompatPersistFile loads the v1.5.0 persist file into the testDir 498 func loadV150CompatPersistFile(testDir string) error { 499 v150FileName := filepath.Join("..", "..", "..", "compatibility", blacklistPersistFile+"_v150") 500 return copyFileToTestDir(v150FileName, filepath.Join(testDir, blacklistPersistFile)) 501 } 502 503 // loadV151CompatPersistFile loads the v1.5.1 persist file into the testDir 504 func loadV151CompatPersistFile(testDir string) error { 505 v151FileName := filepath.Join("..", "..", "..", "compatibility", persistFile+"_v151") 506 return copyFileToTestDir(v151FileName, filepath.Join(testDir, persistFile)) 507 } 508 509 // readAndComparePersistence reads the persistence from the reader and compares 510 // it to the provided oldPersistence 511 func readAndComparePersistence(reader io.Reader, oldVersion, newVersion types.Specifier, oldPersistence map[crypto.Hash]struct{}) (err error) { 512 // Grab the new persistence 513 var newPersistence map[crypto.Hash]int64 514 var newPersistenceV151 map[crypto.Hash]struct{} 515 var newPersistLength int 516 switch newVersion { 517 case persist.MetadataVersionv150, metadataVersionV151: 518 newPersistenceV151, err = unmarshalObjectsCompat(reader, newVersion) 519 if err != nil { 520 return errors.AddContext(err, "unable to unmarshal new persistence v151 and below") 521 } 522 newPersistLength = len(newPersistenceV151) 523 case metadataVersion: 524 newPersistence, err = unmarshalObjects(reader, newVersion) 525 if err != nil { 526 return errors.AddContext(err, "unable to unmarshal new persistence") 527 } 528 newPersistLength = len(newPersistence) 529 default: 530 return fmt.Errorf("%v is not a valid newVersion", newVersion) 531 } 532 if newPersistLength == 0 { 533 return errors.New("no data in new version's persist file") 534 } 535 536 // Verify that the original persistence was properly updated 537 if len(oldPersistence) != newPersistLength { 538 return fmt.Errorf("Expected %v hashes but got %v", newPersistLength, len(oldPersistence)) 539 } 540 for p := range oldPersistence { 541 var hash crypto.Hash 542 switch oldVersion { 543 case metadataVersionV143: 544 hash = crypto.HashObject(p) 545 case persist.MetadataVersionv150, metadataVersionV151: 546 hash = p 547 default: 548 return errors.New("invalid version") 549 } 550 switch newVersion { 551 case persist.MetadataVersionv150, metadataVersionV151: 552 _, ok := newPersistenceV151[hash] 553 if !ok { 554 return fmt.Errorf("Original persistence: %v \nLoaded persistence: %v \n Persist hash not found in list of hashes", oldPersistence, newPersistenceV151) 555 } 556 case metadataVersion: 557 ppe, ok := newPersistence[hash] 558 if !ok { 559 return fmt.Errorf("Original persistence: %v \nLoaded persistence: %v \n Persist hash not found in list of hashes", oldPersistence, newPersistence) 560 } 561 if ppe == 0 { 562 return errors.New("uninitialized probationaryPeriodEnd") 563 } 564 default: 565 return fmt.Errorf("%v is not a valid new version", newVersion) 566 } 567 } 568 return nil 569 }