gitlab.com/SkynetLabs/skyd@v1.6.9/skymodules/renter/hostdb/hosttree/hosttree_test.go (about) 1 package hosttree 2 3 import ( 4 "fmt" 5 "net" 6 "strconv" 7 "sync" 8 "testing" 9 "time" 10 11 "gitlab.com/NebulousLabs/errors" 12 "gitlab.com/NebulousLabs/fastrand" 13 "gitlab.com/NebulousLabs/threadgroup" 14 15 "gitlab.com/SkynetLabs/skyd/skymodules" 16 "go.sia.tech/siad/crypto" 17 "go.sia.tech/siad/modules" 18 "go.sia.tech/siad/types" 19 ) 20 21 // customScoreBreakdown is a helper struct to create scoreBreakdown's for 22 // testing which return a specific Score. 23 type customScoreBreakdown struct { 24 score types.Currency 25 } 26 27 func (sb customScoreBreakdown) Score() types.Currency { 28 return sb.score 29 } 30 func (sb customScoreBreakdown) ConversionRate(_ types.Currency) float64 { 31 return 0.0 32 } 33 func (sb customScoreBreakdown) HostScoreBreakdown(_ types.Currency, _, _, _ bool) skymodules.HostScoreBreakdown { 34 return skymodules.HostScoreBreakdown{} 35 } 36 func newCustomScoreBreakdown(score types.Currency) ScoreBreakdown { 37 return customScoreBreakdown{ 38 score: score, 39 } 40 } 41 42 func verifyTree(tree *HostTree, nentries int) error { 43 expectedWeight := tree.root.entry.weight.Mul64(uint64(nentries)) 44 if tree.root.weight.Cmp(expectedWeight) != 0 { 45 return fmt.Errorf("expected weight is incorrect: got %v wanted %v", tree.root.weight, expectedWeight) 46 } 47 48 // Check that the length of activeHosts and the count of hostTree are 49 // consistent. 50 if len(tree.hosts) != nentries { 51 return fmt.Errorf("unexpected number of hosts: got %v wanted %v", len(tree.hosts), nentries) 52 } 53 54 // Select many random hosts and do naive statistical analysis on the 55 // results. 56 if !testing.Short() { 57 // Pull a bunch of random hosts and count how many times we pull each 58 // host. 59 selectionMap := make(map[string]int) 60 expected := 100 61 for i := 0; i < expected*nentries; i++ { 62 entries := tree.SelectRandom(1, nil, nil) 63 if len(entries) == 0 { 64 return errors.New("no hosts") 65 } 66 selectionMap[entries[0].PublicKey.String()]++ 67 } 68 69 // See if each host was selected enough times. 70 errorBound := 64 // Pretty large, but will still detect if something is seriously wrong. 71 for _, count := range selectionMap { 72 if count < expected-errorBound || count > expected+errorBound { 73 return errors.New("error bound was breached") 74 } 75 } 76 } 77 78 // Try removing an re-adding all hosts. 79 var removedEntries []*hostEntry 80 for { 81 if tree.root.weight.IsZero() { 82 break 83 } 84 randWeight := fastrand.BigIntn(tree.root.weight.Big()) 85 node := tree.root.nodeAtWeight(types.NewCurrency(randWeight)) 86 node.remove() 87 delete(tree.hosts, node.entry.PublicKey.String()) 88 89 // remove the entry from the hostdb so it won't be selected as a 90 // repeat 91 removedEntries = append(removedEntries, node.entry) 92 } 93 for _, entry := range removedEntries { 94 err := tree.Insert(entry.HostDBEntry) 95 if err != nil { 96 return err 97 } 98 } 99 return nil 100 } 101 102 // makeHostDBEntry makes a new host entry with a random public key. 103 func makeHostDBEntry() skymodules.HostDBEntry { 104 dbe := skymodules.HostDBEntry{} 105 106 _, pk := crypto.GenerateKeyPair() 107 dbe.AcceptingContracts = true 108 dbe.PublicKey = types.Ed25519PublicKey(pk) 109 dbe.ScanHistory = skymodules.HostDBScans{{ 110 Timestamp: time.Now(), 111 Success: true, 112 }} 113 114 return dbe 115 } 116 117 func TestHostTree(t *testing.T) { 118 tree := New(func(hdbe skymodules.HostDBEntry) ScoreBreakdown { 119 return newCustomScoreBreakdown(types.NewCurrency64(20)) 120 }, modules.ProductionResolver{}) 121 122 // Create a bunch of host entries of equal weight. 123 firstInsertions := 64 124 var keys []types.SiaPublicKey 125 for i := 0; i < firstInsertions; i++ { 126 entry := makeHostDBEntry() 127 keys = append(keys, entry.PublicKey) 128 err := tree.Insert(entry) 129 if err != nil { 130 t.Fatal(err) 131 } 132 } 133 err := verifyTree(tree, firstInsertions) 134 if err != nil { 135 t.Error(err) 136 } 137 138 var removed []types.SiaPublicKey 139 // Randomly remove hosts from the tree and check that it is still in order. 140 for _, key := range keys { 141 if fastrand.Intn(1) == 0 { 142 err := tree.Remove(key) 143 if err != nil { 144 t.Fatal(err) 145 } 146 removed = append(removed, key) 147 } 148 } 149 150 err = verifyTree(tree, firstInsertions-len(removed)) 151 if err != nil { 152 t.Error(err) 153 } 154 155 // Do some more insertions. 156 secondInsertions := 64 157 for i := firstInsertions; i < firstInsertions+secondInsertions; i++ { 158 entry := makeHostDBEntry() 159 err := tree.Insert(entry) 160 if err != nil { 161 t.Error(err) 162 } 163 } 164 err = verifyTree(tree, firstInsertions-len(removed)+secondInsertions) 165 if err != nil { 166 t.Error(err) 167 } 168 } 169 170 // Verify that inserting, fetching, deleting, and modifying in parallel from 171 // the hosttree does not cause inconsistency. 172 func TestHostTreeParallel(t *testing.T) { 173 if testing.Short() { 174 t.SkipNow() 175 } 176 177 tree := New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 178 return newCustomScoreBreakdown(types.NewCurrency64(10)) 179 }, modules.ProductionResolver{}) 180 181 // spin up 100 goroutines all randomly inserting, removing, modifying, and 182 // fetching nodes from the tree. 183 var tg threadgroup.ThreadGroup 184 nthreads := 100 185 nelements := 0 186 var mu sync.Mutex 187 for i := 0; i < nthreads; i++ { 188 go func() { 189 if err := tg.Add(); err != nil { 190 t.Error(err) 191 } 192 defer tg.Done() 193 194 inserted := make(map[string]skymodules.HostDBEntry) 195 randEntry := func() *skymodules.HostDBEntry { 196 for _, entry := range inserted { 197 return &entry 198 } 199 return nil 200 } 201 202 for { 203 select { 204 case <-tg.StopChan(): 205 return 206 default: 207 switch fastrand.Intn(4) { 208 // INSERT 209 case 0: 210 entry := makeHostDBEntry() 211 err := tree.Insert(entry) 212 if err != nil { 213 t.Error(err) 214 } 215 inserted[entry.PublicKey.String()] = entry 216 217 mu.Lock() 218 nelements++ 219 mu.Unlock() 220 221 // REMOVE 222 case 1: 223 entry := randEntry() 224 if entry == nil { 225 continue 226 } 227 err := tree.Remove(entry.PublicKey) 228 if err != nil { 229 t.Error(err) 230 } 231 delete(inserted, entry.PublicKey.String()) 232 233 mu.Lock() 234 nelements-- 235 mu.Unlock() 236 237 // MODIFY 238 case 2: 239 entry := randEntry() 240 if entry == nil { 241 continue 242 } 243 newentry := makeHostDBEntry() 244 newentry.PublicKey = entry.PublicKey 245 newentry.NetAddress = "127.0.0.1:31337" 246 247 err := tree.Modify(newentry) 248 if err != nil { 249 t.Error(err) 250 } 251 inserted[entry.PublicKey.String()] = newentry 252 253 // FETCH 254 case 3: 255 tree.SelectRandom(3, nil, nil) 256 } 257 } 258 } 259 }() 260 } 261 262 // let these goroutines operate on the tree for 5 seconds 263 time.Sleep(time.Second * 5) 264 265 // stop the goroutines 266 if err := tg.Stop(); err != nil { 267 t.Fatal(err) 268 } 269 270 // verify the consistency of the tree 271 err := verifyTree(tree, int(nelements)) 272 if err != nil { 273 t.Fatal(err) 274 } 275 } 276 277 func TestHostTreeModify(t *testing.T) { 278 tree := New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 279 return newCustomScoreBreakdown(types.NewCurrency64(10)) 280 }, modules.ProductionResolver{}) 281 282 treeSize := 100 283 var keys []types.SiaPublicKey 284 for i := 0; i < treeSize; i++ { 285 entry := makeHostDBEntry() 286 keys = append(keys, entry.PublicKey) 287 err := tree.Insert(entry) 288 if err != nil { 289 t.Fatal(err) 290 } 291 } 292 293 // should fail with a nonexistent key 294 err := tree.Modify(skymodules.HostDBEntry{}) 295 if !errors.Contains(err, ErrNoSuchHost) { 296 t.Fatalf("modify should fail with ErrNoSuchHost when provided a nonexistent public key. Got error: %v\n", err) 297 } 298 299 targetKey := keys[fastrand.Intn(treeSize)] 300 301 oldEntry := tree.hosts[targetKey.String()].entry 302 newEntry := makeHostDBEntry() 303 newEntry.AcceptingContracts = false 304 newEntry.PublicKey = oldEntry.PublicKey 305 306 err = tree.Modify(newEntry) 307 if err != nil { 308 t.Fatal(err) 309 } 310 311 if tree.hosts[targetKey.String()].entry.AcceptingContracts { 312 t.Fatal("modify did not update host entry") 313 } 314 } 315 316 // TestVariedWeights runs broad statistical tests on selecting hosts with 317 // multiple different weights. 318 func TestVariedWeights(t *testing.T) { 319 if testing.Short() { 320 t.SkipNow() 321 } 322 323 // insert i hosts with the weights 0, 1, ..., i-1. 100e3 selections will be made 324 // per weight added to the tree, the total number of selections necessary 325 // will be tallied up as hosts are created. 326 i := 0 327 328 tree := New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 329 return newCustomScoreBreakdown(types.NewCurrency64(uint64(i))) 330 }, modules.ProductionResolver{}) 331 332 hostCount := 5 333 expectedPerWeight := int(10e3) 334 selections := 0 335 for i = 0; i < hostCount; i++ { 336 entry := makeHostDBEntry() 337 err := tree.Insert(entry) 338 if err != nil { 339 t.Error(err) 340 } 341 selections += i * expectedPerWeight 342 } 343 344 // Perform many random selections, noting which host was selected each 345 // time. 346 selectionMap := make(map[string]int) 347 for i := 0; i < selections; i++ { 348 randEntry := tree.SelectRandom(1, nil, nil) 349 if len(randEntry) == 0 { 350 t.Fatal("no hosts!") 351 } 352 node, exists := tree.hosts[randEntry[0].PublicKey.String()] 353 if !exists { 354 t.Fatal("can't find randomly selected node in tree") 355 } 356 selectionMap[node.entry.weight.String()]++ 357 } 358 359 // Check that each host was selected an expected number of times. An error 360 // will be reported if the host of 0 weight is ever selected. 361 acceptableError := 0.2 362 for weight, timesSelected := range selectionMap { 363 intWeight, err := strconv.Atoi(weight) 364 if err != nil { 365 t.Fatal(err) 366 } 367 368 expectedSelected := float64(intWeight * expectedPerWeight) 369 if float64(expectedSelected)*acceptableError > float64(timesSelected) || float64(expectedSelected)/acceptableError < float64(timesSelected) { 370 t.Error("weighted list not selecting in a uniform distribution based on weight") 371 t.Error(expectedSelected) 372 t.Error(timesSelected) 373 } 374 } 375 } 376 377 // TestRepeatInsert inserts 2 hosts with the same public key. 378 func TestRepeatInsert(t *testing.T) { 379 if testing.Short() { 380 t.SkipNow() 381 } 382 383 tree := New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 384 return newCustomScoreBreakdown(types.NewCurrency64(10)) 385 }, modules.ProductionResolver{}) 386 387 entry1 := makeHostDBEntry() 388 entry2 := entry1 389 390 err := tree.Insert(entry1) 391 if err != nil { 392 t.Fatal(err) 393 } 394 err = tree.Insert(entry2) 395 if !errors.Contains(err, ErrHostExists) { 396 t.Fatal(err) 397 } 398 if len(tree.hosts) != 1 { 399 t.Error("inserting the same entry twice should result in only 1 entry") 400 } 401 } 402 403 // TestNodeAtWeight tests the nodeAtWeight method. 404 func TestNodeAtWeight(t *testing.T) { 405 weight := types.NewCurrency64(10) 406 // create hostTree 407 tree := New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 408 return newCustomScoreBreakdown(weight) 409 }, modules.ProductionResolver{}) 410 411 entry := makeHostDBEntry() 412 err := tree.Insert(entry) 413 if err != nil { 414 t.Fatal(err) 415 } 416 417 h := tree.root.nodeAtWeight(weight) 418 if !h.entry.HostDBEntry.PublicKey.Equals(entry.PublicKey) { 419 t.Errorf("nodeAtWeight returned wrong node: expected %v, got %v", entry, h.entry) 420 } 421 } 422 423 // TestRandomHosts probes the SelectRandom method. 424 func TestRandomHosts(t *testing.T) { 425 // Create the tree. 426 tree := New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 427 return newCustomScoreBreakdown(dbe.StoragePrice) 428 }, modules.ProductionResolver{}) 429 430 // Empty. 431 hosts := tree.SelectRandom(1, nil, nil) 432 if len(hosts) != 0 { 433 t.Errorf("empty hostdb returns %v hosts: %v", len(hosts), hosts) 434 } 435 436 // Insert 3 hosts to be selected. 437 entry1 := makeHostDBEntry() 438 entry1.StoragePrice = types.NewCurrency64(2) 439 entry2 := makeHostDBEntry() 440 entry2.StoragePrice = types.NewCurrency64(3) 441 entry3 := makeHostDBEntry() 442 entry3.StoragePrice = types.NewCurrency64(4) 443 444 if err := tree.Insert(entry1); err != nil { 445 t.Fatal(err) 446 } 447 if err := tree.Insert(entry2); err != nil { 448 t.Fatal(err) 449 } 450 if err := tree.Insert(entry3); err != nil { 451 t.Fatal(err) 452 } 453 454 if len(tree.hosts) != 3 { 455 t.Error("wrong number of hosts") 456 } 457 if tree.root.weight.Cmp(types.NewCurrency64(9)) != 0 { 458 t.Error("unexpected weight at initialization") 459 t.Error(tree.root.weight) 460 } 461 462 // Grab 1 random host. 463 randHosts := tree.SelectRandom(1, nil, nil) 464 if len(randHosts) != 1 { 465 t.Error("didn't get 1 hosts") 466 } 467 468 // Grab 2 random hosts. 469 randHosts = tree.SelectRandom(2, nil, nil) 470 if len(randHosts) != 2 { 471 t.Fatal("didn't get 2 hosts") 472 } 473 if randHosts[0].PublicKey.Equals(randHosts[1].PublicKey) { 474 t.Error("doubled up") 475 } 476 477 // Grab 3 random hosts. 478 randHosts = tree.SelectRandom(3, nil, nil) 479 if len(randHosts) != 3 { 480 t.Fatal("didn't get 3 hosts", len(randHosts)) 481 } 482 483 if randHosts[0].PublicKey.Equals(randHosts[1].PublicKey) || randHosts[0].PublicKey.Equals(randHosts[2].PublicKey) || randHosts[1].PublicKey.Equals(randHosts[2].PublicKey) { 484 t.Error("doubled up") 485 } 486 487 // Grab 4 random hosts. 3 should be returned. 488 randHosts = tree.SelectRandom(4, nil, nil) 489 if len(randHosts) != 3 { 490 t.Fatal("didn't get 3 hosts", len(randHosts)) 491 } 492 493 entry4 := makeHostDBEntry() 494 entry4.StoragePrice = types.NewCurrency64(1) 495 if err := tree.Insert(entry4); err != nil { 496 t.Fatal(err) 497 } 498 499 // Grab 4 random hosts. 3 should be returned because the fourth has a score 500 // of 1. 501 randHosts = tree.SelectRandom(4, nil, nil) 502 if len(randHosts) != 3 { 503 t.Error("didn't get 3 hosts") 504 } 505 506 // Grab 4 random hosts. 3 should be returned. 507 randHosts = tree.SelectRandom(4, nil, nil) 508 if len(randHosts) != 3 { 509 t.Error("didn't get 3 hosts") 510 } 511 512 if randHosts[0].PublicKey.Equals(randHosts[1].PublicKey) || randHosts[0].PublicKey.Equals(randHosts[2].PublicKey) || randHosts[1].PublicKey.Equals(randHosts[2].PublicKey) { 513 t.Error("doubled up") 514 } 515 516 // Ask for 3 hosts that are not in randHosts. No hosts should be 517 // returned. 518 uniqueHosts := tree.SelectRandom(3, []types.SiaPublicKey{ 519 randHosts[0].PublicKey, 520 randHosts[1].PublicKey, 521 randHosts[2].PublicKey, 522 }, nil) 523 if len(uniqueHosts) != 0 { 524 t.Error("didn't get 0 hosts") 525 } 526 527 // Ask for 3 hosts, blacklisting non-existent hosts. 3 should be returned. 528 randHosts = tree.SelectRandom(3, []types.SiaPublicKey{{}, {}, {}}, nil) 529 if len(randHosts) != 3 { 530 t.Error("didn't get 3 hosts") 531 } 532 533 if randHosts[0].PublicKey.Equals(randHosts[1].PublicKey) || randHosts[0].PublicKey.Equals(randHosts[2].PublicKey) || randHosts[1].PublicKey.Equals(randHosts[2].PublicKey) { 534 t.Error("doubled up") 535 } 536 537 // Ask for 3 hosts while only whitelisting 2. 538 whitelist := map[string]struct{}{ 539 entry1.PublicKey.String(): {}, 540 entry3.PublicKey.String(): {}, 541 } 542 randHosts = tree.SelectRandomWithWhitelist(3, nil, nil, whitelist) 543 if len(randHosts) != len(whitelist) { 544 t.Errorf("wrong number of hosts returned %v", len(randHosts)) 545 } 546 for _, host := range randHosts { 547 _, ok := whitelist[host.PublicKey.String()] 548 if !ok { 549 t.Error("non whitelisted host returned") 550 } 551 } 552 } 553 554 // testHostTreeFilterResolver is a resolver for the TestTwoAddresses test. 555 type testHostTreeFilterResolver struct{} 556 557 func (testHostTreeFilterResolver) LookupIP(host string) ([]net.IP, error) { 558 switch host { 559 case "host1": 560 return []net.IP{{127, 0, 0, 1}}, nil 561 case "host2": 562 return []net.IP{{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}}, nil 563 case "host3": 564 return []net.IP{{127, 0, 0, 2}, {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2}}, nil 565 default: 566 panic("shouldn't happen") 567 } 568 } 569 570 // TestHostTreeFilter verifies that two hosts with the IP submask won't be 571 // returned by SelectRandom. 572 func TestHostTreeFilter(t *testing.T) { 573 // Insert 3 hosts to be selected. 574 entry1 := makeHostDBEntry() 575 entry1.NetAddress = "host1:1234" 576 entry2 := makeHostDBEntry() 577 entry2.NetAddress = "host2:1234" 578 entry3 := makeHostDBEntry() 579 entry3.NetAddress = "host3:1234" 580 581 // Create the tree. 582 tree := New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 583 // All entries have the same weight. 584 return newCustomScoreBreakdown(types.NewCurrency64(uint64(10))) 585 }, testHostTreeFilterResolver{}) 586 587 // Insert host1 and host2. Both should be returned by SelectRandom. 588 err := tree.Insert(entry1) 589 if err != nil { 590 t.Fatal(err) 591 } 592 err = tree.Insert(entry2) 593 if err != nil { 594 t.Fatal(err) 595 } 596 if len(tree.SelectRandom(2, nil, nil)) != 2 { 597 t.Error("Expected both hosts to be returned") 598 } 599 600 // Get a new empty tree. 601 tree = New(func(dbe skymodules.HostDBEntry) ScoreBreakdown { 602 // All entries have the same weight. 603 return newCustomScoreBreakdown(types.NewCurrency64(uint64(10))) 604 }, testHostTreeFilterResolver{}) 605 606 // Insert host1 and host3. Only a single host should be returned. 607 err = tree.Insert(entry1) 608 if err != nil { 609 t.Fatal(err) 610 } 611 err = tree.Insert(entry3) 612 if err != nil { 613 t.Fatal(err) 614 } 615 if numHosts := len(tree.SelectRandom(2, nil, nil)); numHosts != 1 { 616 t.Error("Expected only one host but was", numHosts) 617 } 618 619 // Add host2 to the tree to have all 3 hosts in it. 620 err = tree.Insert(entry2) 621 if err != nil { 622 t.Fatal(err) 623 } 624 625 // Call SelectRandom again but ignore host 2. This should give us only 1 626 // host. 627 if numHosts := len(tree.SelectRandom(2, nil, []types.SiaPublicKey{entry2.PublicKey})); numHosts != 1 { 628 t.Error("Expected only one host but was", numHosts) 629 } 630 631 // Call SelectRandom again but ignore host 3. This should give us no host. 632 if numHosts := len(tree.SelectRandom(2, nil, []types.SiaPublicKey{entry3.PublicKey})); numHosts != 0 { 633 t.Error("Expected 0 hosts but was", numHosts) 634 } 635 }