gitlab.com/SiaPrime/SiaPrime@v1.4.1/modules/renter/hostdb/scan.go (about) 1 package hostdb 2 3 // scan.go contains the functions which periodically scan the list of all hosts 4 // to see which hosts are online or offline, and to get any updates to the 5 // settings of the hosts. 6 7 import ( 8 "encoding/json" 9 "fmt" 10 "net" 11 "sort" 12 "time" 13 14 "gitlab.com/NebulousLabs/fastrand" 15 16 "gitlab.com/SiaPrime/SiaPrime/build" 17 "gitlab.com/SiaPrime/SiaPrime/crypto" 18 "gitlab.com/SiaPrime/SiaPrime/encoding" 19 "gitlab.com/SiaPrime/SiaPrime/modules" 20 "gitlab.com/SiaPrime/SiaPrime/modules/renter/hostdb/hosttree" 21 "gitlab.com/SiaPrime/SiaPrime/types" 22 ) 23 24 // equalIPNets checks if two slices of IP subnets contain the same subnets. 25 func equalIPNets(ipNetsA, ipNetsB []string) bool { 26 // Check the length first. 27 if len(ipNetsA) != len(ipNetsB) { 28 return false 29 } 30 // Create a map of all the subnets in ipNetsA. 31 mapNetsA := make(map[string]struct{}) 32 for _, subnet := range ipNetsA { 33 mapNetsA[subnet] = struct{}{} 34 } 35 // Make sure that all the subnets from ipNetsB are in the map. 36 for _, subnet := range ipNetsB { 37 if _, exists := mapNetsA[subnet]; !exists { 38 return false 39 } 40 } 41 return true 42 } 43 44 // feeChangeSignificant determines if the difference between two transaction 45 // fees is significant enough to warrant rebuilding the hosttree. 46 func feeChangeSignificant(oldTxnFees, newTxnFees types.Currency) bool { 47 maxChange := oldTxnFees.MulFloat(txnFeesUpdateRatio) 48 return newTxnFees.Cmp(oldTxnFees.Sub(maxChange)) <= 0 || newTxnFees.Cmp(oldTxnFees.Add(maxChange)) >= 0 49 } 50 51 // managedUpdateTxnFees checks if the txnFees have changed significantly since 52 // the last time they were updated and updates them if necessary. 53 func (hdb *HostDB) managedUpdateTxnFees() { 54 // Get the old txnFees from the hostdb. 55 hdb.mu.RLock() 56 allowance := hdb.allowance 57 oldTxnFees := hdb.txnFees 58 hdb.mu.RUnlock() 59 60 // Get the new fees from the tpool. 61 _, newTxnFees := hdb.tpool.FeeEstimation() 62 63 // If the change is not significant we are done. 64 if !feeChangeSignificant(oldTxnFees, newTxnFees) { 65 hdb.log.Debugf("No need to update txnFees oldFees %v newFees %v", 66 oldTxnFees.HumanString(), newTxnFees.HumanString()) 67 return 68 } 69 // Update the txnFees. 70 hdb.mu.Lock() 71 hdb.txnFees = newTxnFees 72 hdb.mu.Unlock() 73 // Recompute the host weight function. 74 hwf := hdb.managedCalculateHostWeightFn(allowance) 75 // Set the weight function. 76 if err := hdb.managedSetWeightFunction(hwf); err != nil { 77 // This shouldn't happen. 78 build.Critical("Failed to set the new weight function", err) 79 } 80 hdb.log.Println("Updated the hostdb txnFees to", newTxnFees.HumanString()) 81 } 82 83 // queueScan will add a host to the queue to be scanned. The host will be added 84 // at a random position which means that the order in which queueScan is called 85 // is not necessarily the order in which the hosts get scanned. That guarantees 86 // a random scan order during the initial scan. 87 func (hdb *HostDB) queueScan(entry modules.HostDBEntry) { 88 // If this entry is already in the scan pool, can return immediately. 89 _, exists := hdb.scanMap[entry.PublicKey.String()] 90 if exists { 91 return 92 } 93 // Add the entry to a random position in the waitlist. 94 hdb.scanMap[entry.PublicKey.String()] = struct{}{} 95 hdb.scanList = append(hdb.scanList, entry) 96 if len(hdb.scanList) > 1 { 97 i := len(hdb.scanList) - 1 98 j := fastrand.Intn(i) 99 hdb.scanList[i], hdb.scanList[j] = hdb.scanList[j], hdb.scanList[i] 100 } 101 // Check if any thread is currently emptying the waitlist. If not, spawn a 102 // thread to empty the waitlist. 103 if hdb.scanWait { 104 // Another thread is emptying the scan list, nothing to worry about. 105 return 106 } 107 108 // Sanity check - the scan map and the scan list should have the same 109 // length. 110 if build.DEBUG && len(hdb.scanMap) > len(hdb.scanList)+maxScanningThreads { 111 hdb.log.Critical("The hostdb scan map has seemingly grown too large:", len(hdb.scanMap), len(hdb.scanList), maxScanningThreads) 112 } 113 114 hdb.scanWait = true 115 go func() { 116 scanPool := make(chan modules.HostDBEntry) 117 defer close(scanPool) 118 119 // Nobody is emptying the scan list, volunteer. 120 if hdb.tg.Add() != nil { 121 // Hostdb is shutting down, don't spin up another thread. It is 122 // okay to leave scanWait set to true as that will not affect 123 // shutdown. 124 return 125 } 126 defer hdb.tg.Done() 127 128 // Block scan when a specific dependency is provided. 129 hdb.deps.Disrupt("BlockScan") 130 131 // Due to the patterns used to spin up scanning threads, it's possible 132 // that we get to this point while all scanning threads are currently 133 // used up, completing jobs that were sent out by the previous pool 134 // managing thread. This thread is at risk of deadlocking if there's 135 // not at least one scanning thread accepting work that it created 136 // itself, so we use a starterThread exception and spin up 137 // one-thread-too-many on the first iteration to ensure that we do not 138 // deadlock. 139 starterThread := false 140 for { 141 // If the scanList is empty, this thread can spin down. 142 hdb.mu.Lock() 143 if len(hdb.scanList) == 0 { 144 // Scan list is empty, can exit. Let the world know that nobody 145 // is emptying the scan list anymore. 146 hdb.scanWait = false 147 hdb.mu.Unlock() 148 return 149 } 150 151 // Get the next host, shrink the scan list. 152 entry := hdb.scanList[0] 153 hdb.scanList = hdb.scanList[1:] 154 delete(hdb.scanMap, entry.PublicKey.String()) 155 scansRemaining := len(hdb.scanList) 156 157 // Grab the most recent entry for this host. 158 recentEntry, exists := hdb.hostTree.Select(entry.PublicKey) 159 if exists { 160 entry = recentEntry 161 } 162 163 // Try to send this entry to an existing idle worker (non-blocking). 164 select { 165 case scanPool <- entry: 166 hdb.log.Debugf("Sending host %v for scan, %v hosts remain", entry.PublicKey.String(), scansRemaining) 167 hdb.mu.Unlock() 168 continue 169 default: 170 } 171 172 // Create new worker thread. 173 if hdb.scanningThreads < maxScanningThreads || !starterThread { 174 starterThread = true 175 hdb.scanningThreads++ 176 if err := hdb.tg.Add(); err != nil { 177 hdb.mu.Unlock() 178 return 179 } 180 go func() { 181 defer hdb.tg.Done() 182 hdb.threadedProbeHosts(scanPool) 183 hdb.mu.Lock() 184 hdb.scanningThreads-- 185 hdb.mu.Unlock() 186 }() 187 } 188 hdb.mu.Unlock() 189 190 // Block while waiting for an opening in the scan pool. 191 hdb.log.Debugf("Sending host %v for scan, %v hosts remain", entry.PublicKey.String(), scansRemaining) 192 select { 193 case scanPool <- entry: 194 // iterate again 195 case <-hdb.tg.StopChan(): 196 // quit 197 return 198 } 199 } 200 }() 201 } 202 203 // updateEntry updates an entry in the hostdb after a scan has taken place. 204 // 205 // CAUTION: This function will automatically add multiple entries to a new host 206 // to give that host some base uptime. This makes this function co-dependent 207 // with the host weight functions. Adjustment of the host weight functions need 208 // to keep this function in mind, and vice-versa. 209 func (hdb *HostDB) updateEntry(entry modules.HostDBEntry, netErr error) { 210 // If the scan failed because we don't have Internet access, toss out this update. 211 if netErr != nil && !hdb.gateway.Online() { 212 return 213 } 214 215 // Grab the host from the host tree, and update it with the new settings. 216 newEntry, exists := hdb.hostTree.Select(entry.PublicKey) 217 if exists { 218 newEntry.HostExternalSettings = entry.HostExternalSettings 219 newEntry.IPNets = entry.IPNets 220 newEntry.LastIPNetChange = entry.LastIPNetChange 221 } else { 222 newEntry = entry 223 } 224 225 // Update the recent interactions with this host. 226 if netErr == nil { 227 newEntry.RecentSuccessfulInteractions++ 228 } else { 229 newEntry.RecentFailedInteractions++ 230 } 231 232 // Add the datapoints for the scan. 233 if len(newEntry.ScanHistory) < 2 { 234 // Add two scans to the scan history. Two are needed because the scans 235 // are forward looking, but we want this first scan to represent as 236 // much as one week of uptime or downtime. 237 earliestStartTime := time.Now().Add(time.Hour * 7 * 24 * -1) // Permit up to a week of starting uptime or downtime. 238 suggestedStartTime := time.Now().Add(time.Minute * 10 * time.Duration(hdb.blockHeight-entry.FirstSeen+1) * -1) // Add one to the FirstSeen in case FirstSeen is this block, guarantees incrementing order. 239 if suggestedStartTime.Before(earliestStartTime) { 240 suggestedStartTime = earliestStartTime 241 } 242 newEntry.ScanHistory = modules.HostDBScans{ 243 {Timestamp: suggestedStartTime, Success: netErr == nil}, 244 {Timestamp: time.Now(), Success: netErr == nil}, 245 } 246 } else { 247 if newEntry.ScanHistory[len(newEntry.ScanHistory)-1].Success && netErr != nil { 248 hdb.log.Debugf("Host %v is being downgraded from an online host to an offline host: %v\n", newEntry.PublicKey.String(), netErr) 249 } 250 251 // Make sure that the current time is after the timestamp of the 252 // previous scan. It may not be if the system clock has changed. This 253 // will prevent the sort-check sanity checks from triggering. 254 newTimestamp := time.Now() 255 prevTimestamp := newEntry.ScanHistory[len(newEntry.ScanHistory)-1].Timestamp 256 if !newTimestamp.After(prevTimestamp) { 257 newTimestamp = prevTimestamp.Add(time.Second) 258 } 259 260 // Before appending, make sure that the scan we just performed is 261 // timestamped after the previous scan performed. It may not be if the 262 // system clock has changed. 263 newEntry.ScanHistory = append(newEntry.ScanHistory, modules.HostDBScan{Timestamp: newTimestamp, Success: netErr == nil}) 264 } 265 266 // Check whether any of the recent scans demonstrate uptime. The pruning and 267 // compression of the history ensure that there are only relatively recent 268 // scans represented. 269 var recentUptime bool 270 for _, scan := range newEntry.ScanHistory { 271 if scan.Success { 272 recentUptime = true 273 } 274 } 275 276 // If the host has been offline for too long, delete the host from the 277 // hostdb. Only delete if there have been enough scans over a long enough 278 // period to be confident that the host really is offline for good. 279 if time.Now().Sub(newEntry.ScanHistory[0].Timestamp) > maxHostDowntime && !recentUptime && len(newEntry.ScanHistory) >= minScans { 280 // Remove from hosttrees 281 err := hdb.remove(newEntry.PublicKey) 282 if err != nil { 283 hdb.log.Println("ERROR: unable to remove host newEntry which has had a ton of downtime:", err) 284 } 285 286 // The function should terminate here as no more interaction is needed 287 // with this host. 288 return 289 } 290 291 // Compress any old scans into the historic values. 292 for len(newEntry.ScanHistory) > minScans && time.Now().Sub(newEntry.ScanHistory[0].Timestamp) > maxHostDowntime { 293 timePassed := newEntry.ScanHistory[1].Timestamp.Sub(newEntry.ScanHistory[0].Timestamp) 294 if newEntry.ScanHistory[0].Success { 295 newEntry.HistoricUptime += timePassed 296 } else { 297 newEntry.HistoricDowntime += timePassed 298 } 299 newEntry.ScanHistory = newEntry.ScanHistory[1:] 300 } 301 302 // Add the updated entry 303 if !exists { 304 // Insert into Hosttrees 305 err := hdb.insert(newEntry) 306 if err != nil { 307 hdb.log.Println("ERROR: unable to insert entry which is was thought to be new:", err) 308 } else { 309 hdb.log.Debugf("Adding host %v to the hostdb. Net error: %v\n", newEntry.PublicKey.String(), netErr) 310 } 311 } else { 312 // Modify hosttrees 313 err := hdb.modify(newEntry) 314 if err != nil { 315 hdb.log.Println("ERROR: unable to modify entry which is thought to exist:", err) 316 } else { 317 hdb.log.Debugf("Adding host %v to the hostdb. Net error: %v\n", newEntry.PublicKey.String(), netErr) 318 } 319 } 320 } 321 322 // managedLookupIPNets returns string representations of the CIDR subnets 323 // used by the host. In case of an error we return nil. We don't really care 324 // about the error because we don't update host entries if we are offline 325 // anyway. So if we fail to resolve a hostname, the problem is not related to 326 // us. 327 func (hdb *HostDB) managedLookupIPNets(address modules.NetAddress) (ipNets []string, err error) { 328 // Lookup the IP addresses of the host. 329 addresses, err := hdb.deps.Resolver().LookupIP(address.Host()) 330 if err != nil { 331 return nil, err 332 } 333 // Get the subnets of the addresses. 334 for _, ip := range addresses { 335 // Set the filterRange according to the type of IP address. 336 var filterRange int 337 if ip.To4() != nil { 338 filterRange = hosttree.IPv4FilterRange 339 } else { 340 filterRange = hosttree.IPv6FilterRange 341 } 342 343 // Get the subnet. 344 _, ipnet, err := net.ParseCIDR(fmt.Sprintf("%s/%d", ip.String(), filterRange)) 345 if err != nil { 346 return nil, err 347 } 348 // Add the subnet to the host. 349 ipNets = append(ipNets, ipnet.String()) 350 } 351 return 352 } 353 354 // managedScanHost will connect to a host and grab the settings, verifying 355 // uptime and updating to the host's preferences. 356 func (hdb *HostDB) managedScanHost(entry modules.HostDBEntry) { 357 // Request settings from the queued host entry. 358 netAddr := entry.NetAddress 359 pubKey := entry.PublicKey 360 hdb.log.Debugf("Scanning host %v at %v", pubKey, netAddr) 361 362 // If we use a custom resolver for testing, we replace the custom domain 363 // with 127.0.0.1. Otherwise the scan will fail. 364 if hdb.deps.Disrupt("customResolver") { 365 port := netAddr.Port() 366 netAddr = modules.NetAddress(fmt.Sprintf("127.0.0.1:%s", port)) 367 } 368 369 // Resolve the host's used subnets and update the timestamp if they 370 // changed. We only update the timestamp if resolving the ipNets was 371 // successful. 372 ipNets, err := hdb.managedLookupIPNets(entry.NetAddress) 373 if err == nil && !equalIPNets(ipNets, entry.IPNets) { 374 entry.IPNets = ipNets 375 entry.LastIPNetChange = time.Now() 376 } 377 if err != nil { 378 hdb.log.Debugln("mangedScanHost: failed to look up IP nets", err) 379 } 380 381 // Update historic interactions of entry if necessary 382 hdb.mu.RLock() 383 updateHostHistoricInteractions(&entry, hdb.blockHeight) 384 hdb.mu.RUnlock() 385 386 var settings modules.HostExternalSettings 387 var latency time.Duration 388 err = func() error { 389 timeout := hostRequestTimeout 390 hdb.mu.RLock() 391 if len(hdb.initialScanLatencies) > minScansForSpeedup { 392 build.Critical("initialScanLatencies should never be greater than minScansForSpeedup") 393 } 394 if !hdb.initialScanComplete && len(hdb.initialScanLatencies) == minScansForSpeedup { 395 // During an initial scan, when we have at least minScansForSpeedup 396 // active scans in initialScanLatencies, we use 397 // 5*median(initialScanLatencies) as the new hostRequestTimeout to 398 // speedup the scanning process. 399 timeout = hdb.initialScanLatencies[len(hdb.initialScanLatencies)/2] 400 timeout *= scanSpeedupMedianMultiplier 401 if hostRequestTimeout < timeout { 402 timeout = hostRequestTimeout 403 } 404 } 405 hdb.mu.RUnlock() 406 407 dialer := &net.Dialer{ 408 Cancel: hdb.tg.StopChan(), 409 Timeout: timeout, 410 } 411 start := time.Now() 412 conn, err := dialer.Dial("tcp", string(netAddr)) 413 latency = time.Since(start) 414 if err != nil { 415 return err 416 } 417 // Create go routine that will close the channel if the hostdb shuts 418 // down or when this method returns as signalled by closing the 419 // connCloseChan channel 420 connCloseChan := make(chan struct{}) 421 go func() { 422 select { 423 case <-hdb.tg.StopChan(): 424 case <-connCloseChan: 425 } 426 conn.Close() 427 }() 428 defer close(connCloseChan) 429 conn.SetDeadline(time.Now().Add(hostScanDeadline)) 430 431 // Assume that the host supports the new protocol, and request its 432 // settings. If we are talking to an old host, they will not recognize 433 // the request and will close the connection. 434 tryNewProtoErr := func() error { 435 s, _, err := modules.NewRenterSession(conn, pubKey) 436 if err != nil { 437 return err 438 } 439 defer s.WriteRequest(modules.RPCLoopExit, nil) // make sure we close cleanly 440 if err := s.WriteRequest(modules.RPCLoopSettings, nil); err != nil { 441 return err 442 } 443 var resp modules.LoopSettingsResponse 444 if err := s.ReadResponse(&resp, maxSettingsLen); err != nil { 445 return err 446 } 447 return json.Unmarshal(resp.Settings, &settings) 448 }() 449 if tryNewProtoErr == nil { 450 return nil 451 } 452 453 // Failed to get settings with the new protocol; fall back to the old 454 // protocol, filling in the missing fields with default values. 455 // 456 // Close current connection 457 conn.Close() 458 459 // Start new connection. We cannot assign this to the first connection 460 // as it creates a Data Race and conflicts with the deferred channel 461 // closing. Additionally, we can't assign the result of Dial to conn, 462 // because if the Dial fails and conn is nil, then the deferred call to 463 // Close will segfault. 464 conn2, err := dialer.Dial("tcp", string(netAddr)) 465 if err != nil { 466 return err 467 } 468 // Create go routine that will close this second channel if the hostdb 469 // shuts down or when this method returns as signalled by closing the 470 // connCloseChan2 channel 471 connCloseChan2 := make(chan struct{}) 472 go func() { 473 select { 474 case <-hdb.tg.StopChan(): 475 case <-connCloseChan2: 476 } 477 conn2.Close() 478 }() 479 defer close(connCloseChan2) 480 conn2.SetDeadline(time.Now().Add(hostScanDeadline)) 481 482 err = encoding.WriteObject(conn2, modules.RPCSettings) 483 if err != nil { 484 return err 485 } 486 var pubkey crypto.PublicKey 487 copy(pubkey[:], pubKey.Key) 488 var oldSettings modules.HostOldExternalSettings 489 err = crypto.ReadSignedObject(conn2, &oldSettings, maxSettingsLen, pubkey) 490 if err != nil { 491 return err 492 } 493 settings = modules.HostExternalSettings{ 494 AcceptingContracts: oldSettings.AcceptingContracts, 495 MaxDownloadBatchSize: oldSettings.MaxDownloadBatchSize, 496 MaxDuration: oldSettings.MaxDuration, 497 MaxReviseBatchSize: oldSettings.MaxReviseBatchSize, 498 NetAddress: oldSettings.NetAddress, 499 RemainingStorage: oldSettings.RemainingStorage, 500 SectorSize: oldSettings.SectorSize, 501 TotalStorage: oldSettings.TotalStorage, 502 UnlockHash: oldSettings.UnlockHash, 503 WindowSize: oldSettings.WindowSize, 504 Collateral: oldSettings.Collateral, 505 MaxCollateral: oldSettings.MaxCollateral, 506 ContractPrice: oldSettings.ContractPrice, 507 DownloadBandwidthPrice: oldSettings.DownloadBandwidthPrice, 508 StoragePrice: oldSettings.StoragePrice, 509 UploadBandwidthPrice: oldSettings.UploadBandwidthPrice, 510 RevisionNumber: oldSettings.RevisionNumber, 511 Version: oldSettings.Version, 512 // New fields are set to zero. 513 BaseRPCPrice: types.ZeroCurrency, 514 SectorAccessPrice: types.ZeroCurrency, 515 } 516 return nil 517 }() 518 if err != nil { 519 hdb.log.Debugf("Scan of host at %v failed: %v", pubKey, err) 520 } else { 521 hdb.log.Debugf("Scan of host at %v succeeded.", pubKey) 522 entry.HostExternalSettings = settings 523 } 524 success := err == nil 525 526 hdb.mu.Lock() 527 defer hdb.mu.Unlock() 528 // We don't want to override the NetAddress during a scan so we need to 529 // retrieve the most recent NetAddress from the tree first. 530 oldEntry, exists := hdb.hostTree.Select(entry.PublicKey) 531 if exists { 532 entry.NetAddress = oldEntry.NetAddress 533 } 534 // Update the host tree to have a new entry, including the new error. Then 535 // delete the entry from the scan map as the scan has been successful. 536 hdb.updateEntry(entry, err) 537 538 // Add the scan to the initialScanLatencies if it was successful. 539 if success && len(hdb.initialScanLatencies) < minScansForSpeedup { 540 hdb.initialScanLatencies = append(hdb.initialScanLatencies, latency) 541 // If the slice has reached its maximum size we sort it. 542 if len(hdb.initialScanLatencies) == minScansForSpeedup { 543 sort.Slice(hdb.initialScanLatencies, func(i, j int) bool { 544 return hdb.initialScanLatencies[i] < hdb.initialScanLatencies[j] 545 }) 546 } 547 } 548 } 549 550 // waitForScans is a helper function that blocks until the hostDB's scanList is 551 // empty. 552 func (hdb *HostDB) managedWaitForScans() { 553 for { 554 hdb.mu.Lock() 555 length := len(hdb.scanList) 556 hdb.mu.Unlock() 557 if length == 0 { 558 break 559 } 560 select { 561 case <-hdb.tg.StopChan(): 562 case <-time.After(scanCheckInterval): 563 } 564 } 565 } 566 567 // threadedProbeHosts pulls hosts from the thread pool and runs a scan on them. 568 func (hdb *HostDB) threadedProbeHosts(scanPool <-chan modules.HostDBEntry) { 569 for hostEntry := range scanPool { 570 // Block until hostdb has internet connectivity. 571 for { 572 hdb.mu.RLock() 573 online := hdb.gateway.Online() 574 hdb.mu.RUnlock() 575 if online { 576 break 577 } 578 select { 579 case <-time.After(time.Second * 30): 580 continue 581 case <-hdb.tg.StopChan(): 582 return 583 } 584 } 585 586 // There appears to be internet connectivity, continue with the 587 // scan. 588 hdb.managedScanHost(hostEntry) 589 } 590 } 591 592 // threadedScan is an ongoing function which will query the full set of hosts 593 // every few hours to see who is online and available for uploading. 594 func (hdb *HostDB) threadedScan() { 595 err := hdb.tg.Add() 596 if err != nil { 597 return 598 } 599 defer hdb.tg.Done() 600 601 // Wait until the consensus set is synced. Only then we can be sure that 602 // the initial scan covers the whole network. 603 for { 604 if hdb.cs.Synced() { 605 break 606 } 607 select { 608 case <-hdb.tg.StopChan(): 609 return 610 case <-time.After(scanCheckInterval): 611 } 612 } 613 614 // Block scan when a specific dependency is provided. 615 hdb.deps.Disrupt("BlockScan") 616 617 // The initial scan might have been interrupted. Queue one scan for every 618 // announced host that was missed by the initial scan and wait for the 619 // scans to finish before starting the scan loop. 620 allHosts := hdb.hostTree.All() 621 hdb.mu.Lock() 622 for _, host := range allHosts { 623 if len(host.ScanHistory) == 0 && host.HistoricUptime == 0 && host.HistoricDowntime == 0 { 624 hdb.queueScan(host) 625 } 626 } 627 hdb.mu.Unlock() 628 hdb.managedWaitForScans() 629 630 hdb.mu.Lock() 631 // Set the flag to indicate that the initial scan is complete. 632 hdb.initialScanComplete = true 633 // Copy the known contracts to avoid having to lock the hdb later. 634 knownContracts := make(map[string]contractInfo) 635 for k, c := range hdb.knownContracts { 636 knownContracts[k] = c 637 } 638 hdb.mu.Unlock() 639 640 for { 641 // Before we start a new iteration of the scanloop we check if the 642 // txnFees need to be updated. 643 hdb.managedUpdateTxnFees() 644 645 // Set up a scan for the hostCheckupQuantity most valuable hosts in the 646 // hostdb. Hosts that fail their scans will be docked significantly, 647 // pushing them further back in the hierarchy, ensuring that for the 648 // most part only online hosts are getting scanned unless there are 649 // fewer than hostCheckupQuantity of them. 650 651 // Grab a set of hosts to scan, grab hosts that are active, inactive, offline 652 // and known to get high diversity. 653 var onlineHosts, offlineHosts, knownHosts []modules.HostDBEntry 654 allHosts := hdb.hostTree.All() 655 for i := len(allHosts) - 1; i >= 0; i-- { 656 if len(onlineHosts) >= hostCheckupQuantity && 657 len(offlineHosts) >= hostCheckupQuantity && 658 len(knownHosts) == len(knownContracts) { 659 break 660 } 661 662 // Figure out if the host is known, online or offline. 663 host := allHosts[i] 664 online := len(host.ScanHistory) > 0 && host.ScanHistory[len(host.ScanHistory)-1].Success 665 _, known := knownContracts[host.PublicKey.String()] 666 if known { 667 knownHosts = append(knownHosts, host) 668 } else if online && len(onlineHosts) < hostCheckupQuantity { 669 onlineHosts = append(onlineHosts, host) 670 } else if !online && len(offlineHosts) < hostCheckupQuantity { 671 offlineHosts = append(offlineHosts, host) 672 } 673 } 674 675 // Queue the scans for each host. 676 hdb.log.Println("Performing scan on", len(onlineHosts), "online hosts and", len(offlineHosts), "offline hosts and", len(knownHosts), "known hosts.") 677 hdb.mu.Lock() 678 for _, host := range knownHosts { 679 hdb.queueScan(host) 680 } 681 for _, host := range onlineHosts { 682 hdb.queueScan(host) 683 } 684 for _, host := range offlineHosts { 685 hdb.queueScan(host) 686 } 687 hdb.mu.Unlock() 688 689 // Sleep for a random amount of time before doing another round of 690 // scanning. The minimums and maximums keep the scan time reasonable, 691 // while the randomness prevents the scanning from always happening at 692 // the same time of day or week. 693 sleepRange := uint64(maxScanSleep - minScanSleep) 694 sleepTime := minScanSleep + time.Duration(fastrand.Uint64n(sleepRange)) 695 696 // Sleep until it's time for the next scan cycle. 697 select { 698 case <-hdb.tg.StopChan(): 699 return 700 case <-time.After(sleepTime): 701 } 702 } 703 }