github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/dom/herd/sub.go (about) 1 package herd 2 3 import ( 4 "errors" 5 "flag" 6 "fmt" 7 "io" 8 "net" 9 "strings" 10 "time" 11 12 "github.com/Cloud-Foundations/Dominator/dom/lib" 13 "github.com/Cloud-Foundations/Dominator/lib/constants" 14 filegenclient "github.com/Cloud-Foundations/Dominator/lib/filegen/client" 15 "github.com/Cloud-Foundations/Dominator/lib/filesystem" 16 "github.com/Cloud-Foundations/Dominator/lib/hash" 17 "github.com/Cloud-Foundations/Dominator/lib/image" 18 "github.com/Cloud-Foundations/Dominator/lib/objectcache" 19 "github.com/Cloud-Foundations/Dominator/lib/resourcepool" 20 "github.com/Cloud-Foundations/Dominator/lib/srpc" 21 subproto "github.com/Cloud-Foundations/Dominator/proto/sub" 22 "github.com/Cloud-Foundations/Dominator/sub/client" 23 ) 24 25 var ( 26 updateConfigurationsForSubs = flag.Bool("updateConfigurationsForSubs", 27 true, "If true, update the configurations for all subs") 28 logUnknownSubConnectErrors = flag.Bool("logUnknownSubConnectErrors", false, 29 "If true, log unknown sub connection errors") 30 showIP = flag.Bool("showIP", false, 31 "If true, prefer to show IP address from MDB if available") 32 useIP = flag.Bool("useIP", true, 33 "If true, prefer to use IP address from MDB if available") 34 35 subPortNumber = fmt.Sprintf(":%d", constants.SubPortNumber) 36 zeroHash hash.Hash 37 ) 38 39 func (sub *Sub) string() string { 40 if *showIP && sub.mdb.IpAddress != "" { 41 return sub.mdb.IpAddress 42 } 43 return sub.mdb.Hostname 44 } 45 46 func (sub *Sub) address() string { 47 if *useIP && sub.mdb.IpAddress != "" { 48 hostInstance := strings.SplitN(sub.mdb.Hostname, "*", 2) 49 if len(hostInstance) > 1 { 50 return sub.mdb.IpAddress + "*" + hostInstance[1] + subPortNumber 51 } 52 return sub.mdb.IpAddress + subPortNumber 53 } 54 return sub.mdb.Hostname + subPortNumber 55 } 56 57 func (sub *Sub) getComputedFiles(im *image.Image) []filegenclient.ComputedFile { 58 if im == nil { 59 return nil 60 } 61 numComputed := im.FileSystem.NumComputedRegularInodes() 62 if numComputed < 1 { 63 return nil 64 } 65 computedFiles := make([]filegenclient.ComputedFile, 0, numComputed) 66 inodeToFilenamesTable := im.FileSystem.InodeToFilenamesTable() 67 for inum, inode := range im.FileSystem.InodeTable { 68 if inode, ok := inode.(*filesystem.ComputedRegularInode); ok { 69 if filenames, ok := inodeToFilenamesTable[inum]; ok { 70 if len(filenames) == 1 { 71 computedFiles = append(computedFiles, 72 filegenclient.ComputedFile{filenames[0], inode.Source}) 73 } 74 } 75 } 76 } 77 return computedFiles 78 } 79 80 func (sub *Sub) tryMakeBusy() bool { 81 sub.busyFlagMutex.Lock() 82 defer sub.busyFlagMutex.Unlock() 83 if sub.busy { 84 return false 85 } 86 sub.busyStartTime = time.Now() 87 sub.busy = true 88 return true 89 } 90 91 func (sub *Sub) makeUnbusy() { 92 sub.busyFlagMutex.Lock() 93 defer sub.busyFlagMutex.Unlock() 94 sub.busyStopTime = time.Now() 95 sub.busy = false 96 } 97 98 func (sub *Sub) connectAndPoll() { 99 sub.loadConfiguration() 100 if sub.processFileUpdates() { 101 sub.generationCount = 0 // Force a full poll. 102 } 103 sub.deletingFlagMutex.Lock() 104 if sub.deleting { 105 sub.deletingFlagMutex.Unlock() 106 return 107 } 108 if sub.clientResource == nil { 109 sub.clientResource = srpc.NewClientResource("tcp", sub.address()) 110 } 111 sub.deletingFlagMutex.Unlock() 112 previousStatus := sub.status 113 timer := time.AfterFunc(time.Second, func() { 114 sub.publishedStatus = sub.status 115 }) 116 defer func() { 117 timer.Stop() 118 sub.publishedStatus = sub.status 119 }() 120 sub.lastConnectionStartTime = time.Now() 121 srpcClient, err := sub.clientResource.GetHTTPWithDialer(sub.cancelChannel, 122 sub.herd.dialer) 123 dialReturnedTime := time.Now() 124 if err != nil { 125 sub.isInsecure = false 126 sub.pollTime = time.Time{} 127 if err == resourcepool.ErrorResourceLimitExceeded { 128 return 129 } 130 if err, ok := err.(*net.OpError); ok { 131 if _, ok := err.Err.(*net.DNSError); ok { 132 sub.status = statusDNSError 133 return 134 } 135 if err.Timeout() { 136 sub.status = statusConnectTimeout 137 return 138 } 139 } 140 if err == srpc.ErrorConnectionRefused { 141 sub.status = statusConnectionRefused 142 return 143 } 144 if err == srpc.ErrorNoRouteToHost { 145 sub.status = statusNoRouteToHost 146 return 147 } 148 if err == srpc.ErrorMissingCertificate { 149 sub.lastReachableTime = dialReturnedTime 150 sub.status = statusMissingCertificate 151 return 152 } 153 if err == srpc.ErrorBadCertificate { 154 sub.lastReachableTime = dialReturnedTime 155 sub.status = statusBadCertificate 156 return 157 } 158 sub.status = statusFailedToConnect 159 if *logUnknownSubConnectErrors { 160 sub.herd.logger.Println(err) 161 } 162 return 163 } 164 defer srpcClient.Put() 165 if srpcClient.IsEncrypted() { 166 sub.isInsecure = false 167 } else { 168 sub.isInsecure = true 169 } 170 sub.lastReachableTime = dialReturnedTime 171 sub.lastConnectionSucceededTime = dialReturnedTime 172 sub.lastConnectDuration = 173 sub.lastConnectionSucceededTime.Sub(sub.lastConnectionStartTime) 174 connectDistribution.Add(sub.lastConnectDuration) 175 waitStartTime := time.Now() 176 sub.herd.cpuSharer.ReleaseCpu() 177 select { 178 case sub.herd.pollSemaphore <- struct{}{}: 179 sub.herd.cpuSharer.GrabCpu() 180 break 181 case <-sub.cancelChannel: 182 sub.herd.cpuSharer.GrabCpu() 183 return 184 } 185 pollWaitTimeDistribution.Add(time.Since(waitStartTime)) 186 sub.status = statusPolling 187 sub.poll(srpcClient, previousStatus) 188 <-sub.herd.pollSemaphore 189 } 190 191 func (sub *Sub) loadConfiguration() { 192 // Get a stable copy of the configuration. 193 newRequiredImageName := sub.mdb.RequiredImage 194 if newRequiredImageName == "" { 195 newRequiredImageName = sub.herd.defaultImageName 196 } 197 if newRequiredImageName != sub.requiredImageName { 198 sub.computedInodes = nil 199 } 200 sub.herd.cpuSharer.ReleaseCpu() 201 defer sub.herd.cpuSharer.GrabCpu() 202 sub.requiredImageName = newRequiredImageName 203 sub.requiredImage = sub.herd.imageManager.GetNoError(sub.requiredImageName) 204 sub.plannedImageName = sub.mdb.PlannedImage 205 sub.plannedImage = sub.herd.imageManager.GetNoError(sub.plannedImageName) 206 } 207 208 func (sub *Sub) processFileUpdates() bool { 209 haveUpdates := false 210 for { 211 image := sub.requiredImage 212 if image != nil && sub.computedInodes == nil { 213 sub.computedInodes = make(map[string]*filesystem.RegularInode) 214 sub.deletingFlagMutex.Lock() 215 if sub.deleting { 216 sub.deletingFlagMutex.Unlock() 217 return false 218 } 219 computedFiles := sub.getComputedFiles(image) 220 sub.herd.cpuSharer.ReleaseCpu() 221 sub.herd.computedFilesManager.Update( 222 filegenclient.Machine{sub.mdb, computedFiles}) 223 sub.herd.cpuSharer.GrabCpu() 224 sub.deletingFlagMutex.Unlock() 225 } 226 select { 227 case fileInfos := <-sub.fileUpdateChannel: 228 if image == nil { 229 continue 230 } 231 filenameToInodeTable := image.FileSystem.FilenameToInodeTable() 232 for _, fileInfo := range fileInfos { 233 if fileInfo.Hash == zeroHash { 234 continue // No object. 235 } 236 inum, ok := filenameToInodeTable[fileInfo.Pathname] 237 if !ok { 238 continue 239 } 240 genericInode, ok := image.FileSystem.InodeTable[inum] 241 if !ok { 242 continue 243 } 244 cInode, ok := genericInode.(*filesystem.ComputedRegularInode) 245 if !ok { 246 continue 247 } 248 rInode := &filesystem.RegularInode{ 249 Mode: cInode.Mode, 250 Uid: cInode.Uid, 251 Gid: cInode.Gid, 252 MtimeSeconds: -1, // The time is set during the compute. 253 Size: fileInfo.Length, 254 Hash: fileInfo.Hash, 255 } 256 sub.computedInodes[fileInfo.Pathname] = rInode 257 haveUpdates = true 258 } 259 default: 260 return haveUpdates 261 } 262 } 263 } 264 265 func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) { 266 // If the planned image has just become available, force a full poll. 267 if previousStatus == statusSynced && 268 !sub.havePlannedImage && 269 sub.plannedImage != nil { 270 sub.havePlannedImage = true 271 sub.generationCount = 0 // Force a full poll. 272 } 273 // If the computed files have changed since the last sync, force a full poll 274 if previousStatus == statusSynced && 275 sub.computedFilesChangeTime.After(sub.lastSyncTime) { 276 sub.generationCount = 0 // Force a full poll. 277 } 278 // If the last update was disabled and updates are enabled now, force a full 279 // poll. 280 if previousStatus == statusUpdatesDisabled && 281 sub.herd.updatesDisabledReason == "" && !sub.mdb.DisableUpdates { 282 sub.generationCount = 0 // Force a full poll. 283 } 284 // If the last update was disabled due to a safety check and there is a 285 // pending SafetyClear, force a full poll to re-compute the update. 286 if previousStatus == statusUnsafeUpdate && sub.pendingSafetyClear { 287 sub.generationCount = 0 // Force a full poll. 288 } 289 var request subproto.PollRequest 290 request.HaveGeneration = sub.generationCount 291 var reply subproto.PollResponse 292 haveImage := false 293 if sub.requiredImage == nil { 294 request.ShortPollOnly = true 295 // Ensure a full poll when the image becomes available later. This will 296 // cover the special case when an image expiration is extended, which 297 // leads to the sub showing "image not ready" until the next generation 298 // increment. 299 sub.generationCount = 0 300 } else { 301 haveImage = true 302 } 303 logger := sub.herd.logger 304 sub.lastPollStartTime = time.Now() 305 if err := client.CallPoll(srpcClient, request, &reply); err != nil { 306 srpcClient.Close() 307 if err == io.EOF { 308 return 309 } 310 sub.pollTime = time.Time{} 311 if err == srpc.ErrorAccessToMethodDenied { 312 sub.status = statusPollDenied 313 } else { 314 sub.status = statusFailedToPoll 315 } 316 logger.Printf("Error calling %s.Poll(): %s\n", sub, err) 317 return 318 } 319 sub.lastPollSucceededTime = time.Now() 320 sub.lastSuccessfulImageName = reply.LastSuccessfulImageName 321 if reply.GenerationCount == 0 { 322 sub.reclaim() 323 sub.generationCount = 0 324 } 325 sub.lastScanDuration = reply.DurationOfLastScan 326 if fs := reply.FileSystem; fs == nil { 327 sub.lastPollWasFull = false 328 sub.lastShortPollDuration = 329 sub.lastPollSucceededTime.Sub(sub.lastPollStartTime) 330 shortPollDistribution.Add(sub.lastShortPollDuration) 331 if !sub.startTime.Equal(reply.StartTime) { 332 sub.generationCount = 0 // Sub has restarted: force a full poll. 333 } 334 if sub.freeSpaceThreshold != nil && reply.FreeSpace != nil { 335 if *reply.FreeSpace > *sub.freeSpaceThreshold { 336 sub.generationCount = 0 // Force a full poll for next time. 337 } 338 } 339 } else { 340 sub.lastPollWasFull = true 341 sub.freeSpaceThreshold = nil 342 if err := fs.RebuildInodePointers(); err != nil { 343 sub.status = statusFailedToPoll 344 logger.Printf("Error building pointers for: %s %s\n", sub, err) 345 return 346 } 347 fs.BuildEntryMap() 348 sub.fileSystem = fs 349 sub.objectCache = reply.ObjectCache 350 sub.generationCount = reply.GenerationCount 351 sub.lastFullPollDuration = 352 sub.lastPollSucceededTime.Sub(sub.lastPollStartTime) 353 fullPollDistribution.Add(sub.lastFullPollDuration) 354 } 355 sub.startTime = reply.StartTime 356 sub.pollTime = reply.PollTime 357 sub.updateConfiguration(srpcClient, reply) 358 if reply.FetchInProgress { 359 sub.status = statusFetching 360 return 361 } 362 if reply.UpdateInProgress { 363 sub.status = statusUpdating 364 return 365 } 366 if reply.GenerationCount < 1 { 367 sub.status = statusSubNotReady 368 return 369 } 370 if previousStatus == statusFetching && reply.LastFetchError != "" { 371 logger.Printf("Fetch failure for: %s: %s\n", sub, reply.LastFetchError) 372 sub.status = statusFailedToFetch 373 if sub.fileSystem == nil { 374 sub.generationCount = 0 // Force a full poll next cycle. 375 return 376 } 377 } 378 if previousStatus == statusUpdating { 379 // Transition from updating to update ended (may be partial/failed). 380 if reply.LastUpdateError != "" { 381 logger.Printf("Update failure for: %s: %s\n", 382 sub, reply.LastUpdateError) 383 sub.status = statusFailedToUpdate 384 } else { 385 sub.status = statusWaitingForNextFullPoll 386 } 387 sub.scanCountAtLastUpdateEnd = reply.ScanCount 388 sub.reclaim() 389 return 390 } 391 if sub.checkCancel() { 392 // Configuration change pending: skip further processing. Do not reclaim 393 // file-system and objectcache data: it will speed up the next Poll. 394 return 395 } 396 if !haveImage { 397 if sub.requiredImageName == "" { 398 sub.status = statusImageUndefined 399 } else { 400 sub.status = statusImageNotReady 401 } 402 return 403 } 404 if previousStatus == statusFailedToUpdate || 405 previousStatus == statusWaitingForNextFullPoll { 406 if sub.scanCountAtLastUpdateEnd == reply.ScanCount { 407 // Need to wait until sub has performed a new scan. 408 if sub.fileSystem != nil { 409 sub.reclaim() 410 } 411 sub.status = previousStatus 412 return 413 } 414 if sub.fileSystem == nil { 415 // Force a full poll next cycle so that we can see the state of the 416 // sub. 417 sub.generationCount = 0 418 sub.status = previousStatus 419 return 420 } 421 } 422 if sub.fileSystem == nil { 423 sub.status = previousStatus 424 return 425 } 426 if idle, status := sub.fetchMissingObjects(srpcClient, sub.requiredImage, 427 reply.FreeSpace, true); !idle { 428 sub.status = status 429 sub.reclaim() 430 return 431 } 432 sub.status = statusComputingUpdate 433 if idle, status := sub.sendUpdate(srpcClient); !idle { 434 sub.status = status 435 sub.reclaim() 436 return 437 } 438 if idle, status := sub.fetchMissingObjects(srpcClient, sub.plannedImage, 439 reply.FreeSpace, false); !idle { 440 if status != statusImageNotReady && status != statusNotEnoughFreeSpace { 441 sub.status = status 442 sub.reclaim() 443 return 444 } 445 } 446 if previousStatus == statusWaitingForNextFullPoll && 447 !sub.lastUpdateTime.IsZero() { 448 sub.lastSyncTime = time.Now() 449 } 450 sub.status = statusSynced 451 sub.cleanup(srpcClient) 452 sub.reclaim() 453 } 454 455 func (sub *Sub) reclaim() { 456 sub.fileSystem = nil // Mark memory for reclaim. 457 sub.objectCache = nil // Mark memory for reclaim. 458 } 459 460 func (sub *Sub) updateConfiguration(srpcClient *srpc.Client, 461 pollReply subproto.PollResponse) { 462 if !*updateConfigurationsForSubs { 463 return 464 } 465 if pollReply.ScanCount < 1 { 466 return 467 } 468 sub.herd.RLockWithTimeout(time.Minute) 469 newConf := sub.herd.configurationForSubs 470 sub.herd.RUnlock() 471 if newConf.CpuPercent < 1 { 472 newConf.CpuPercent = pollReply.CurrentConfiguration.CpuPercent 473 } 474 if newConf.NetworkSpeedPercent < 1 { 475 newConf.NetworkSpeedPercent = 476 pollReply.CurrentConfiguration.NetworkSpeedPercent 477 } 478 if newConf.ScanSpeedPercent < 1 { 479 newConf.ScanSpeedPercent = 480 pollReply.CurrentConfiguration.ScanSpeedPercent 481 } 482 if compareConfigs(pollReply.CurrentConfiguration, newConf) { 483 return 484 } 485 if err := client.SetConfiguration(srpcClient, newConf); err != nil { 486 srpcClient.Close() 487 logger := sub.herd.logger 488 logger.Printf("Error setting configuration for sub: %s: %s\n", 489 sub, err) 490 return 491 } 492 } 493 494 func compareConfigs(oldConf, newConf subproto.Configuration) bool { 495 if newConf.CpuPercent != oldConf.CpuPercent { 496 return false 497 } 498 if newConf.NetworkSpeedPercent != oldConf.NetworkSpeedPercent { 499 return false 500 } 501 if newConf.ScanSpeedPercent != oldConf.ScanSpeedPercent { 502 return false 503 } 504 if len(newConf.ScanExclusionList) != len(oldConf.ScanExclusionList) { 505 return false 506 } 507 for index, newString := range newConf.ScanExclusionList { 508 if newString != oldConf.ScanExclusionList[index] { 509 return false 510 } 511 } 512 return true 513 } 514 515 // Returns true if all required objects are available. 516 func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, image *image.Image, 517 freeSpace *uint64, pushComputedFiles bool) ( 518 bool, subStatus) { 519 if image == nil { 520 return false, statusImageNotReady 521 } 522 logger := sub.herd.logger 523 subObj := lib.Sub{ 524 Hostname: sub.mdb.Hostname, 525 Client: srpcClient, 526 FileSystem: sub.fileSystem, 527 ComputedInodes: sub.computedInodes, 528 ObjectCache: sub.objectCache, 529 ObjectGetter: sub.herd.objectServer} 530 objectsToFetch, objectsToPush := lib.BuildMissingLists(subObj, image, 531 pushComputedFiles, false, logger) 532 if objectsToPush == nil { 533 return false, statusMissingComputedFile 534 } 535 var returnAvailable bool = true 536 var returnStatus subStatus = statusSynced 537 if len(objectsToFetch) > 0 { 538 if !sub.checkForEnoughSpace(freeSpace, objectsToFetch) { 539 return false, statusNotEnoughFreeSpace 540 } 541 logger.Printf("Calling %s:Subd.Fetch() for: %d objects\n", 542 sub, len(objectsToFetch)) 543 err := client.Fetch(srpcClient, sub.herd.imageManager.String(), 544 objectcache.ObjectMapToCache(objectsToFetch)) 545 if err != nil { 546 srpcClient.Close() 547 logger.Printf("Error calling %s:Subd.Fetch(): %s\n", sub, err) 548 if err == srpc.ErrorAccessToMethodDenied { 549 return false, statusFetchDenied 550 } 551 return false, statusFailedToFetch 552 } 553 returnAvailable = false 554 returnStatus = statusFetching 555 } 556 if len(objectsToPush) > 0 { 557 sub.herd.cpuSharer.GrabSemaphore(sub.herd.pushSemaphore) 558 defer func() { <-sub.herd.pushSemaphore }() 559 sub.status = statusPushing 560 err := lib.PushObjects(subObj, objectsToPush, logger) 561 if err != nil { 562 if err == srpc.ErrorAccessToMethodDenied { 563 return false, statusPushDenied 564 } 565 if err == lib.ErrorFailedToGetObject { 566 return false, statusFailedToGetObject 567 } 568 return false, statusFailedToPush 569 } 570 if returnAvailable { 571 // Update local copy of objectcache, since there will not be 572 // another Poll() before the update computation. 573 for hashVal := range objectsToPush { 574 sub.objectCache = append(sub.objectCache, hashVal) 575 } 576 } 577 } 578 return returnAvailable, returnStatus 579 } 580 581 // Returns true if no update needs to be performed. 582 func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) { 583 logger := sub.herd.logger 584 var request subproto.UpdateRequest 585 var reply subproto.UpdateResponse 586 if idle, missing := sub.buildUpdateRequest(&request); missing { 587 return false, statusMissingComputedFile 588 } else if idle { 589 return true, statusSynced 590 } 591 if sub.mdb.DisableUpdates || sub.herd.updatesDisabledReason != "" { 592 return false, statusUpdatesDisabled 593 } 594 if !sub.pendingSafetyClear { 595 // Perform a cheap safety check: if over half the inodes will be deleted 596 // then mark the update as unsafe. 597 if sub.checkForUnsafeChange(request) { 598 return false, statusUnsafeUpdate 599 } 600 } 601 sub.status = statusSendingUpdate 602 sub.lastUpdateTime = time.Now() 603 logger.Printf("Calling %s:Subd.Update() for image: %s\n", 604 sub, sub.requiredImageName) 605 if err := client.CallUpdate(srpcClient, request, &reply); err != nil { 606 srpcClient.Close() 607 logger.Printf("Error calling %s:Subd.Update(): %s\n", sub, err) 608 if err == srpc.ErrorAccessToMethodDenied { 609 return false, statusUpdateDenied 610 } 611 return false, statusFailedToUpdate 612 } 613 sub.pendingSafetyClear = false 614 return false, statusUpdating 615 } 616 617 // Returns true if the change is unsafe (very large number of deletions). 618 func (sub *Sub) checkForUnsafeChange(request subproto.UpdateRequest) bool { 619 if sub.requiredImage.Filter == nil { 620 return false // Sparse image: no deletions. 621 } 622 if len(sub.requiredImage.FileSystem.InodeTable) < 623 len(sub.fileSystem.InodeTable)>>1 { 624 return true 625 } 626 if len(request.PathsToDelete) > len(sub.fileSystem.InodeTable)>>1 { 627 return true 628 } 629 return false 630 } 631 632 func (sub *Sub) cleanup(srpcClient *srpc.Client) { 633 logger := sub.herd.logger 634 unusedObjects := make(map[hash.Hash]bool) 635 for _, hash := range sub.objectCache { 636 unusedObjects[hash] = false // Potential cleanup candidate. 637 } 638 for _, inode := range sub.fileSystem.InodeTable { 639 if inode, ok := inode.(*filesystem.RegularInode); ok { 640 if inode.Size > 0 { 641 if _, ok := unusedObjects[inode.Hash]; ok { 642 unusedObjects[inode.Hash] = true // Must clean this one up. 643 } 644 } 645 } 646 } 647 image := sub.plannedImage 648 if image != nil { 649 for _, inode := range image.FileSystem.InodeTable { 650 if inode, ok := inode.(*filesystem.RegularInode); ok { 651 if inode.Size > 0 { 652 if clean, ok := unusedObjects[inode.Hash]; !clean && ok { 653 delete(unusedObjects, inode.Hash) 654 } 655 } 656 } 657 } 658 } 659 if len(unusedObjects) < 1 { 660 return 661 } 662 hashes := make([]hash.Hash, 0, len(unusedObjects)) 663 for hash := range unusedObjects { 664 hashes = append(hashes, hash) 665 } 666 if err := client.Cleanup(srpcClient, hashes); err != nil { 667 srpcClient.Close() 668 logger.Printf("Error calling %s:Subd.Cleanup(): %s\n", sub, err) 669 } 670 } 671 672 func (sub *Sub) checkForEnoughSpace(freeSpace *uint64, 673 objects map[hash.Hash]uint64) bool { 674 if freeSpace == nil { 675 sub.freeSpaceThreshold = nil 676 return true // Don't know, assume OK. 677 } 678 var totalUsage uint64 679 for _, size := range objects { 680 usage := (size >> 12) << 12 681 if usage < size { 682 usage += 1 << 12 683 } 684 totalUsage += usage 685 } 686 if *freeSpace > totalUsage { 687 sub.freeSpaceThreshold = nil 688 return true 689 } 690 sub.freeSpaceThreshold = &totalUsage 691 return false 692 } 693 694 func (sub *Sub) clearSafetyShutoff() error { 695 if sub.status != statusUnsafeUpdate { 696 return errors.New("no pending unsafe update") 697 } 698 sub.pendingSafetyClear = true 699 return nil 700 } 701 702 func (sub *Sub) checkCancel() bool { 703 select { 704 case <-sub.cancelChannel: 705 return true 706 default: 707 return false 708 } 709 } 710 711 func (sub *Sub) sendCancel() { 712 select { 713 case sub.cancelChannel <- struct{}{}: 714 default: 715 } 716 }