github.com/cloud-foundations/dominator@v0.0.0-20221004181915-6e4fee580046/dom/herd/sub.go (about)

     1  package herd
     2  
     3  import (
     4  	"errors"
     5  	"flag"
     6  	"fmt"
     7  	"io"
     8  	"net"
     9  	"strings"
    10  	"time"
    11  
    12  	"github.com/Cloud-Foundations/Dominator/dom/lib"
    13  	"github.com/Cloud-Foundations/Dominator/lib/constants"
    14  	filegenclient "github.com/Cloud-Foundations/Dominator/lib/filegen/client"
    15  	"github.com/Cloud-Foundations/Dominator/lib/filesystem"
    16  	"github.com/Cloud-Foundations/Dominator/lib/hash"
    17  	"github.com/Cloud-Foundations/Dominator/lib/image"
    18  	"github.com/Cloud-Foundations/Dominator/lib/objectcache"
    19  	"github.com/Cloud-Foundations/Dominator/lib/resourcepool"
    20  	"github.com/Cloud-Foundations/Dominator/lib/srpc"
    21  	subproto "github.com/Cloud-Foundations/Dominator/proto/sub"
    22  	"github.com/Cloud-Foundations/Dominator/sub/client"
    23  )
    24  
    25  var (
    26  	updateConfigurationsForSubs = flag.Bool("updateConfigurationsForSubs",
    27  		true, "If true, update the configurations for all subs")
    28  	logUnknownSubConnectErrors = flag.Bool("logUnknownSubConnectErrors", false,
    29  		"If true, log unknown sub connection errors")
    30  	showIP = flag.Bool("showIP", false,
    31  		"If true, prefer to show IP address from MDB if available")
    32  	useIP = flag.Bool("useIP", true,
    33  		"If true, prefer to use IP address from MDB if available")
    34  
    35  	subPortNumber = fmt.Sprintf(":%d", constants.SubPortNumber)
    36  	zeroHash      hash.Hash
    37  )
    38  
    39  func (sub *Sub) string() string {
    40  	if *showIP && sub.mdb.IpAddress != "" {
    41  		return sub.mdb.IpAddress
    42  	}
    43  	return sub.mdb.Hostname
    44  }
    45  
    46  func (sub *Sub) address() string {
    47  	if *useIP && sub.mdb.IpAddress != "" {
    48  		hostInstance := strings.SplitN(sub.mdb.Hostname, "*", 2)
    49  		if len(hostInstance) > 1 {
    50  			return sub.mdb.IpAddress + "*" + hostInstance[1] + subPortNumber
    51  		}
    52  		return sub.mdb.IpAddress + subPortNumber
    53  	}
    54  	return sub.mdb.Hostname + subPortNumber
    55  }
    56  
    57  func (sub *Sub) getComputedFiles(im *image.Image) []filegenclient.ComputedFile {
    58  	if im == nil {
    59  		return nil
    60  	}
    61  	numComputed := im.FileSystem.NumComputedRegularInodes()
    62  	if numComputed < 1 {
    63  		return nil
    64  	}
    65  	computedFiles := make([]filegenclient.ComputedFile, 0, numComputed)
    66  	inodeToFilenamesTable := im.FileSystem.InodeToFilenamesTable()
    67  	for inum, inode := range im.FileSystem.InodeTable {
    68  		if inode, ok := inode.(*filesystem.ComputedRegularInode); ok {
    69  			if filenames, ok := inodeToFilenamesTable[inum]; ok {
    70  				if len(filenames) == 1 {
    71  					computedFiles = append(computedFiles,
    72  						filegenclient.ComputedFile{filenames[0], inode.Source})
    73  				}
    74  			}
    75  		}
    76  	}
    77  	return computedFiles
    78  }
    79  
    80  func (sub *Sub) tryMakeBusy() bool {
    81  	sub.busyFlagMutex.Lock()
    82  	defer sub.busyFlagMutex.Unlock()
    83  	if sub.busy {
    84  		return false
    85  	}
    86  	sub.busyStartTime = time.Now()
    87  	sub.busy = true
    88  	return true
    89  }
    90  
    91  func (sub *Sub) makeUnbusy() {
    92  	sub.busyFlagMutex.Lock()
    93  	defer sub.busyFlagMutex.Unlock()
    94  	sub.busyStopTime = time.Now()
    95  	sub.busy = false
    96  }
    97  
    98  func (sub *Sub) connectAndPoll() {
    99  	sub.loadConfiguration()
   100  	if sub.processFileUpdates() {
   101  		sub.generationCount = 0 // Force a full poll.
   102  	}
   103  	sub.deletingFlagMutex.Lock()
   104  	if sub.deleting {
   105  		sub.deletingFlagMutex.Unlock()
   106  		return
   107  	}
   108  	if sub.clientResource == nil {
   109  		sub.clientResource = srpc.NewClientResource("tcp", sub.address())
   110  	}
   111  	sub.deletingFlagMutex.Unlock()
   112  	previousStatus := sub.status
   113  	timer := time.AfterFunc(time.Second, func() {
   114  		sub.publishedStatus = sub.status
   115  	})
   116  	defer func() {
   117  		timer.Stop()
   118  		sub.publishedStatus = sub.status
   119  	}()
   120  	sub.lastConnectionStartTime = time.Now()
   121  	srpcClient, err := sub.clientResource.GetHTTPWithDialer(sub.cancelChannel,
   122  		sub.herd.dialer)
   123  	dialReturnedTime := time.Now()
   124  	if err != nil {
   125  		sub.isInsecure = false
   126  		sub.pollTime = time.Time{}
   127  		if err == resourcepool.ErrorResourceLimitExceeded {
   128  			return
   129  		}
   130  		if err, ok := err.(*net.OpError); ok {
   131  			if _, ok := err.Err.(*net.DNSError); ok {
   132  				sub.status = statusDNSError
   133  				return
   134  			}
   135  			if err.Timeout() {
   136  				sub.status = statusConnectTimeout
   137  				return
   138  			}
   139  		}
   140  		if err == srpc.ErrorConnectionRefused {
   141  			sub.status = statusConnectionRefused
   142  			return
   143  		}
   144  		if err == srpc.ErrorNoRouteToHost {
   145  			sub.status = statusNoRouteToHost
   146  			return
   147  		}
   148  		if err == srpc.ErrorMissingCertificate {
   149  			sub.lastReachableTime = dialReturnedTime
   150  			sub.status = statusMissingCertificate
   151  			return
   152  		}
   153  		if err == srpc.ErrorBadCertificate {
   154  			sub.lastReachableTime = dialReturnedTime
   155  			sub.status = statusBadCertificate
   156  			return
   157  		}
   158  		sub.status = statusFailedToConnect
   159  		if *logUnknownSubConnectErrors {
   160  			sub.herd.logger.Println(err)
   161  		}
   162  		return
   163  	}
   164  	defer srpcClient.Put()
   165  	if srpcClient.IsEncrypted() {
   166  		sub.isInsecure = false
   167  	} else {
   168  		sub.isInsecure = true
   169  	}
   170  	sub.lastReachableTime = dialReturnedTime
   171  	sub.lastConnectionSucceededTime = dialReturnedTime
   172  	sub.lastConnectDuration =
   173  		sub.lastConnectionSucceededTime.Sub(sub.lastConnectionStartTime)
   174  	connectDistribution.Add(sub.lastConnectDuration)
   175  	waitStartTime := time.Now()
   176  	sub.herd.cpuSharer.ReleaseCpu()
   177  	select {
   178  	case sub.herd.pollSemaphore <- struct{}{}:
   179  		sub.herd.cpuSharer.GrabCpu()
   180  		break
   181  	case <-sub.cancelChannel:
   182  		sub.herd.cpuSharer.GrabCpu()
   183  		return
   184  	}
   185  	pollWaitTimeDistribution.Add(time.Since(waitStartTime))
   186  	sub.status = statusPolling
   187  	sub.poll(srpcClient, previousStatus)
   188  	<-sub.herd.pollSemaphore
   189  }
   190  
   191  func (sub *Sub) loadConfiguration() {
   192  	// Get a stable copy of the configuration.
   193  	newRequiredImageName := sub.mdb.RequiredImage
   194  	if newRequiredImageName == "" {
   195  		newRequiredImageName = sub.herd.defaultImageName
   196  	}
   197  	if newRequiredImageName != sub.requiredImageName {
   198  		sub.computedInodes = nil
   199  	}
   200  	sub.herd.cpuSharer.ReleaseCpu()
   201  	defer sub.herd.cpuSharer.GrabCpu()
   202  	sub.requiredImageName = newRequiredImageName
   203  	sub.requiredImage = sub.herd.imageManager.GetNoError(sub.requiredImageName)
   204  	sub.plannedImageName = sub.mdb.PlannedImage
   205  	sub.plannedImage = sub.herd.imageManager.GetNoError(sub.plannedImageName)
   206  }
   207  
   208  func (sub *Sub) processFileUpdates() bool {
   209  	haveUpdates := false
   210  	for {
   211  		image := sub.requiredImage
   212  		if image != nil && sub.computedInodes == nil {
   213  			sub.computedInodes = make(map[string]*filesystem.RegularInode)
   214  			sub.deletingFlagMutex.Lock()
   215  			if sub.deleting {
   216  				sub.deletingFlagMutex.Unlock()
   217  				return false
   218  			}
   219  			computedFiles := sub.getComputedFiles(image)
   220  			sub.herd.cpuSharer.ReleaseCpu()
   221  			sub.herd.computedFilesManager.Update(
   222  				filegenclient.Machine{sub.mdb, computedFiles})
   223  			sub.herd.cpuSharer.GrabCpu()
   224  			sub.deletingFlagMutex.Unlock()
   225  		}
   226  		select {
   227  		case fileInfos := <-sub.fileUpdateChannel:
   228  			if image == nil {
   229  				continue
   230  			}
   231  			filenameToInodeTable := image.FileSystem.FilenameToInodeTable()
   232  			for _, fileInfo := range fileInfos {
   233  				if fileInfo.Hash == zeroHash {
   234  					continue // No object.
   235  				}
   236  				inum, ok := filenameToInodeTable[fileInfo.Pathname]
   237  				if !ok {
   238  					continue
   239  				}
   240  				genericInode, ok := image.FileSystem.InodeTable[inum]
   241  				if !ok {
   242  					continue
   243  				}
   244  				cInode, ok := genericInode.(*filesystem.ComputedRegularInode)
   245  				if !ok {
   246  					continue
   247  				}
   248  				rInode := &filesystem.RegularInode{
   249  					Mode:         cInode.Mode,
   250  					Uid:          cInode.Uid,
   251  					Gid:          cInode.Gid,
   252  					MtimeSeconds: -1, // The time is set during the compute.
   253  					Size:         fileInfo.Length,
   254  					Hash:         fileInfo.Hash,
   255  				}
   256  				sub.computedInodes[fileInfo.Pathname] = rInode
   257  				haveUpdates = true
   258  			}
   259  		default:
   260  			return haveUpdates
   261  		}
   262  	}
   263  }
   264  
   265  func (sub *Sub) poll(srpcClient *srpc.Client, previousStatus subStatus) {
   266  	// If the planned image has just become available, force a full poll.
   267  	if previousStatus == statusSynced &&
   268  		!sub.havePlannedImage &&
   269  		sub.plannedImage != nil {
   270  		sub.havePlannedImage = true
   271  		sub.generationCount = 0 // Force a full poll.
   272  	}
   273  	// If the computed files have changed since the last sync, force a full poll
   274  	if previousStatus == statusSynced &&
   275  		sub.computedFilesChangeTime.After(sub.lastSyncTime) {
   276  		sub.generationCount = 0 // Force a full poll.
   277  	}
   278  	// If the last update was disabled and updates are enabled now, force a full
   279  	// poll.
   280  	if previousStatus == statusUpdatesDisabled &&
   281  		sub.herd.updatesDisabledReason == "" && !sub.mdb.DisableUpdates {
   282  		sub.generationCount = 0 // Force a full poll.
   283  	}
   284  	// If the last update was disabled due to a safety check and there is a
   285  	// pending SafetyClear, force a full poll to re-compute the update.
   286  	if previousStatus == statusUnsafeUpdate && sub.pendingSafetyClear {
   287  		sub.generationCount = 0 // Force a full poll.
   288  	}
   289  	var request subproto.PollRequest
   290  	request.HaveGeneration = sub.generationCount
   291  	var reply subproto.PollResponse
   292  	haveImage := false
   293  	if sub.requiredImage == nil {
   294  		request.ShortPollOnly = true
   295  		// Ensure a full poll when the image becomes available later. This will
   296  		// cover the special case when an image expiration is extended, which
   297  		// leads to the sub showing "image not ready" until the next generation
   298  		// increment.
   299  		sub.generationCount = 0
   300  	} else {
   301  		haveImage = true
   302  	}
   303  	logger := sub.herd.logger
   304  	sub.lastPollStartTime = time.Now()
   305  	if err := client.CallPoll(srpcClient, request, &reply); err != nil {
   306  		srpcClient.Close()
   307  		if err == io.EOF {
   308  			return
   309  		}
   310  		sub.pollTime = time.Time{}
   311  		if err == srpc.ErrorAccessToMethodDenied {
   312  			sub.status = statusPollDenied
   313  		} else {
   314  			sub.status = statusFailedToPoll
   315  		}
   316  		logger.Printf("Error calling %s.Poll(): %s\n", sub, err)
   317  		return
   318  	}
   319  	sub.lastPollSucceededTime = time.Now()
   320  	sub.lastSuccessfulImageName = reply.LastSuccessfulImageName
   321  	if reply.GenerationCount == 0 {
   322  		sub.reclaim()
   323  		sub.generationCount = 0
   324  	}
   325  	sub.lastScanDuration = reply.DurationOfLastScan
   326  	if fs := reply.FileSystem; fs == nil {
   327  		sub.lastPollWasFull = false
   328  		sub.lastShortPollDuration =
   329  			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
   330  		shortPollDistribution.Add(sub.lastShortPollDuration)
   331  		if !sub.startTime.Equal(reply.StartTime) {
   332  			sub.generationCount = 0 // Sub has restarted: force a full poll.
   333  		}
   334  		if sub.freeSpaceThreshold != nil && reply.FreeSpace != nil {
   335  			if *reply.FreeSpace > *sub.freeSpaceThreshold {
   336  				sub.generationCount = 0 // Force a full poll for next time.
   337  			}
   338  		}
   339  	} else {
   340  		sub.lastPollWasFull = true
   341  		sub.freeSpaceThreshold = nil
   342  		if err := fs.RebuildInodePointers(); err != nil {
   343  			sub.status = statusFailedToPoll
   344  			logger.Printf("Error building pointers for: %s %s\n", sub, err)
   345  			return
   346  		}
   347  		fs.BuildEntryMap()
   348  		sub.fileSystem = fs
   349  		sub.objectCache = reply.ObjectCache
   350  		sub.generationCount = reply.GenerationCount
   351  		sub.lastFullPollDuration =
   352  			sub.lastPollSucceededTime.Sub(sub.lastPollStartTime)
   353  		fullPollDistribution.Add(sub.lastFullPollDuration)
   354  	}
   355  	sub.startTime = reply.StartTime
   356  	sub.pollTime = reply.PollTime
   357  	sub.updateConfiguration(srpcClient, reply)
   358  	if reply.FetchInProgress {
   359  		sub.status = statusFetching
   360  		return
   361  	}
   362  	if reply.UpdateInProgress {
   363  		sub.status = statusUpdating
   364  		return
   365  	}
   366  	if reply.GenerationCount < 1 {
   367  		sub.status = statusSubNotReady
   368  		return
   369  	}
   370  	if previousStatus == statusFetching && reply.LastFetchError != "" {
   371  		logger.Printf("Fetch failure for: %s: %s\n", sub, reply.LastFetchError)
   372  		sub.status = statusFailedToFetch
   373  		if sub.fileSystem == nil {
   374  			sub.generationCount = 0 // Force a full poll next cycle.
   375  			return
   376  		}
   377  	}
   378  	if previousStatus == statusUpdating {
   379  		// Transition from updating to update ended (may be partial/failed).
   380  		if reply.LastUpdateError != "" {
   381  			logger.Printf("Update failure for: %s: %s\n",
   382  				sub, reply.LastUpdateError)
   383  			sub.status = statusFailedToUpdate
   384  		} else {
   385  			sub.status = statusWaitingForNextFullPoll
   386  		}
   387  		sub.scanCountAtLastUpdateEnd = reply.ScanCount
   388  		sub.reclaim()
   389  		return
   390  	}
   391  	if sub.checkCancel() {
   392  		// Configuration change pending: skip further processing. Do not reclaim
   393  		// file-system and objectcache data: it will speed up the next Poll.
   394  		return
   395  	}
   396  	if !haveImage {
   397  		if sub.requiredImageName == "" {
   398  			sub.status = statusImageUndefined
   399  		} else {
   400  			sub.status = statusImageNotReady
   401  		}
   402  		return
   403  	}
   404  	if previousStatus == statusFailedToUpdate ||
   405  		previousStatus == statusWaitingForNextFullPoll {
   406  		if sub.scanCountAtLastUpdateEnd == reply.ScanCount {
   407  			// Need to wait until sub has performed a new scan.
   408  			if sub.fileSystem != nil {
   409  				sub.reclaim()
   410  			}
   411  			sub.status = previousStatus
   412  			return
   413  		}
   414  		if sub.fileSystem == nil {
   415  			// Force a full poll next cycle so that we can see the state of the
   416  			// sub.
   417  			sub.generationCount = 0
   418  			sub.status = previousStatus
   419  			return
   420  		}
   421  	}
   422  	if sub.fileSystem == nil {
   423  		sub.status = previousStatus
   424  		return
   425  	}
   426  	if idle, status := sub.fetchMissingObjects(srpcClient, sub.requiredImage,
   427  		reply.FreeSpace, true); !idle {
   428  		sub.status = status
   429  		sub.reclaim()
   430  		return
   431  	}
   432  	sub.status = statusComputingUpdate
   433  	if idle, status := sub.sendUpdate(srpcClient); !idle {
   434  		sub.status = status
   435  		sub.reclaim()
   436  		return
   437  	}
   438  	if idle, status := sub.fetchMissingObjects(srpcClient, sub.plannedImage,
   439  		reply.FreeSpace, false); !idle {
   440  		if status != statusImageNotReady && status != statusNotEnoughFreeSpace {
   441  			sub.status = status
   442  			sub.reclaim()
   443  			return
   444  		}
   445  	}
   446  	if previousStatus == statusWaitingForNextFullPoll &&
   447  		!sub.lastUpdateTime.IsZero() {
   448  		sub.lastSyncTime = time.Now()
   449  	}
   450  	sub.status = statusSynced
   451  	sub.cleanup(srpcClient)
   452  	sub.reclaim()
   453  }
   454  
   455  func (sub *Sub) reclaim() {
   456  	sub.fileSystem = nil  // Mark memory for reclaim.
   457  	sub.objectCache = nil // Mark memory for reclaim.
   458  }
   459  
   460  func (sub *Sub) updateConfiguration(srpcClient *srpc.Client,
   461  	pollReply subproto.PollResponse) {
   462  	if !*updateConfigurationsForSubs {
   463  		return
   464  	}
   465  	if pollReply.ScanCount < 1 {
   466  		return
   467  	}
   468  	sub.herd.RLockWithTimeout(time.Minute)
   469  	newConf := sub.herd.configurationForSubs
   470  	sub.herd.RUnlock()
   471  	if newConf.CpuPercent < 1 {
   472  		newConf.CpuPercent = pollReply.CurrentConfiguration.CpuPercent
   473  	}
   474  	if newConf.NetworkSpeedPercent < 1 {
   475  		newConf.NetworkSpeedPercent =
   476  			pollReply.CurrentConfiguration.NetworkSpeedPercent
   477  	}
   478  	if newConf.ScanSpeedPercent < 1 {
   479  		newConf.ScanSpeedPercent =
   480  			pollReply.CurrentConfiguration.ScanSpeedPercent
   481  	}
   482  	if compareConfigs(pollReply.CurrentConfiguration, newConf) {
   483  		return
   484  	}
   485  	if err := client.SetConfiguration(srpcClient, newConf); err != nil {
   486  		srpcClient.Close()
   487  		logger := sub.herd.logger
   488  		logger.Printf("Error setting configuration for sub: %s: %s\n",
   489  			sub, err)
   490  		return
   491  	}
   492  }
   493  
   494  func compareConfigs(oldConf, newConf subproto.Configuration) bool {
   495  	if newConf.CpuPercent != oldConf.CpuPercent {
   496  		return false
   497  	}
   498  	if newConf.NetworkSpeedPercent != oldConf.NetworkSpeedPercent {
   499  		return false
   500  	}
   501  	if newConf.ScanSpeedPercent != oldConf.ScanSpeedPercent {
   502  		return false
   503  	}
   504  	if len(newConf.ScanExclusionList) != len(oldConf.ScanExclusionList) {
   505  		return false
   506  	}
   507  	for index, newString := range newConf.ScanExclusionList {
   508  		if newString != oldConf.ScanExclusionList[index] {
   509  			return false
   510  		}
   511  	}
   512  	return true
   513  }
   514  
   515  // Returns true if all required objects are available.
   516  func (sub *Sub) fetchMissingObjects(srpcClient *srpc.Client, image *image.Image,
   517  	freeSpace *uint64, pushComputedFiles bool) (
   518  	bool, subStatus) {
   519  	if image == nil {
   520  		return false, statusImageNotReady
   521  	}
   522  	logger := sub.herd.logger
   523  	subObj := lib.Sub{
   524  		Hostname:       sub.mdb.Hostname,
   525  		Client:         srpcClient,
   526  		FileSystem:     sub.fileSystem,
   527  		ComputedInodes: sub.computedInodes,
   528  		ObjectCache:    sub.objectCache,
   529  		ObjectGetter:   sub.herd.objectServer}
   530  	objectsToFetch, objectsToPush := lib.BuildMissingLists(subObj, image,
   531  		pushComputedFiles, false, logger)
   532  	if objectsToPush == nil {
   533  		return false, statusMissingComputedFile
   534  	}
   535  	var returnAvailable bool = true
   536  	var returnStatus subStatus = statusSynced
   537  	if len(objectsToFetch) > 0 {
   538  		if !sub.checkForEnoughSpace(freeSpace, objectsToFetch) {
   539  			return false, statusNotEnoughFreeSpace
   540  		}
   541  		logger.Printf("Calling %s:Subd.Fetch() for: %d objects\n",
   542  			sub, len(objectsToFetch))
   543  		err := client.Fetch(srpcClient, sub.herd.imageManager.String(),
   544  			objectcache.ObjectMapToCache(objectsToFetch))
   545  		if err != nil {
   546  			srpcClient.Close()
   547  			logger.Printf("Error calling %s:Subd.Fetch(): %s\n", sub, err)
   548  			if err == srpc.ErrorAccessToMethodDenied {
   549  				return false, statusFetchDenied
   550  			}
   551  			return false, statusFailedToFetch
   552  		}
   553  		returnAvailable = false
   554  		returnStatus = statusFetching
   555  	}
   556  	if len(objectsToPush) > 0 {
   557  		sub.herd.cpuSharer.GrabSemaphore(sub.herd.pushSemaphore)
   558  		defer func() { <-sub.herd.pushSemaphore }()
   559  		sub.status = statusPushing
   560  		err := lib.PushObjects(subObj, objectsToPush, logger)
   561  		if err != nil {
   562  			if err == srpc.ErrorAccessToMethodDenied {
   563  				return false, statusPushDenied
   564  			}
   565  			if err == lib.ErrorFailedToGetObject {
   566  				return false, statusFailedToGetObject
   567  			}
   568  			return false, statusFailedToPush
   569  		}
   570  		if returnAvailable {
   571  			// Update local copy of objectcache, since there will not be
   572  			// another Poll() before the update computation.
   573  			for hashVal := range objectsToPush {
   574  				sub.objectCache = append(sub.objectCache, hashVal)
   575  			}
   576  		}
   577  	}
   578  	return returnAvailable, returnStatus
   579  }
   580  
   581  // Returns true if no update needs to be performed.
   582  func (sub *Sub) sendUpdate(srpcClient *srpc.Client) (bool, subStatus) {
   583  	logger := sub.herd.logger
   584  	var request subproto.UpdateRequest
   585  	var reply subproto.UpdateResponse
   586  	if idle, missing := sub.buildUpdateRequest(&request); missing {
   587  		return false, statusMissingComputedFile
   588  	} else if idle {
   589  		return true, statusSynced
   590  	}
   591  	if sub.mdb.DisableUpdates || sub.herd.updatesDisabledReason != "" {
   592  		return false, statusUpdatesDisabled
   593  	}
   594  	if !sub.pendingSafetyClear {
   595  		// Perform a cheap safety check: if over half the inodes will be deleted
   596  		// then mark the update as unsafe.
   597  		if sub.checkForUnsafeChange(request) {
   598  			return false, statusUnsafeUpdate
   599  		}
   600  	}
   601  	sub.status = statusSendingUpdate
   602  	sub.lastUpdateTime = time.Now()
   603  	logger.Printf("Calling %s:Subd.Update() for image: %s\n",
   604  		sub, sub.requiredImageName)
   605  	if err := client.CallUpdate(srpcClient, request, &reply); err != nil {
   606  		srpcClient.Close()
   607  		logger.Printf("Error calling %s:Subd.Update(): %s\n", sub, err)
   608  		if err == srpc.ErrorAccessToMethodDenied {
   609  			return false, statusUpdateDenied
   610  		}
   611  		return false, statusFailedToUpdate
   612  	}
   613  	sub.pendingSafetyClear = false
   614  	return false, statusUpdating
   615  }
   616  
   617  // Returns true if the change is unsafe (very large number of deletions).
   618  func (sub *Sub) checkForUnsafeChange(request subproto.UpdateRequest) bool {
   619  	if sub.requiredImage.Filter == nil {
   620  		return false // Sparse image: no deletions.
   621  	}
   622  	if len(sub.requiredImage.FileSystem.InodeTable) <
   623  		len(sub.fileSystem.InodeTable)>>1 {
   624  		return true
   625  	}
   626  	if len(request.PathsToDelete) > len(sub.fileSystem.InodeTable)>>1 {
   627  		return true
   628  	}
   629  	return false
   630  }
   631  
   632  func (sub *Sub) cleanup(srpcClient *srpc.Client) {
   633  	logger := sub.herd.logger
   634  	unusedObjects := make(map[hash.Hash]bool)
   635  	for _, hash := range sub.objectCache {
   636  		unusedObjects[hash] = false // Potential cleanup candidate.
   637  	}
   638  	for _, inode := range sub.fileSystem.InodeTable {
   639  		if inode, ok := inode.(*filesystem.RegularInode); ok {
   640  			if inode.Size > 0 {
   641  				if _, ok := unusedObjects[inode.Hash]; ok {
   642  					unusedObjects[inode.Hash] = true // Must clean this one up.
   643  				}
   644  			}
   645  		}
   646  	}
   647  	image := sub.plannedImage
   648  	if image != nil {
   649  		for _, inode := range image.FileSystem.InodeTable {
   650  			if inode, ok := inode.(*filesystem.RegularInode); ok {
   651  				if inode.Size > 0 {
   652  					if clean, ok := unusedObjects[inode.Hash]; !clean && ok {
   653  						delete(unusedObjects, inode.Hash)
   654  					}
   655  				}
   656  			}
   657  		}
   658  	}
   659  	if len(unusedObjects) < 1 {
   660  		return
   661  	}
   662  	hashes := make([]hash.Hash, 0, len(unusedObjects))
   663  	for hash := range unusedObjects {
   664  		hashes = append(hashes, hash)
   665  	}
   666  	if err := client.Cleanup(srpcClient, hashes); err != nil {
   667  		srpcClient.Close()
   668  		logger.Printf("Error calling %s:Subd.Cleanup(): %s\n", sub, err)
   669  	}
   670  }
   671  
   672  func (sub *Sub) checkForEnoughSpace(freeSpace *uint64,
   673  	objects map[hash.Hash]uint64) bool {
   674  	if freeSpace == nil {
   675  		sub.freeSpaceThreshold = nil
   676  		return true // Don't know, assume OK.
   677  	}
   678  	var totalUsage uint64
   679  	for _, size := range objects {
   680  		usage := (size >> 12) << 12
   681  		if usage < size {
   682  			usage += 1 << 12
   683  		}
   684  		totalUsage += usage
   685  	}
   686  	if *freeSpace > totalUsage {
   687  		sub.freeSpaceThreshold = nil
   688  		return true
   689  	}
   690  	sub.freeSpaceThreshold = &totalUsage
   691  	return false
   692  }
   693  
   694  func (sub *Sub) clearSafetyShutoff() error {
   695  	if sub.status != statusUnsafeUpdate {
   696  		return errors.New("no pending unsafe update")
   697  	}
   698  	sub.pendingSafetyClear = true
   699  	return nil
   700  }
   701  
   702  func (sub *Sub) checkCancel() bool {
   703  	select {
   704  	case <-sub.cancelChannel:
   705  		return true
   706  	default:
   707  		return false
   708  	}
   709  }
   710  
   711  func (sub *Sub) sendCancel() {
   712  	select {
   713  	case sub.cancelChannel <- struct{}{}:
   714  	default:
   715  	}
   716  }