github.com/swiftstack/ProxyFS@v0.0.0-20210203235616-4017c267d62f/inode/cron.go (about)

     1  // Copyright (c) 2015-2021, NVIDIA CORPORATION.
     2  // SPDX-License-Identifier: Apache-2.0
     3  
     4  package inode
     5  
     6  import (
     7  	"fmt"
     8  	"strconv"
     9  	"strings"
    10  	"sync"
    11  	"time"
    12  
    13  	"github.com/swiftstack/ProxyFS/conf"
    14  	"github.com/swiftstack/ProxyFS/headhunter"
    15  	"github.com/swiftstack/ProxyFS/logger"
    16  )
    17  
    18  type snapShotScheduleStruct struct {
    19  	name                string
    20  	policy              *snapShotPolicyStruct
    21  	minuteSpecified     bool
    22  	minute              int // 0-59
    23  	hourSpecified       bool
    24  	hour                int // 0-23
    25  	dayOfMonthSpecified bool
    26  	dayOfMonth          int // 1-31
    27  	monthSpecified      bool
    28  	month               time.Month // 1-12
    29  	dayOfWeekSpecified  bool
    30  	dayOfWeek           time.Weekday // 0-6 (0 == Sunday)
    31  	keep                uint64
    32  	count               uint64 // computed by scanning each time daemon() awakes
    33  }
    34  
    35  type snapShotPolicyStruct struct {
    36  	name          string
    37  	volume        *volumeStruct
    38  	schedule      []*snapShotScheduleStruct
    39  	location      *time.Location
    40  	stopChan      chan struct{}
    41  	doneWaitGroup sync.WaitGroup
    42  }
    43  
    44  func (vS *volumeStruct) loadSnapShotPolicy(confMap conf.ConfMap) (err error) {
    45  	var (
    46  		cronTabStringSlice          []string
    47  		dayOfMonthAsU64             uint64
    48  		dayOfWeekAsU64              uint64
    49  		hourAsU64                   uint64
    50  		minuteAsU64                 uint64
    51  		monthAsU64                  uint64
    52  		snapShotPolicy              *snapShotPolicyStruct
    53  		snapShotPolicyName          string
    54  		snapShotPolicySectionName   string
    55  		snapShotSchedule            *snapShotScheduleStruct
    56  		snapShotScheduleList        []string
    57  		snapShotScheduleName        string
    58  		snapShotScheduleSectionName string
    59  		timeZone                    string
    60  		volumeSectionName           string
    61  	)
    62  
    63  	// Default to no snapShotPolicy found
    64  	vS.snapShotPolicy = nil
    65  
    66  	volumeSectionName = "Volume:" + vS.volumeName
    67  
    68  	snapShotPolicyName, err = confMap.FetchOptionValueString(volumeSectionName, "SnapShotPolicy")
    69  	if nil != err {
    70  		// Default to setting snapShotPolicy to nil and returning success
    71  		err = nil
    72  		return
    73  	}
    74  
    75  	snapShotPolicy = &snapShotPolicyStruct{name: snapShotPolicyName, volume: vS}
    76  
    77  	snapShotPolicySectionName = "SnapShotPolicy:" + snapShotPolicyName
    78  
    79  	snapShotScheduleList, err = confMap.FetchOptionValueStringSlice(snapShotPolicySectionName, "ScheduleList")
    80  	if nil != err {
    81  		return
    82  	}
    83  	if 0 == len(snapShotScheduleList) {
    84  		// If ScheduleList is empty, set snapShotPolicy to nil and return success
    85  		err = nil
    86  		return
    87  	}
    88  	snapShotPolicy.schedule = make([]*snapShotScheduleStruct, 0, len(snapShotScheduleList))
    89  	for _, snapShotScheduleName = range snapShotScheduleList {
    90  		snapShotScheduleSectionName = "SnapShotSchedule:" + snapShotScheduleName
    91  
    92  		snapShotSchedule = &snapShotScheduleStruct{name: snapShotScheduleName, policy: snapShotPolicy}
    93  
    94  		cronTabStringSlice, err = confMap.FetchOptionValueStringSlice(snapShotScheduleSectionName, "CronTab")
    95  		if nil != err {
    96  			return
    97  		}
    98  		if 5 != len(cronTabStringSlice) {
    99  			err = fmt.Errorf("%v.CronTab must be a 5 element crontab time specification", snapShotScheduleSectionName)
   100  			return
   101  		}
   102  
   103  		if "*" == cronTabStringSlice[0] {
   104  			snapShotSchedule.minuteSpecified = false
   105  		} else {
   106  			snapShotSchedule.minuteSpecified = true
   107  
   108  			minuteAsU64, err = strconv.ParseUint(cronTabStringSlice[0], 10, 8)
   109  			if nil != err {
   110  				return
   111  			}
   112  			if 59 < minuteAsU64 {
   113  				err = fmt.Errorf("%v.CronTab[0] must be valid minute (0-59)", snapShotScheduleSectionName)
   114  				return
   115  			}
   116  
   117  			snapShotSchedule.minute = int(minuteAsU64)
   118  		}
   119  
   120  		if "*" == cronTabStringSlice[1] {
   121  			snapShotSchedule.hourSpecified = false
   122  		} else {
   123  			snapShotSchedule.hourSpecified = true
   124  
   125  			hourAsU64, err = strconv.ParseUint(cronTabStringSlice[1], 10, 8)
   126  			if nil != err {
   127  				return
   128  			}
   129  			if 23 < hourAsU64 {
   130  				err = fmt.Errorf("%v.CronTab[1] must be valid hour (0-23)", snapShotScheduleSectionName)
   131  				return
   132  			}
   133  
   134  			snapShotSchedule.hour = int(hourAsU64)
   135  		}
   136  
   137  		if "*" == cronTabStringSlice[2] {
   138  			snapShotSchedule.dayOfMonthSpecified = false
   139  		} else {
   140  			snapShotSchedule.dayOfMonthSpecified = true
   141  
   142  			dayOfMonthAsU64, err = strconv.ParseUint(cronTabStringSlice[2], 10, 8)
   143  			if nil != err {
   144  				return
   145  			}
   146  			if (0 == dayOfMonthAsU64) || (31 < dayOfMonthAsU64) {
   147  				err = fmt.Errorf("%v.CronTab[2] must be valid dayOfMonth (1-31)", snapShotScheduleSectionName)
   148  				return
   149  			}
   150  
   151  			snapShotSchedule.dayOfMonth = int(dayOfMonthAsU64)
   152  		}
   153  
   154  		if "*" == cronTabStringSlice[3] {
   155  			snapShotSchedule.monthSpecified = false
   156  		} else {
   157  			snapShotSchedule.monthSpecified = true
   158  
   159  			monthAsU64, err = strconv.ParseUint(cronTabStringSlice[3], 10, 8)
   160  			if nil != err {
   161  				return
   162  			}
   163  			if (0 == monthAsU64) || (12 < monthAsU64) {
   164  				err = fmt.Errorf("%v.CronTab[3] must be valid month (1-12)", snapShotScheduleSectionName)
   165  				return
   166  			}
   167  
   168  			snapShotSchedule.month = time.Month(monthAsU64)
   169  		}
   170  
   171  		if "*" == cronTabStringSlice[4] {
   172  			snapShotSchedule.dayOfWeekSpecified = false
   173  		} else {
   174  			snapShotSchedule.dayOfWeekSpecified = true
   175  
   176  			dayOfWeekAsU64, err = strconv.ParseUint(cronTabStringSlice[4], 10, 8)
   177  			if nil != err {
   178  				return
   179  			}
   180  			if 6 < dayOfWeekAsU64 {
   181  				err = fmt.Errorf("%v.CronTab[4] must be valid dayOfWeek (0-6)", snapShotScheduleSectionName)
   182  				return
   183  			}
   184  
   185  			snapShotSchedule.dayOfWeek = time.Weekday(dayOfWeekAsU64)
   186  		}
   187  
   188  		snapShotSchedule.keep, err = confMap.FetchOptionValueUint64(snapShotScheduleSectionName, "Keep")
   189  		if nil != err {
   190  			return
   191  		}
   192  
   193  		if snapShotSchedule.dayOfWeekSpecified && (snapShotSchedule.dayOfMonthSpecified || snapShotSchedule.monthSpecified) {
   194  			err = fmt.Errorf("%v.CronTab must not specify DayOfWeek if DayOfMonth and/or Month are specified", snapShotScheduleSectionName)
   195  			return
   196  		}
   197  
   198  		snapShotPolicy.schedule = append(snapShotPolicy.schedule, snapShotSchedule)
   199  	}
   200  
   201  	timeZone, err = confMap.FetchOptionValueString(snapShotPolicySectionName, "TimeZone")
   202  
   203  	if nil == err {
   204  		snapShotPolicy.location, err = time.LoadLocation(timeZone)
   205  		if nil != err {
   206  			return
   207  		}
   208  	} else { // nil != err
   209  		// If not present, default to UTC
   210  		snapShotPolicy.location = time.UTC
   211  	}
   212  
   213  	// If we reach here, we've successfully loaded the snapShotPolicy
   214  
   215  	vS.snapShotPolicy = snapShotPolicy
   216  	err = nil
   217  	return
   218  }
   219  
   220  func (snapShotPolicy *snapShotPolicyStruct) up() {
   221  	snapShotPolicy.stopChan = make(chan struct{}, 1)
   222  	snapShotPolicy.doneWaitGroup.Add(1)
   223  	go snapShotPolicy.daemon()
   224  }
   225  
   226  func (snapShotPolicy *snapShotPolicyStruct) down() {
   227  	snapShotPolicy.stopChan <- struct{}{}
   228  	snapShotPolicy.doneWaitGroup.Wait()
   229  }
   230  
   231  func (snapShotPolicy *snapShotPolicyStruct) daemon() {
   232  	var (
   233  		err                error
   234  		nextDuration       time.Duration
   235  		nextTime           time.Time
   236  		nextTimePreviously time.Time
   237  		snapShotName       string
   238  		timeNow            time.Time
   239  	)
   240  
   241  	nextTimePreviously = time.Date(2000, time.January, 1, 0, 0, 0, 0, snapShotPolicy.location)
   242  
   243  	for {
   244  		timeNow = time.Now().In(snapShotPolicy.location)
   245  		nextTime = snapShotPolicy.next(timeNow)
   246  		for {
   247  			if !nextTime.Equal(nextTimePreviously) {
   248  				break
   249  			}
   250  			// We took the last snapshot so quickly, next() returned the same nextTime
   251  			time.Sleep(time.Second)
   252  			timeNow = time.Now().In(snapShotPolicy.location)
   253  			nextTime = snapShotPolicy.next(timeNow)
   254  		}
   255  		nextDuration = nextTime.Sub(timeNow)
   256  		select {
   257  		case _ = <-snapShotPolicy.stopChan:
   258  			snapShotPolicy.doneWaitGroup.Done()
   259  			return
   260  		case <-time.After(nextDuration):
   261  			for time.Now().In(snapShotPolicy.location).Before(nextTime) {
   262  				// If time.After() returned a bit too soon, loop until it is our time
   263  				time.Sleep(100 * time.Millisecond)
   264  			}
   265  			snapShotName = strings.Replace(nextTime.Format(time.RFC3339), ":", ".", -1)
   266  			_, err = snapShotPolicy.volume.SnapShotCreate(snapShotName)
   267  			if nil != err {
   268  				logger.WarnWithError(err)
   269  			}
   270  			snapShotPolicy.prune()
   271  		}
   272  		nextTimePreviously = nextTime
   273  	}
   274  }
   275  
   276  func (snapShotPolicy *snapShotPolicyStruct) prune() {
   277  	var (
   278  		err               error
   279  		keep              bool
   280  		matches           bool
   281  		matchesAtLeastOne bool
   282  		snapShot          headhunter.SnapShotStruct
   283  		snapShotList      []headhunter.SnapShotStruct
   284  		snapShotSchedule  *snapShotScheduleStruct
   285  		snapShotTime      time.Time
   286  	)
   287  
   288  	// First, zero each snapShotSchedule.count
   289  
   290  	for _, snapShotSchedule = range snapShotPolicy.schedule {
   291  		snapShotSchedule.count = 0
   292  	}
   293  
   294  	// Now fetch the reverse time-ordered list of snapshots
   295  
   296  	snapShotList = snapShotPolicy.volume.headhunterVolumeHandle.SnapShotListByTime(true)
   297  
   298  	// Now walk snapShotList looking for snapshots to prune
   299  
   300  	for _, snapShot = range snapShotList {
   301  		snapShotTime, err = time.Parse(time.RFC3339, strings.Replace(snapShot.Name, ".", ":", -1))
   302  		if nil != err {
   303  			// SnapShot was not formatted to match a potential SnapShotPolicy/Schedule...skip it
   304  			continue
   305  		}
   306  
   307  		// Compare against each snapShotSchedule
   308  
   309  		keep = false
   310  		matchesAtLeastOne = false
   311  
   312  		for _, snapShotSchedule = range snapShotPolicy.schedule {
   313  			matches = snapShotSchedule.compare(snapShotTime)
   314  			if matches {
   315  				matchesAtLeastOne = true
   316  				snapShotSchedule.count++
   317  				if snapShotSchedule.count <= snapShotSchedule.keep {
   318  					keep = true
   319  				}
   320  			}
   321  		}
   322  
   323  		if matchesAtLeastOne && !keep {
   324  			// Although this snapshot "matchesAtLeastOne",
   325  			//   no snapShotSchedule said "keep" it
   326  
   327  			err = snapShotPolicy.volume.SnapShotDelete(snapShot.ID)
   328  			if nil != err {
   329  				logger.WarnWithError(err)
   330  			}
   331  		}
   332  	}
   333  }
   334  
   335  // thisTime is presumably the snapShotSchedule.policy.location-local parsed snapShotStruct.name
   336  func (snapShotSchedule *snapShotScheduleStruct) compare(thisTime time.Time) (matches bool) {
   337  	var (
   338  		dayOfMonth    int
   339  		dayOfWeek     time.Weekday
   340  		hour          int
   341  		minute        int
   342  		month         time.Month
   343  		truncatedTime time.Time
   344  		year          int
   345  	)
   346  
   347  	hour, minute, _ = thisTime.Clock()
   348  	year, month, dayOfMonth = thisTime.Date()
   349  	dayOfWeek = thisTime.Weekday()
   350  
   351  	truncatedTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location)
   352  	if !truncatedTime.Equal(thisTime) {
   353  		matches = false
   354  		return
   355  	}
   356  
   357  	if snapShotSchedule.minuteSpecified {
   358  		if snapShotSchedule.minute != minute {
   359  			matches = false
   360  			return
   361  		}
   362  	}
   363  
   364  	if snapShotSchedule.hourSpecified {
   365  		if snapShotSchedule.hour != hour {
   366  			matches = false
   367  			return
   368  		}
   369  	}
   370  
   371  	if snapShotSchedule.dayOfMonthSpecified {
   372  		if snapShotSchedule.dayOfMonth != dayOfMonth {
   373  			matches = false
   374  			return
   375  		}
   376  	}
   377  
   378  	if snapShotSchedule.monthSpecified {
   379  		if snapShotSchedule.month != month {
   380  			matches = false
   381  			return
   382  		}
   383  	}
   384  
   385  	if snapShotSchedule.dayOfWeekSpecified {
   386  		if snapShotSchedule.dayOfWeek != dayOfWeek {
   387  			matches = false
   388  			return
   389  		}
   390  	}
   391  
   392  	// If we make it this far, thisTime matches snapShotSchedule
   393  
   394  	matches = true
   395  	return
   396  }
   397  
   398  // Since time.Truncate() only truncates with respect to UTC, it is unsafe
   399  
   400  func truncateToStartOfMinute(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) {
   401  	var (
   402  		day   int
   403  		hour  int
   404  		min   int
   405  		month time.Month
   406  		year  int
   407  	)
   408  
   409  	hour, min, _ = untruncatedTime.Clock()
   410  	year, month, day = untruncatedTime.Date()
   411  
   412  	truncatedTime = time.Date(year, month, day, hour, min, 0, 0, loc)
   413  
   414  	return
   415  }
   416  
   417  func truncateToStartOfHour(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) {
   418  	var (
   419  		day   int
   420  		hour  int
   421  		month time.Month
   422  		year  int
   423  	)
   424  
   425  	hour, _, _ = untruncatedTime.Clock()
   426  	year, month, day = untruncatedTime.Date()
   427  
   428  	truncatedTime = time.Date(year, month, day, hour, 0, 0, 0, loc)
   429  
   430  	return
   431  }
   432  
   433  func truncateToStartOfDay(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) {
   434  	var (
   435  		day   int
   436  		month time.Month
   437  		year  int
   438  	)
   439  
   440  	year, month, day = untruncatedTime.Date()
   441  
   442  	truncatedTime = time.Date(year, month, day, 0, 0, 0, 0, loc)
   443  
   444  	return
   445  }
   446  
   447  func truncateToStartOfMonth(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) {
   448  	var (
   449  		month time.Month
   450  		year  int
   451  	)
   452  
   453  	year, month, _ = untruncatedTime.Date()
   454  
   455  	truncatedTime = time.Date(year, month, 1, 0, 0, 0, 0, loc)
   456  
   457  	return
   458  }
   459  
   460  // timeNow is presumably time.Now() localized to snapShotSchedule.policy.location...
   461  //   ...but provided here so that each invocation of the per snapShotSchedule
   462  //      within a snapShotPolicy can use the same value
   463  func (snapShotSchedule *snapShotScheduleStruct) next(timeNow time.Time) (nextTime time.Time) {
   464  	var (
   465  		dayOfMonth      int
   466  		dayOfWeek       time.Weekday
   467  		hour            int
   468  		minute          int
   469  		month           time.Month
   470  		numDaysToAdd    int
   471  		numHoursToAdd   int
   472  		numMinutesToAdd int
   473  		year            int
   474  	)
   475  
   476  	// Ensure nextTime is at least at the start of the next minute
   477  	nextTime = truncateToStartOfMinute(timeNow, snapShotSchedule.policy.location).Add(time.Minute)
   478  
   479  	if snapShotSchedule.minuteSpecified {
   480  		minute = nextTime.Minute()
   481  		if snapShotSchedule.minute == minute {
   482  			// We don't need to advance nextTime
   483  		} else {
   484  			// No need to (again) truncate nextTime back to the start of the minute
   485  			// Now advance nextTime to align with minute
   486  			if snapShotSchedule.minute > minute {
   487  				numMinutesToAdd = snapShotSchedule.minute - minute
   488  			} else { // snapShotSchedule.minute < minute
   489  				numMinutesToAdd = snapShotSchedule.minute + 60 - minute
   490  			}
   491  			nextTime = nextTime.Add(time.Duration(numMinutesToAdd) * time.Minute)
   492  		}
   493  	}
   494  
   495  	if snapShotSchedule.hourSpecified {
   496  		hour = nextTime.Hour()
   497  		if snapShotSchedule.hour == hour {
   498  			// We don't need to advance nextTime
   499  		} else {
   500  			// First truncate nextTime back to the start of the hour
   501  			nextTime = truncateToStartOfHour(nextTime, snapShotSchedule.policy.location)
   502  			// Restore minuteSpecified if necessary
   503  			if snapShotSchedule.minuteSpecified {
   504  				nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute)
   505  			}
   506  			// Now advance nextTime to align with hour
   507  			if snapShotSchedule.hour > hour {
   508  				numHoursToAdd = snapShotSchedule.hour - hour
   509  			} else { // snapShotSchedule.hour < hour
   510  				numHoursToAdd = snapShotSchedule.hour + 24 - hour
   511  			}
   512  			nextTime = nextTime.Add(time.Duration(numHoursToAdd) * time.Hour)
   513  		}
   514  	}
   515  
   516  	if snapShotSchedule.dayOfMonthSpecified {
   517  		dayOfMonth = nextTime.Day()
   518  		if snapShotSchedule.dayOfMonth == dayOfMonth {
   519  			// We don't need to advance nextTime
   520  		} else {
   521  			// First truncate nextTime back to the start of the day
   522  			nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location)
   523  			// Restore minuteSpecified and/or hourSpecified if necessary
   524  			if snapShotSchedule.minuteSpecified {
   525  				nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute)
   526  			}
   527  			if snapShotSchedule.hourSpecified {
   528  				nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour)
   529  			}
   530  			// Now advance nextTime to align with dayOfMonth
   531  			// Note: This unfortunately iterative approach avoids complicated
   532  			//       adjustments for the non-fixed number of days in a month
   533  			for {
   534  				nextTime = nextTime.Add(24 * time.Hour)
   535  				dayOfMonth = nextTime.Day()
   536  				if snapShotSchedule.dayOfMonth == dayOfMonth {
   537  					break
   538  				}
   539  			}
   540  		}
   541  	}
   542  
   543  	if snapShotSchedule.monthSpecified {
   544  		month = nextTime.Month()
   545  		if snapShotSchedule.month == month {
   546  			// We don't need to advance nextTime
   547  		} else {
   548  			// First truncate nextTime back to the start of the month
   549  			nextTime = truncateToStartOfMonth(nextTime, snapShotSchedule.policy.location)
   550  			// Restore minuteSpecified, hourSpecified, and/or dayOfMonthSpecified if necessary
   551  			if snapShotSchedule.minuteSpecified {
   552  				nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute)
   553  			}
   554  			if snapShotSchedule.hourSpecified {
   555  				nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour)
   556  			}
   557  			if snapShotSchedule.dayOfMonthSpecified {
   558  				nextTime = nextTime.Add(time.Duration((snapShotSchedule.dayOfMonth-1)*24) * time.Hour)
   559  			}
   560  			// Now advance nextTime to align with month
   561  			// Note: This unfortunately iterative approach avoids complicated
   562  			//       adjustments for the non-fixed number of days in a month
   563  			hour, minute, _ = nextTime.Clock()
   564  			year, month, dayOfMonth = nextTime.Date()
   565  			if !snapShotSchedule.dayOfMonthSpecified {
   566  				dayOfMonth = 1
   567  			}
   568  			for {
   569  				if time.December == month {
   570  					month = time.January
   571  					year++
   572  				} else {
   573  					month++
   574  				}
   575  				nextTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location)
   576  				year, month, dayOfMonth = nextTime.Date()
   577  				if snapShotSchedule.dayOfMonthSpecified {
   578  					if (snapShotSchedule.month == month) && (snapShotSchedule.dayOfMonth == dayOfMonth) {
   579  						break
   580  					} else {
   581  						dayOfMonth = snapShotSchedule.dayOfMonth
   582  					}
   583  				} else {
   584  					if (snapShotSchedule.month == month) && (1 == dayOfMonth) {
   585  						break
   586  					} else {
   587  						dayOfMonth = 1
   588  					}
   589  				}
   590  			}
   591  		}
   592  	}
   593  
   594  	if snapShotSchedule.dayOfWeekSpecified {
   595  		dayOfWeek = nextTime.Weekday()
   596  		if time.Weekday(snapShotSchedule.dayOfWeek) == dayOfWeek {
   597  			// We don't need to advance nextTime
   598  		} else {
   599  			// First truncate nextTime back to the start of the day
   600  			nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location)
   601  			// Restore minuteSpecified and/or hourSpecified if necessary
   602  			if snapShotSchedule.minuteSpecified {
   603  				nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute)
   604  			}
   605  			if snapShotSchedule.hourSpecified {
   606  				nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour)
   607  			}
   608  			// Now advance nextTime to align with dayOfWeek
   609  			if time.Weekday(snapShotSchedule.dayOfWeek) > dayOfWeek {
   610  				numDaysToAdd = int(snapShotSchedule.dayOfWeek) - int(dayOfWeek)
   611  			} else { // time.Weekday(snapShotSchedule.dayOfWeek) < dayOfWeek
   612  				numDaysToAdd = int(snapShotSchedule.dayOfWeek) + 7 - int(dayOfWeek)
   613  			}
   614  			nextTime = nextTime.Add(time.Duration(24*numDaysToAdd) * time.Hour)
   615  		}
   616  	}
   617  
   618  	return
   619  }
   620  
   621  // timeNow is presumably time.Now() localized to snapShotPolicy.location...
   622  //   ...but provided here primarily to enable easy testing
   623  func (snapShotPolicy *snapShotPolicyStruct) next(timeNow time.Time) (nextTime time.Time) {
   624  	var (
   625  		nextTimeForSnapShotSchedule time.Time
   626  		nextTimeHasBeenSet          bool
   627  		snapShotSchedule            *snapShotScheduleStruct
   628  	)
   629  
   630  	nextTimeHasBeenSet = false
   631  
   632  	for _, snapShotSchedule = range snapShotPolicy.schedule {
   633  		nextTimeForSnapShotSchedule = snapShotSchedule.next(timeNow)
   634  		if nextTimeHasBeenSet {
   635  			if nextTimeForSnapShotSchedule.Before(nextTime) {
   636  				nextTime = nextTimeForSnapShotSchedule
   637  			}
   638  		} else {
   639  			nextTime = nextTimeForSnapShotSchedule
   640  			nextTimeHasBeenSet = true
   641  		}
   642  	}
   643  
   644  	return
   645  }