github.com/swiftstack/ProxyFS@v0.0.0-20210203235616-4017c267d62f/inode/cron.go (about) 1 // Copyright (c) 2015-2021, NVIDIA CORPORATION. 2 // SPDX-License-Identifier: Apache-2.0 3 4 package inode 5 6 import ( 7 "fmt" 8 "strconv" 9 "strings" 10 "sync" 11 "time" 12 13 "github.com/swiftstack/ProxyFS/conf" 14 "github.com/swiftstack/ProxyFS/headhunter" 15 "github.com/swiftstack/ProxyFS/logger" 16 ) 17 18 type snapShotScheduleStruct struct { 19 name string 20 policy *snapShotPolicyStruct 21 minuteSpecified bool 22 minute int // 0-59 23 hourSpecified bool 24 hour int // 0-23 25 dayOfMonthSpecified bool 26 dayOfMonth int // 1-31 27 monthSpecified bool 28 month time.Month // 1-12 29 dayOfWeekSpecified bool 30 dayOfWeek time.Weekday // 0-6 (0 == Sunday) 31 keep uint64 32 count uint64 // computed by scanning each time daemon() awakes 33 } 34 35 type snapShotPolicyStruct struct { 36 name string 37 volume *volumeStruct 38 schedule []*snapShotScheduleStruct 39 location *time.Location 40 stopChan chan struct{} 41 doneWaitGroup sync.WaitGroup 42 } 43 44 func (vS *volumeStruct) loadSnapShotPolicy(confMap conf.ConfMap) (err error) { 45 var ( 46 cronTabStringSlice []string 47 dayOfMonthAsU64 uint64 48 dayOfWeekAsU64 uint64 49 hourAsU64 uint64 50 minuteAsU64 uint64 51 monthAsU64 uint64 52 snapShotPolicy *snapShotPolicyStruct 53 snapShotPolicyName string 54 snapShotPolicySectionName string 55 snapShotSchedule *snapShotScheduleStruct 56 snapShotScheduleList []string 57 snapShotScheduleName string 58 snapShotScheduleSectionName string 59 timeZone string 60 volumeSectionName string 61 ) 62 63 // Default to no snapShotPolicy found 64 vS.snapShotPolicy = nil 65 66 volumeSectionName = "Volume:" + vS.volumeName 67 68 snapShotPolicyName, err = confMap.FetchOptionValueString(volumeSectionName, "SnapShotPolicy") 69 if nil != err { 70 // Default to setting snapShotPolicy to nil and returning success 71 err = nil 72 return 73 } 74 75 snapShotPolicy = &snapShotPolicyStruct{name: snapShotPolicyName, volume: vS} 76 77 snapShotPolicySectionName = "SnapShotPolicy:" + snapShotPolicyName 78 79 snapShotScheduleList, err = confMap.FetchOptionValueStringSlice(snapShotPolicySectionName, "ScheduleList") 80 if nil != err { 81 return 82 } 83 if 0 == len(snapShotScheduleList) { 84 // If ScheduleList is empty, set snapShotPolicy to nil and return success 85 err = nil 86 return 87 } 88 snapShotPolicy.schedule = make([]*snapShotScheduleStruct, 0, len(snapShotScheduleList)) 89 for _, snapShotScheduleName = range snapShotScheduleList { 90 snapShotScheduleSectionName = "SnapShotSchedule:" + snapShotScheduleName 91 92 snapShotSchedule = &snapShotScheduleStruct{name: snapShotScheduleName, policy: snapShotPolicy} 93 94 cronTabStringSlice, err = confMap.FetchOptionValueStringSlice(snapShotScheduleSectionName, "CronTab") 95 if nil != err { 96 return 97 } 98 if 5 != len(cronTabStringSlice) { 99 err = fmt.Errorf("%v.CronTab must be a 5 element crontab time specification", snapShotScheduleSectionName) 100 return 101 } 102 103 if "*" == cronTabStringSlice[0] { 104 snapShotSchedule.minuteSpecified = false 105 } else { 106 snapShotSchedule.minuteSpecified = true 107 108 minuteAsU64, err = strconv.ParseUint(cronTabStringSlice[0], 10, 8) 109 if nil != err { 110 return 111 } 112 if 59 < minuteAsU64 { 113 err = fmt.Errorf("%v.CronTab[0] must be valid minute (0-59)", snapShotScheduleSectionName) 114 return 115 } 116 117 snapShotSchedule.minute = int(minuteAsU64) 118 } 119 120 if "*" == cronTabStringSlice[1] { 121 snapShotSchedule.hourSpecified = false 122 } else { 123 snapShotSchedule.hourSpecified = true 124 125 hourAsU64, err = strconv.ParseUint(cronTabStringSlice[1], 10, 8) 126 if nil != err { 127 return 128 } 129 if 23 < hourAsU64 { 130 err = fmt.Errorf("%v.CronTab[1] must be valid hour (0-23)", snapShotScheduleSectionName) 131 return 132 } 133 134 snapShotSchedule.hour = int(hourAsU64) 135 } 136 137 if "*" == cronTabStringSlice[2] { 138 snapShotSchedule.dayOfMonthSpecified = false 139 } else { 140 snapShotSchedule.dayOfMonthSpecified = true 141 142 dayOfMonthAsU64, err = strconv.ParseUint(cronTabStringSlice[2], 10, 8) 143 if nil != err { 144 return 145 } 146 if (0 == dayOfMonthAsU64) || (31 < dayOfMonthAsU64) { 147 err = fmt.Errorf("%v.CronTab[2] must be valid dayOfMonth (1-31)", snapShotScheduleSectionName) 148 return 149 } 150 151 snapShotSchedule.dayOfMonth = int(dayOfMonthAsU64) 152 } 153 154 if "*" == cronTabStringSlice[3] { 155 snapShotSchedule.monthSpecified = false 156 } else { 157 snapShotSchedule.monthSpecified = true 158 159 monthAsU64, err = strconv.ParseUint(cronTabStringSlice[3], 10, 8) 160 if nil != err { 161 return 162 } 163 if (0 == monthAsU64) || (12 < monthAsU64) { 164 err = fmt.Errorf("%v.CronTab[3] must be valid month (1-12)", snapShotScheduleSectionName) 165 return 166 } 167 168 snapShotSchedule.month = time.Month(monthAsU64) 169 } 170 171 if "*" == cronTabStringSlice[4] { 172 snapShotSchedule.dayOfWeekSpecified = false 173 } else { 174 snapShotSchedule.dayOfWeekSpecified = true 175 176 dayOfWeekAsU64, err = strconv.ParseUint(cronTabStringSlice[4], 10, 8) 177 if nil != err { 178 return 179 } 180 if 6 < dayOfWeekAsU64 { 181 err = fmt.Errorf("%v.CronTab[4] must be valid dayOfWeek (0-6)", snapShotScheduleSectionName) 182 return 183 } 184 185 snapShotSchedule.dayOfWeek = time.Weekday(dayOfWeekAsU64) 186 } 187 188 snapShotSchedule.keep, err = confMap.FetchOptionValueUint64(snapShotScheduleSectionName, "Keep") 189 if nil != err { 190 return 191 } 192 193 if snapShotSchedule.dayOfWeekSpecified && (snapShotSchedule.dayOfMonthSpecified || snapShotSchedule.monthSpecified) { 194 err = fmt.Errorf("%v.CronTab must not specify DayOfWeek if DayOfMonth and/or Month are specified", snapShotScheduleSectionName) 195 return 196 } 197 198 snapShotPolicy.schedule = append(snapShotPolicy.schedule, snapShotSchedule) 199 } 200 201 timeZone, err = confMap.FetchOptionValueString(snapShotPolicySectionName, "TimeZone") 202 203 if nil == err { 204 snapShotPolicy.location, err = time.LoadLocation(timeZone) 205 if nil != err { 206 return 207 } 208 } else { // nil != err 209 // If not present, default to UTC 210 snapShotPolicy.location = time.UTC 211 } 212 213 // If we reach here, we've successfully loaded the snapShotPolicy 214 215 vS.snapShotPolicy = snapShotPolicy 216 err = nil 217 return 218 } 219 220 func (snapShotPolicy *snapShotPolicyStruct) up() { 221 snapShotPolicy.stopChan = make(chan struct{}, 1) 222 snapShotPolicy.doneWaitGroup.Add(1) 223 go snapShotPolicy.daemon() 224 } 225 226 func (snapShotPolicy *snapShotPolicyStruct) down() { 227 snapShotPolicy.stopChan <- struct{}{} 228 snapShotPolicy.doneWaitGroup.Wait() 229 } 230 231 func (snapShotPolicy *snapShotPolicyStruct) daemon() { 232 var ( 233 err error 234 nextDuration time.Duration 235 nextTime time.Time 236 nextTimePreviously time.Time 237 snapShotName string 238 timeNow time.Time 239 ) 240 241 nextTimePreviously = time.Date(2000, time.January, 1, 0, 0, 0, 0, snapShotPolicy.location) 242 243 for { 244 timeNow = time.Now().In(snapShotPolicy.location) 245 nextTime = snapShotPolicy.next(timeNow) 246 for { 247 if !nextTime.Equal(nextTimePreviously) { 248 break 249 } 250 // We took the last snapshot so quickly, next() returned the same nextTime 251 time.Sleep(time.Second) 252 timeNow = time.Now().In(snapShotPolicy.location) 253 nextTime = snapShotPolicy.next(timeNow) 254 } 255 nextDuration = nextTime.Sub(timeNow) 256 select { 257 case _ = <-snapShotPolicy.stopChan: 258 snapShotPolicy.doneWaitGroup.Done() 259 return 260 case <-time.After(nextDuration): 261 for time.Now().In(snapShotPolicy.location).Before(nextTime) { 262 // If time.After() returned a bit too soon, loop until it is our time 263 time.Sleep(100 * time.Millisecond) 264 } 265 snapShotName = strings.Replace(nextTime.Format(time.RFC3339), ":", ".", -1) 266 _, err = snapShotPolicy.volume.SnapShotCreate(snapShotName) 267 if nil != err { 268 logger.WarnWithError(err) 269 } 270 snapShotPolicy.prune() 271 } 272 nextTimePreviously = nextTime 273 } 274 } 275 276 func (snapShotPolicy *snapShotPolicyStruct) prune() { 277 var ( 278 err error 279 keep bool 280 matches bool 281 matchesAtLeastOne bool 282 snapShot headhunter.SnapShotStruct 283 snapShotList []headhunter.SnapShotStruct 284 snapShotSchedule *snapShotScheduleStruct 285 snapShotTime time.Time 286 ) 287 288 // First, zero each snapShotSchedule.count 289 290 for _, snapShotSchedule = range snapShotPolicy.schedule { 291 snapShotSchedule.count = 0 292 } 293 294 // Now fetch the reverse time-ordered list of snapshots 295 296 snapShotList = snapShotPolicy.volume.headhunterVolumeHandle.SnapShotListByTime(true) 297 298 // Now walk snapShotList looking for snapshots to prune 299 300 for _, snapShot = range snapShotList { 301 snapShotTime, err = time.Parse(time.RFC3339, strings.Replace(snapShot.Name, ".", ":", -1)) 302 if nil != err { 303 // SnapShot was not formatted to match a potential SnapShotPolicy/Schedule...skip it 304 continue 305 } 306 307 // Compare against each snapShotSchedule 308 309 keep = false 310 matchesAtLeastOne = false 311 312 for _, snapShotSchedule = range snapShotPolicy.schedule { 313 matches = snapShotSchedule.compare(snapShotTime) 314 if matches { 315 matchesAtLeastOne = true 316 snapShotSchedule.count++ 317 if snapShotSchedule.count <= snapShotSchedule.keep { 318 keep = true 319 } 320 } 321 } 322 323 if matchesAtLeastOne && !keep { 324 // Although this snapshot "matchesAtLeastOne", 325 // no snapShotSchedule said "keep" it 326 327 err = snapShotPolicy.volume.SnapShotDelete(snapShot.ID) 328 if nil != err { 329 logger.WarnWithError(err) 330 } 331 } 332 } 333 } 334 335 // thisTime is presumably the snapShotSchedule.policy.location-local parsed snapShotStruct.name 336 func (snapShotSchedule *snapShotScheduleStruct) compare(thisTime time.Time) (matches bool) { 337 var ( 338 dayOfMonth int 339 dayOfWeek time.Weekday 340 hour int 341 minute int 342 month time.Month 343 truncatedTime time.Time 344 year int 345 ) 346 347 hour, minute, _ = thisTime.Clock() 348 year, month, dayOfMonth = thisTime.Date() 349 dayOfWeek = thisTime.Weekday() 350 351 truncatedTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location) 352 if !truncatedTime.Equal(thisTime) { 353 matches = false 354 return 355 } 356 357 if snapShotSchedule.minuteSpecified { 358 if snapShotSchedule.minute != minute { 359 matches = false 360 return 361 } 362 } 363 364 if snapShotSchedule.hourSpecified { 365 if snapShotSchedule.hour != hour { 366 matches = false 367 return 368 } 369 } 370 371 if snapShotSchedule.dayOfMonthSpecified { 372 if snapShotSchedule.dayOfMonth != dayOfMonth { 373 matches = false 374 return 375 } 376 } 377 378 if snapShotSchedule.monthSpecified { 379 if snapShotSchedule.month != month { 380 matches = false 381 return 382 } 383 } 384 385 if snapShotSchedule.dayOfWeekSpecified { 386 if snapShotSchedule.dayOfWeek != dayOfWeek { 387 matches = false 388 return 389 } 390 } 391 392 // If we make it this far, thisTime matches snapShotSchedule 393 394 matches = true 395 return 396 } 397 398 // Since time.Truncate() only truncates with respect to UTC, it is unsafe 399 400 func truncateToStartOfMinute(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 401 var ( 402 day int 403 hour int 404 min int 405 month time.Month 406 year int 407 ) 408 409 hour, min, _ = untruncatedTime.Clock() 410 year, month, day = untruncatedTime.Date() 411 412 truncatedTime = time.Date(year, month, day, hour, min, 0, 0, loc) 413 414 return 415 } 416 417 func truncateToStartOfHour(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 418 var ( 419 day int 420 hour int 421 month time.Month 422 year int 423 ) 424 425 hour, _, _ = untruncatedTime.Clock() 426 year, month, day = untruncatedTime.Date() 427 428 truncatedTime = time.Date(year, month, day, hour, 0, 0, 0, loc) 429 430 return 431 } 432 433 func truncateToStartOfDay(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 434 var ( 435 day int 436 month time.Month 437 year int 438 ) 439 440 year, month, day = untruncatedTime.Date() 441 442 truncatedTime = time.Date(year, month, day, 0, 0, 0, 0, loc) 443 444 return 445 } 446 447 func truncateToStartOfMonth(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 448 var ( 449 month time.Month 450 year int 451 ) 452 453 year, month, _ = untruncatedTime.Date() 454 455 truncatedTime = time.Date(year, month, 1, 0, 0, 0, 0, loc) 456 457 return 458 } 459 460 // timeNow is presumably time.Now() localized to snapShotSchedule.policy.location... 461 // ...but provided here so that each invocation of the per snapShotSchedule 462 // within a snapShotPolicy can use the same value 463 func (snapShotSchedule *snapShotScheduleStruct) next(timeNow time.Time) (nextTime time.Time) { 464 var ( 465 dayOfMonth int 466 dayOfWeek time.Weekday 467 hour int 468 minute int 469 month time.Month 470 numDaysToAdd int 471 numHoursToAdd int 472 numMinutesToAdd int 473 year int 474 ) 475 476 // Ensure nextTime is at least at the start of the next minute 477 nextTime = truncateToStartOfMinute(timeNow, snapShotSchedule.policy.location).Add(time.Minute) 478 479 if snapShotSchedule.minuteSpecified { 480 minute = nextTime.Minute() 481 if snapShotSchedule.minute == minute { 482 // We don't need to advance nextTime 483 } else { 484 // No need to (again) truncate nextTime back to the start of the minute 485 // Now advance nextTime to align with minute 486 if snapShotSchedule.minute > minute { 487 numMinutesToAdd = snapShotSchedule.minute - minute 488 } else { // snapShotSchedule.minute < minute 489 numMinutesToAdd = snapShotSchedule.minute + 60 - minute 490 } 491 nextTime = nextTime.Add(time.Duration(numMinutesToAdd) * time.Minute) 492 } 493 } 494 495 if snapShotSchedule.hourSpecified { 496 hour = nextTime.Hour() 497 if snapShotSchedule.hour == hour { 498 // We don't need to advance nextTime 499 } else { 500 // First truncate nextTime back to the start of the hour 501 nextTime = truncateToStartOfHour(nextTime, snapShotSchedule.policy.location) 502 // Restore minuteSpecified if necessary 503 if snapShotSchedule.minuteSpecified { 504 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 505 } 506 // Now advance nextTime to align with hour 507 if snapShotSchedule.hour > hour { 508 numHoursToAdd = snapShotSchedule.hour - hour 509 } else { // snapShotSchedule.hour < hour 510 numHoursToAdd = snapShotSchedule.hour + 24 - hour 511 } 512 nextTime = nextTime.Add(time.Duration(numHoursToAdd) * time.Hour) 513 } 514 } 515 516 if snapShotSchedule.dayOfMonthSpecified { 517 dayOfMonth = nextTime.Day() 518 if snapShotSchedule.dayOfMonth == dayOfMonth { 519 // We don't need to advance nextTime 520 } else { 521 // First truncate nextTime back to the start of the day 522 nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location) 523 // Restore minuteSpecified and/or hourSpecified if necessary 524 if snapShotSchedule.minuteSpecified { 525 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 526 } 527 if snapShotSchedule.hourSpecified { 528 nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) 529 } 530 // Now advance nextTime to align with dayOfMonth 531 // Note: This unfortunately iterative approach avoids complicated 532 // adjustments for the non-fixed number of days in a month 533 for { 534 nextTime = nextTime.Add(24 * time.Hour) 535 dayOfMonth = nextTime.Day() 536 if snapShotSchedule.dayOfMonth == dayOfMonth { 537 break 538 } 539 } 540 } 541 } 542 543 if snapShotSchedule.monthSpecified { 544 month = nextTime.Month() 545 if snapShotSchedule.month == month { 546 // We don't need to advance nextTime 547 } else { 548 // First truncate nextTime back to the start of the month 549 nextTime = truncateToStartOfMonth(nextTime, snapShotSchedule.policy.location) 550 // Restore minuteSpecified, hourSpecified, and/or dayOfMonthSpecified if necessary 551 if snapShotSchedule.minuteSpecified { 552 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 553 } 554 if snapShotSchedule.hourSpecified { 555 nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) 556 } 557 if snapShotSchedule.dayOfMonthSpecified { 558 nextTime = nextTime.Add(time.Duration((snapShotSchedule.dayOfMonth-1)*24) * time.Hour) 559 } 560 // Now advance nextTime to align with month 561 // Note: This unfortunately iterative approach avoids complicated 562 // adjustments for the non-fixed number of days in a month 563 hour, minute, _ = nextTime.Clock() 564 year, month, dayOfMonth = nextTime.Date() 565 if !snapShotSchedule.dayOfMonthSpecified { 566 dayOfMonth = 1 567 } 568 for { 569 if time.December == month { 570 month = time.January 571 year++ 572 } else { 573 month++ 574 } 575 nextTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location) 576 year, month, dayOfMonth = nextTime.Date() 577 if snapShotSchedule.dayOfMonthSpecified { 578 if (snapShotSchedule.month == month) && (snapShotSchedule.dayOfMonth == dayOfMonth) { 579 break 580 } else { 581 dayOfMonth = snapShotSchedule.dayOfMonth 582 } 583 } else { 584 if (snapShotSchedule.month == month) && (1 == dayOfMonth) { 585 break 586 } else { 587 dayOfMonth = 1 588 } 589 } 590 } 591 } 592 } 593 594 if snapShotSchedule.dayOfWeekSpecified { 595 dayOfWeek = nextTime.Weekday() 596 if time.Weekday(snapShotSchedule.dayOfWeek) == dayOfWeek { 597 // We don't need to advance nextTime 598 } else { 599 // First truncate nextTime back to the start of the day 600 nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location) 601 // Restore minuteSpecified and/or hourSpecified if necessary 602 if snapShotSchedule.minuteSpecified { 603 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 604 } 605 if snapShotSchedule.hourSpecified { 606 nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) 607 } 608 // Now advance nextTime to align with dayOfWeek 609 if time.Weekday(snapShotSchedule.dayOfWeek) > dayOfWeek { 610 numDaysToAdd = int(snapShotSchedule.dayOfWeek) - int(dayOfWeek) 611 } else { // time.Weekday(snapShotSchedule.dayOfWeek) < dayOfWeek 612 numDaysToAdd = int(snapShotSchedule.dayOfWeek) + 7 - int(dayOfWeek) 613 } 614 nextTime = nextTime.Add(time.Duration(24*numDaysToAdd) * time.Hour) 615 } 616 } 617 618 return 619 } 620 621 // timeNow is presumably time.Now() localized to snapShotPolicy.location... 622 // ...but provided here primarily to enable easy testing 623 func (snapShotPolicy *snapShotPolicyStruct) next(timeNow time.Time) (nextTime time.Time) { 624 var ( 625 nextTimeForSnapShotSchedule time.Time 626 nextTimeHasBeenSet bool 627 snapShotSchedule *snapShotScheduleStruct 628 ) 629 630 nextTimeHasBeenSet = false 631 632 for _, snapShotSchedule = range snapShotPolicy.schedule { 633 nextTimeForSnapShotSchedule = snapShotSchedule.next(timeNow) 634 if nextTimeHasBeenSet { 635 if nextTimeForSnapShotSchedule.Before(nextTime) { 636 nextTime = nextTimeForSnapShotSchedule 637 } 638 } else { 639 nextTime = nextTimeForSnapShotSchedule 640 nextTimeHasBeenSet = true 641 } 642 } 643 644 return 645 }