github.com/rohankumardubey/proxyfs@v0.0.0-20210108201508-653efa9ab00e/inode/cron.go (about) 1 package inode 2 3 import ( 4 "fmt" 5 "strconv" 6 "strings" 7 "sync" 8 "time" 9 10 "github.com/swiftstack/ProxyFS/conf" 11 "github.com/swiftstack/ProxyFS/headhunter" 12 "github.com/swiftstack/ProxyFS/logger" 13 ) 14 15 type snapShotScheduleStruct struct { 16 name string 17 policy *snapShotPolicyStruct 18 minuteSpecified bool 19 minute int // 0-59 20 hourSpecified bool 21 hour int // 0-23 22 dayOfMonthSpecified bool 23 dayOfMonth int // 1-31 24 monthSpecified bool 25 month time.Month // 1-12 26 dayOfWeekSpecified bool 27 dayOfWeek time.Weekday // 0-6 (0 == Sunday) 28 keep uint64 29 count uint64 // computed by scanning each time daemon() awakes 30 } 31 32 type snapShotPolicyStruct struct { 33 name string 34 volume *volumeStruct 35 schedule []*snapShotScheduleStruct 36 location *time.Location 37 stopChan chan struct{} 38 doneWaitGroup sync.WaitGroup 39 } 40 41 func (vS *volumeStruct) loadSnapShotPolicy(confMap conf.ConfMap) (err error) { 42 var ( 43 cronTabStringSlice []string 44 dayOfMonthAsU64 uint64 45 dayOfWeekAsU64 uint64 46 hourAsU64 uint64 47 minuteAsU64 uint64 48 monthAsU64 uint64 49 snapShotPolicy *snapShotPolicyStruct 50 snapShotPolicyName string 51 snapShotPolicySectionName string 52 snapShotSchedule *snapShotScheduleStruct 53 snapShotScheduleList []string 54 snapShotScheduleName string 55 snapShotScheduleSectionName string 56 timeZone string 57 volumeSectionName string 58 ) 59 60 // Default to no snapShotPolicy found 61 vS.snapShotPolicy = nil 62 63 volumeSectionName = "Volume:" + vS.volumeName 64 65 snapShotPolicyName, err = confMap.FetchOptionValueString(volumeSectionName, "SnapShotPolicy") 66 if nil != err { 67 // Default to setting snapShotPolicy to nil and returning success 68 err = nil 69 return 70 } 71 72 snapShotPolicy = &snapShotPolicyStruct{name: snapShotPolicyName, volume: vS} 73 74 snapShotPolicySectionName = "SnapShotPolicy:" + snapShotPolicyName 75 76 snapShotScheduleList, err = confMap.FetchOptionValueStringSlice(snapShotPolicySectionName, "ScheduleList") 77 if nil != err { 78 return 79 } 80 if 0 == len(snapShotScheduleList) { 81 // If ScheduleList is empty, set snapShotPolicy to nil and return success 82 err = nil 83 return 84 } 85 snapShotPolicy.schedule = make([]*snapShotScheduleStruct, 0, len(snapShotScheduleList)) 86 for _, snapShotScheduleName = range snapShotScheduleList { 87 snapShotScheduleSectionName = "SnapShotSchedule:" + snapShotScheduleName 88 89 snapShotSchedule = &snapShotScheduleStruct{name: snapShotScheduleName, policy: snapShotPolicy} 90 91 cronTabStringSlice, err = confMap.FetchOptionValueStringSlice(snapShotScheduleSectionName, "CronTab") 92 if nil != err { 93 return 94 } 95 if 5 != len(cronTabStringSlice) { 96 err = fmt.Errorf("%v.CronTab must be a 5 element crontab time specification", snapShotScheduleSectionName) 97 return 98 } 99 100 if "*" == cronTabStringSlice[0] { 101 snapShotSchedule.minuteSpecified = false 102 } else { 103 snapShotSchedule.minuteSpecified = true 104 105 minuteAsU64, err = strconv.ParseUint(cronTabStringSlice[0], 10, 8) 106 if nil != err { 107 return 108 } 109 if 59 < minuteAsU64 { 110 err = fmt.Errorf("%v.CronTab[0] must be valid minute (0-59)", snapShotScheduleSectionName) 111 return 112 } 113 114 snapShotSchedule.minute = int(minuteAsU64) 115 } 116 117 if "*" == cronTabStringSlice[1] { 118 snapShotSchedule.hourSpecified = false 119 } else { 120 snapShotSchedule.hourSpecified = true 121 122 hourAsU64, err = strconv.ParseUint(cronTabStringSlice[1], 10, 8) 123 if nil != err { 124 return 125 } 126 if 23 < hourAsU64 { 127 err = fmt.Errorf("%v.CronTab[1] must be valid hour (0-23)", snapShotScheduleSectionName) 128 return 129 } 130 131 snapShotSchedule.hour = int(hourAsU64) 132 } 133 134 if "*" == cronTabStringSlice[2] { 135 snapShotSchedule.dayOfMonthSpecified = false 136 } else { 137 snapShotSchedule.dayOfMonthSpecified = true 138 139 dayOfMonthAsU64, err = strconv.ParseUint(cronTabStringSlice[2], 10, 8) 140 if nil != err { 141 return 142 } 143 if (0 == dayOfMonthAsU64) || (31 < dayOfMonthAsU64) { 144 err = fmt.Errorf("%v.CronTab[2] must be valid dayOfMonth (1-31)", snapShotScheduleSectionName) 145 return 146 } 147 148 snapShotSchedule.dayOfMonth = int(dayOfMonthAsU64) 149 } 150 151 if "*" == cronTabStringSlice[3] { 152 snapShotSchedule.monthSpecified = false 153 } else { 154 snapShotSchedule.monthSpecified = true 155 156 monthAsU64, err = strconv.ParseUint(cronTabStringSlice[3], 10, 8) 157 if nil != err { 158 return 159 } 160 if (0 == monthAsU64) || (12 < monthAsU64) { 161 err = fmt.Errorf("%v.CronTab[3] must be valid month (1-12)", snapShotScheduleSectionName) 162 return 163 } 164 165 snapShotSchedule.month = time.Month(monthAsU64) 166 } 167 168 if "*" == cronTabStringSlice[4] { 169 snapShotSchedule.dayOfWeekSpecified = false 170 } else { 171 snapShotSchedule.dayOfWeekSpecified = true 172 173 dayOfWeekAsU64, err = strconv.ParseUint(cronTabStringSlice[4], 10, 8) 174 if nil != err { 175 return 176 } 177 if 6 < dayOfWeekAsU64 { 178 err = fmt.Errorf("%v.CronTab[4] must be valid dayOfWeek (0-6)", snapShotScheduleSectionName) 179 return 180 } 181 182 snapShotSchedule.dayOfWeek = time.Weekday(dayOfWeekAsU64) 183 } 184 185 snapShotSchedule.keep, err = confMap.FetchOptionValueUint64(snapShotScheduleSectionName, "Keep") 186 if nil != err { 187 return 188 } 189 190 if snapShotSchedule.dayOfWeekSpecified && (snapShotSchedule.dayOfMonthSpecified || snapShotSchedule.monthSpecified) { 191 err = fmt.Errorf("%v.CronTab must not specify DayOfWeek if DayOfMonth and/or Month are specified", snapShotScheduleSectionName) 192 return 193 } 194 195 snapShotPolicy.schedule = append(snapShotPolicy.schedule, snapShotSchedule) 196 } 197 198 timeZone, err = confMap.FetchOptionValueString(snapShotPolicySectionName, "TimeZone") 199 200 if nil == err { 201 snapShotPolicy.location, err = time.LoadLocation(timeZone) 202 if nil != err { 203 return 204 } 205 } else { // nil != err 206 // If not present, default to UTC 207 snapShotPolicy.location = time.UTC 208 } 209 210 // If we reach here, we've successfully loaded the snapShotPolicy 211 212 vS.snapShotPolicy = snapShotPolicy 213 err = nil 214 return 215 } 216 217 func (snapShotPolicy *snapShotPolicyStruct) up() { 218 snapShotPolicy.stopChan = make(chan struct{}, 1) 219 snapShotPolicy.doneWaitGroup.Add(1) 220 go snapShotPolicy.daemon() 221 } 222 223 func (snapShotPolicy *snapShotPolicyStruct) down() { 224 snapShotPolicy.stopChan <- struct{}{} 225 snapShotPolicy.doneWaitGroup.Wait() 226 } 227 228 func (snapShotPolicy *snapShotPolicyStruct) daemon() { 229 var ( 230 err error 231 nextDuration time.Duration 232 nextTime time.Time 233 nextTimePreviously time.Time 234 snapShotName string 235 timeNow time.Time 236 ) 237 238 nextTimePreviously = time.Date(2000, time.January, 1, 0, 0, 0, 0, snapShotPolicy.location) 239 240 for { 241 timeNow = time.Now().In(snapShotPolicy.location) 242 nextTime = snapShotPolicy.next(timeNow) 243 for { 244 if !nextTime.Equal(nextTimePreviously) { 245 break 246 } 247 // We took the last snapshot so quickly, next() returned the same nextTime 248 time.Sleep(time.Second) 249 timeNow = time.Now().In(snapShotPolicy.location) 250 nextTime = snapShotPolicy.next(timeNow) 251 } 252 nextDuration = nextTime.Sub(timeNow) 253 select { 254 case _ = <-snapShotPolicy.stopChan: 255 snapShotPolicy.doneWaitGroup.Done() 256 return 257 case <-time.After(nextDuration): 258 for time.Now().In(snapShotPolicy.location).Before(nextTime) { 259 // If time.After() returned a bit too soon, loop until it is our time 260 time.Sleep(100 * time.Millisecond) 261 } 262 snapShotName = strings.Replace(nextTime.Format(time.RFC3339), ":", ".", -1) 263 _, err = snapShotPolicy.volume.SnapShotCreate(snapShotName) 264 if nil != err { 265 logger.WarnWithError(err) 266 } 267 snapShotPolicy.prune() 268 } 269 nextTimePreviously = nextTime 270 } 271 } 272 273 func (snapShotPolicy *snapShotPolicyStruct) prune() { 274 var ( 275 err error 276 keep bool 277 matches bool 278 matchesAtLeastOne bool 279 snapShot headhunter.SnapShotStruct 280 snapShotList []headhunter.SnapShotStruct 281 snapShotSchedule *snapShotScheduleStruct 282 snapShotTime time.Time 283 ) 284 285 // First, zero each snapShotSchedule.count 286 287 for _, snapShotSchedule = range snapShotPolicy.schedule { 288 snapShotSchedule.count = 0 289 } 290 291 // Now fetch the reverse time-ordered list of snapshots 292 293 snapShotList = snapShotPolicy.volume.headhunterVolumeHandle.SnapShotListByTime(true) 294 295 // Now walk snapShotList looking for snapshots to prune 296 297 for _, snapShot = range snapShotList { 298 snapShotTime, err = time.Parse(time.RFC3339, strings.Replace(snapShot.Name, ".", ":", -1)) 299 if nil != err { 300 // SnapShot was not formatted to match a potential SnapShotPolicy/Schedule...skip it 301 continue 302 } 303 304 // Compare against each snapShotSchedule 305 306 keep = false 307 matchesAtLeastOne = false 308 309 for _, snapShotSchedule = range snapShotPolicy.schedule { 310 matches = snapShotSchedule.compare(snapShotTime) 311 if matches { 312 matchesAtLeastOne = true 313 snapShotSchedule.count++ 314 if snapShotSchedule.count <= snapShotSchedule.keep { 315 keep = true 316 } 317 } 318 } 319 320 if matchesAtLeastOne && !keep { 321 // Although this snapshot "matchesAtLeastOne", 322 // no snapShotSchedule said "keep" it 323 324 err = snapShotPolicy.volume.SnapShotDelete(snapShot.ID) 325 if nil != err { 326 logger.WarnWithError(err) 327 } 328 } 329 } 330 } 331 332 // thisTime is presumably the snapShotSchedule.policy.location-local parsed snapShotStruct.name 333 func (snapShotSchedule *snapShotScheduleStruct) compare(thisTime time.Time) (matches bool) { 334 var ( 335 dayOfMonth int 336 dayOfWeek time.Weekday 337 hour int 338 minute int 339 month time.Month 340 truncatedTime time.Time 341 year int 342 ) 343 344 hour, minute, _ = thisTime.Clock() 345 year, month, dayOfMonth = thisTime.Date() 346 dayOfWeek = thisTime.Weekday() 347 348 truncatedTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location) 349 if !truncatedTime.Equal(thisTime) { 350 matches = false 351 return 352 } 353 354 if snapShotSchedule.minuteSpecified { 355 if snapShotSchedule.minute != minute { 356 matches = false 357 return 358 } 359 } 360 361 if snapShotSchedule.hourSpecified { 362 if snapShotSchedule.hour != hour { 363 matches = false 364 return 365 } 366 } 367 368 if snapShotSchedule.dayOfMonthSpecified { 369 if snapShotSchedule.dayOfMonth != dayOfMonth { 370 matches = false 371 return 372 } 373 } 374 375 if snapShotSchedule.monthSpecified { 376 if snapShotSchedule.month != month { 377 matches = false 378 return 379 } 380 } 381 382 if snapShotSchedule.dayOfWeekSpecified { 383 if snapShotSchedule.dayOfWeek != dayOfWeek { 384 matches = false 385 return 386 } 387 } 388 389 // If we make it this far, thisTime matches snapShotSchedule 390 391 matches = true 392 return 393 } 394 395 // Since time.Truncate() only truncates with respect to UTC, it is unsafe 396 397 func truncateToStartOfMinute(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 398 var ( 399 day int 400 hour int 401 min int 402 month time.Month 403 year int 404 ) 405 406 hour, min, _ = untruncatedTime.Clock() 407 year, month, day = untruncatedTime.Date() 408 409 truncatedTime = time.Date(year, month, day, hour, min, 0, 0, loc) 410 411 return 412 } 413 414 func truncateToStartOfHour(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 415 var ( 416 day int 417 hour int 418 month time.Month 419 year int 420 ) 421 422 hour, _, _ = untruncatedTime.Clock() 423 year, month, day = untruncatedTime.Date() 424 425 truncatedTime = time.Date(year, month, day, hour, 0, 0, 0, loc) 426 427 return 428 } 429 430 func truncateToStartOfDay(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 431 var ( 432 day int 433 month time.Month 434 year int 435 ) 436 437 year, month, day = untruncatedTime.Date() 438 439 truncatedTime = time.Date(year, month, day, 0, 0, 0, 0, loc) 440 441 return 442 } 443 444 func truncateToStartOfMonth(untruncatedTime time.Time, loc *time.Location) (truncatedTime time.Time) { 445 var ( 446 month time.Month 447 year int 448 ) 449 450 year, month, _ = untruncatedTime.Date() 451 452 truncatedTime = time.Date(year, month, 1, 0, 0, 0, 0, loc) 453 454 return 455 } 456 457 // timeNow is presumably time.Now() localized to snapShotSchedule.policy.location... 458 // ...but provided here so that each invocation of the per snapShotSchedule 459 // within a snapShotPolicy can use the same value 460 func (snapShotSchedule *snapShotScheduleStruct) next(timeNow time.Time) (nextTime time.Time) { 461 var ( 462 dayOfMonth int 463 dayOfWeek time.Weekday 464 hour int 465 minute int 466 month time.Month 467 numDaysToAdd int 468 numHoursToAdd int 469 numMinutesToAdd int 470 year int 471 ) 472 473 // Ensure nextTime is at least at the start of the next minute 474 nextTime = truncateToStartOfMinute(timeNow, snapShotSchedule.policy.location).Add(time.Minute) 475 476 if snapShotSchedule.minuteSpecified { 477 minute = nextTime.Minute() 478 if snapShotSchedule.minute == minute { 479 // We don't need to advance nextTime 480 } else { 481 // No need to (again) truncate nextTime back to the start of the minute 482 // Now advance nextTime to align with minute 483 if snapShotSchedule.minute > minute { 484 numMinutesToAdd = snapShotSchedule.minute - minute 485 } else { // snapShotSchedule.minute < minute 486 numMinutesToAdd = snapShotSchedule.minute + 60 - minute 487 } 488 nextTime = nextTime.Add(time.Duration(numMinutesToAdd) * time.Minute) 489 } 490 } 491 492 if snapShotSchedule.hourSpecified { 493 hour = nextTime.Hour() 494 if snapShotSchedule.hour == hour { 495 // We don't need to advance nextTime 496 } else { 497 // First truncate nextTime back to the start of the hour 498 nextTime = truncateToStartOfHour(nextTime, snapShotSchedule.policy.location) 499 // Restore minuteSpecified if necessary 500 if snapShotSchedule.minuteSpecified { 501 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 502 } 503 // Now advance nextTime to align with hour 504 if snapShotSchedule.hour > hour { 505 numHoursToAdd = snapShotSchedule.hour - hour 506 } else { // snapShotSchedule.hour < hour 507 numHoursToAdd = snapShotSchedule.hour + 24 - hour 508 } 509 nextTime = nextTime.Add(time.Duration(numHoursToAdd) * time.Hour) 510 } 511 } 512 513 if snapShotSchedule.dayOfMonthSpecified { 514 dayOfMonth = nextTime.Day() 515 if snapShotSchedule.dayOfMonth == dayOfMonth { 516 // We don't need to advance nextTime 517 } else { 518 // First truncate nextTime back to the start of the day 519 nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location) 520 // Restore minuteSpecified and/or hourSpecified if necessary 521 if snapShotSchedule.minuteSpecified { 522 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 523 } 524 if snapShotSchedule.hourSpecified { 525 nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) 526 } 527 // Now advance nextTime to align with dayOfMonth 528 // Note: This unfortunately iterative approach avoids complicated 529 // adjustments for the non-fixed number of days in a month 530 for { 531 nextTime = nextTime.Add(24 * time.Hour) 532 dayOfMonth = nextTime.Day() 533 if snapShotSchedule.dayOfMonth == dayOfMonth { 534 break 535 } 536 } 537 } 538 } 539 540 if snapShotSchedule.monthSpecified { 541 month = nextTime.Month() 542 if snapShotSchedule.month == month { 543 // We don't need to advance nextTime 544 } else { 545 // First truncate nextTime back to the start of the month 546 nextTime = truncateToStartOfMonth(nextTime, snapShotSchedule.policy.location) 547 // Restore minuteSpecified, hourSpecified, and/or dayOfMonthSpecified if necessary 548 if snapShotSchedule.minuteSpecified { 549 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 550 } 551 if snapShotSchedule.hourSpecified { 552 nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) 553 } 554 if snapShotSchedule.dayOfMonthSpecified { 555 nextTime = nextTime.Add(time.Duration((snapShotSchedule.dayOfMonth-1)*24) * time.Hour) 556 } 557 // Now advance nextTime to align with month 558 // Note: This unfortunately iterative approach avoids complicated 559 // adjustments for the non-fixed number of days in a month 560 hour, minute, _ = nextTime.Clock() 561 year, month, dayOfMonth = nextTime.Date() 562 if !snapShotSchedule.dayOfMonthSpecified { 563 dayOfMonth = 1 564 } 565 for { 566 if time.December == month { 567 month = time.January 568 year++ 569 } else { 570 month++ 571 } 572 nextTime = time.Date(year, month, dayOfMonth, hour, minute, 0, 0, snapShotSchedule.policy.location) 573 year, month, dayOfMonth = nextTime.Date() 574 if snapShotSchedule.dayOfMonthSpecified { 575 if (snapShotSchedule.month == month) && (snapShotSchedule.dayOfMonth == dayOfMonth) { 576 break 577 } else { 578 dayOfMonth = snapShotSchedule.dayOfMonth 579 } 580 } else { 581 if (snapShotSchedule.month == month) && (1 == dayOfMonth) { 582 break 583 } else { 584 dayOfMonth = 1 585 } 586 } 587 } 588 } 589 } 590 591 if snapShotSchedule.dayOfWeekSpecified { 592 dayOfWeek = nextTime.Weekday() 593 if time.Weekday(snapShotSchedule.dayOfWeek) == dayOfWeek { 594 // We don't need to advance nextTime 595 } else { 596 // First truncate nextTime back to the start of the day 597 nextTime = truncateToStartOfDay(nextTime, snapShotSchedule.policy.location) 598 // Restore minuteSpecified and/or hourSpecified if necessary 599 if snapShotSchedule.minuteSpecified { 600 nextTime = nextTime.Add(time.Duration(snapShotSchedule.minute) * time.Minute) 601 } 602 if snapShotSchedule.hourSpecified { 603 nextTime = nextTime.Add(time.Duration(snapShotSchedule.hour) * time.Hour) 604 } 605 // Now advance nextTime to align with dayOfWeek 606 if time.Weekday(snapShotSchedule.dayOfWeek) > dayOfWeek { 607 numDaysToAdd = int(snapShotSchedule.dayOfWeek) - int(dayOfWeek) 608 } else { // time.Weekday(snapShotSchedule.dayOfWeek) < dayOfWeek 609 numDaysToAdd = int(snapShotSchedule.dayOfWeek) + 7 - int(dayOfWeek) 610 } 611 nextTime = nextTime.Add(time.Duration(24*numDaysToAdd) * time.Hour) 612 } 613 } 614 615 return 616 } 617 618 // timeNow is presumably time.Now() localized to snapShotPolicy.location... 619 // ...but provided here primarily to enable easy testing 620 func (snapShotPolicy *snapShotPolicyStruct) next(timeNow time.Time) (nextTime time.Time) { 621 var ( 622 nextTimeForSnapShotSchedule time.Time 623 nextTimeHasBeenSet bool 624 snapShotSchedule *snapShotScheduleStruct 625 ) 626 627 nextTimeHasBeenSet = false 628 629 for _, snapShotSchedule = range snapShotPolicy.schedule { 630 nextTimeForSnapShotSchedule = snapShotSchedule.next(timeNow) 631 if nextTimeHasBeenSet { 632 if nextTimeForSnapShotSchedule.Before(nextTime) { 633 nextTime = nextTimeForSnapShotSchedule 634 } 635 } else { 636 nextTime = nextTimeForSnapShotSchedule 637 nextTimeHasBeenSet = true 638 } 639 } 640 641 return 642 }