github.com/bugraaydogar/snapd@v0.0.0-20210315170335-8c70bb858939/overlord/snapstate/autorefresh.go (about) 1 // -*- Mode: Go; indent-tabs-mode: t -*- 2 3 /* 4 * Copyright (C) 2017-2020 Canonical Ltd 5 * 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 3 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 package snapstate 21 22 import ( 23 "context" 24 "fmt" 25 "os" 26 "time" 27 28 "github.com/snapcore/snapd/httputil" 29 "github.com/snapcore/snapd/i18n" 30 "github.com/snapcore/snapd/logger" 31 "github.com/snapcore/snapd/overlord/auth" 32 "github.com/snapcore/snapd/overlord/configstate/config" 33 "github.com/snapcore/snapd/overlord/state" 34 "github.com/snapcore/snapd/release" 35 "github.com/snapcore/snapd/snap" 36 "github.com/snapcore/snapd/strutil" 37 "github.com/snapcore/snapd/timeutil" 38 "github.com/snapcore/snapd/timings" 39 userclient "github.com/snapcore/snapd/usersession/client" 40 ) 41 42 // the default refresh pattern 43 const defaultRefreshSchedule = "00:00~24:00/4" 44 45 // cannot keep without refreshing for more than maxPostponement 46 const maxPostponement = 60 * 24 * time.Hour 47 48 // cannot inhibit refreshes for more than maxInhibition 49 const maxInhibition = 14 * 24 * time.Hour 50 51 // hooks setup by devicestate 52 var ( 53 CanAutoRefresh func(st *state.State) (bool, error) 54 CanManageRefreshes func(st *state.State) bool 55 IsOnMeteredConnection func() (bool, error) 56 ) 57 58 // refreshRetryDelay specified the minimum time to retry failed refreshes 59 var refreshRetryDelay = 20 * time.Minute 60 61 // autoRefresh will ensure that snaps are refreshed automatically 62 // according to the refresh schedule. 63 type autoRefresh struct { 64 state *state.State 65 66 lastRefreshSchedule string 67 nextRefresh time.Time 68 lastRefreshAttempt time.Time 69 managedDeniedLogged bool 70 } 71 72 func newAutoRefresh(st *state.State) *autoRefresh { 73 return &autoRefresh{ 74 state: st, 75 } 76 } 77 78 // RefreshSchedule will return a user visible string with the current schedule 79 // for the automatic refreshes and a flag indicating whether the schedule is a 80 // legacy one. 81 func (m *autoRefresh) RefreshSchedule() (schedule string, legacy bool, err error) { 82 _, schedule, legacy, err = m.refreshScheduleWithDefaultsFallback() 83 return schedule, legacy, err 84 } 85 86 // NextRefresh returns when the next automatic refresh will happen. 87 func (m *autoRefresh) NextRefresh() time.Time { 88 return m.nextRefresh 89 } 90 91 // LastRefresh returns when the last refresh happened. 92 func (m *autoRefresh) LastRefresh() (time.Time, error) { 93 return getTime(m.state, "last-refresh") 94 } 95 96 // EffectiveRefreshHold returns the time until to which refreshes are 97 // held if refresh.hold configuration is set and accounting for the 98 // max postponement since the last refresh. 99 func (m *autoRefresh) EffectiveRefreshHold() (time.Time, error) { 100 var holdTime time.Time 101 102 tr := config.NewTransaction(m.state) 103 err := tr.Get("core", "refresh.hold", &holdTime) 104 if err != nil && !config.IsNoOption(err) { 105 return time.Time{}, err 106 } 107 108 // cannot hold beyond last-refresh + max-postponement 109 lastRefresh, err := m.LastRefresh() 110 if err != nil { 111 return time.Time{}, err 112 } 113 if lastRefresh.IsZero() { 114 seedTime, err := getTime(m.state, "seed-time") 115 if err != nil { 116 return time.Time{}, err 117 } 118 if seedTime.IsZero() { 119 // no reference to know whether holding is reasonable 120 return time.Time{}, nil 121 } 122 lastRefresh = seedTime 123 } 124 125 limitTime := lastRefresh.Add(maxPostponement) 126 if holdTime.After(limitTime) { 127 return limitTime, nil 128 } 129 130 return holdTime, nil 131 } 132 133 func (m *autoRefresh) ensureRefreshHoldAtLeast(duration time.Duration) error { 134 now := time.Now() 135 136 // get the effective refresh hold and check if it is sooner than the 137 // specified duration in the future 138 effective, err := m.EffectiveRefreshHold() 139 if err != nil { 140 return err 141 } 142 143 if effective.IsZero() || effective.Sub(now) < duration { 144 // the effective refresh hold is sooner than the desired delay, so 145 // move it out to the specified duration 146 holdTime := now.Add(duration) 147 tr := config.NewTransaction(m.state) 148 err := tr.Set("core", "refresh.hold", &holdTime) 149 if err != nil && !config.IsNoOption(err) { 150 return err 151 } 152 tr.Commit() 153 } 154 155 return nil 156 } 157 158 // clearRefreshHold clears refresh.hold configuration. 159 func (m *autoRefresh) clearRefreshHold() { 160 tr := config.NewTransaction(m.state) 161 tr.Set("core", "refresh.hold", nil) 162 tr.Commit() 163 } 164 165 // AtSeed configures refresh policies at end of seeding. 166 func (m *autoRefresh) AtSeed() error { 167 // on classic hold refreshes for 2h after seeding 168 if release.OnClassic { 169 var t1 time.Time 170 tr := config.NewTransaction(m.state) 171 err := tr.Get("core", "refresh.hold", &t1) 172 if !config.IsNoOption(err) { 173 // already set or error 174 return err 175 } 176 // TODO: have a policy that if the snapd exe itself 177 // is older than X weeks/months we skip the holding? 178 now := time.Now().UTC() 179 tr.Set("core", "refresh.hold", now.Add(2*time.Hour)) 180 tr.Commit() 181 m.nextRefresh = now 182 } 183 return nil 184 } 185 186 func canRefreshOnMeteredConnection(st *state.State) (bool, error) { 187 tr := config.NewTransaction(st) 188 var onMetered string 189 err := tr.GetMaybe("core", "refresh.metered", &onMetered) 190 if err != nil && err != state.ErrNoState { 191 return false, err 192 } 193 194 return onMetered != "hold", nil 195 } 196 197 func (m *autoRefresh) canRefreshRespectingMetered(now, lastRefresh time.Time) (can bool, err error) { 198 can, err = canRefreshOnMeteredConnection(m.state) 199 if err != nil { 200 return false, err 201 } 202 if can { 203 return true, nil 204 } 205 206 // ignore any errors that occurred while checking if we are on a metered 207 // connection 208 metered, _ := IsOnMeteredConnection() 209 if !metered { 210 return true, nil 211 } 212 213 if now.Sub(lastRefresh) >= maxPostponement { 214 // TODO use warnings when the infra becomes available 215 logger.Noticef("Auto refresh disabled while on metered connections, but pending for too long (%d days). Trying to refresh now.", int(maxPostponement.Hours()/24)) 216 return true, nil 217 } 218 219 logger.Debugf("Auto refresh disabled on metered connections") 220 221 return false, nil 222 } 223 224 // Ensure ensures that we refresh all installed snaps periodically 225 func (m *autoRefresh) Ensure() error { 226 m.state.Lock() 227 defer m.state.Unlock() 228 229 // see if it even makes sense to try to refresh 230 if CanAutoRefresh == nil { 231 return nil 232 } 233 if ok, err := CanAutoRefresh(m.state); err != nil || !ok { 234 return err 235 } 236 237 // get lastRefresh and schedule 238 lastRefresh, err := m.LastRefresh() 239 if err != nil { 240 return err 241 } 242 243 refreshSchedule, refreshScheduleStr, _, err := m.refreshScheduleWithDefaultsFallback() 244 if err != nil { 245 return err 246 } 247 if len(refreshSchedule) == 0 { 248 m.nextRefresh = time.Time{} 249 return nil 250 } 251 // we already have a refresh time, check if we got a new config 252 if !m.nextRefresh.IsZero() { 253 if m.lastRefreshSchedule != refreshScheduleStr { 254 // the refresh schedule has changed 255 logger.Debugf("Refresh timer changed.") 256 m.nextRefresh = time.Time{} 257 } 258 } 259 m.lastRefreshSchedule = refreshScheduleStr 260 261 // ensure nothing is in flight already 262 if autoRefreshInFlight(m.state) { 263 return nil 264 } 265 266 now := time.Now() 267 // compute next refresh attempt time (if needed) 268 if m.nextRefresh.IsZero() { 269 // store attempts in memory so that we can backoff 270 if !lastRefresh.IsZero() { 271 delta := timeutil.Next(refreshSchedule, lastRefresh, maxPostponement) 272 now = time.Now() 273 m.nextRefresh = now.Add(delta) 274 } else { 275 // make sure either seed-time or last-refresh 276 // are set for hold code below 277 m.ensureLastRefreshAnchor() 278 // immediate 279 m.nextRefresh = now 280 } 281 logger.Debugf("Next refresh scheduled for %s.", m.nextRefresh.Format(time.RFC3339)) 282 } 283 284 held, holdTime, err := m.isRefreshHeld(refreshSchedule) 285 if err != nil { 286 return err 287 } 288 289 // do refresh attempt (if needed) 290 if !held { 291 if !holdTime.IsZero() { 292 // expired hold case 293 m.clearRefreshHold() 294 if m.nextRefresh.Before(holdTime) { 295 // next refresh is obsolete, compute the next one 296 delta := timeutil.Next(refreshSchedule, holdTime, maxPostponement) 297 now = time.Now() 298 m.nextRefresh = now.Add(delta) 299 } 300 } 301 302 // refresh is also "held" if the next time is in the future 303 // note that the two times here could be exactly equal, so we use 304 // !After() because that is true in the case that the next refresh is 305 // before now, and the next refresh is equal to now without requiring an 306 // or operation 307 if !m.nextRefresh.After(now) { 308 var can bool 309 can, err = m.canRefreshRespectingMetered(now, lastRefresh) 310 if err != nil { 311 return err 312 } 313 if !can { 314 // clear nextRefresh so that another refresh time is calculated 315 m.nextRefresh = time.Time{} 316 return nil 317 } 318 319 // Check that we have reasonable delays between attempts. 320 // If the store is under stress we need to make sure we do not 321 // hammer it too often 322 if !m.lastRefreshAttempt.IsZero() && m.lastRefreshAttempt.Add(refreshRetryDelay).After(time.Now()) { 323 return nil 324 } 325 326 err = m.launchAutoRefresh(refreshSchedule) 327 if _, ok := err.(*httputil.PersistentNetworkError); !ok { 328 m.nextRefresh = time.Time{} 329 } // else - refresh will be retried after refreshRetryDelay 330 } 331 } 332 333 return err 334 } 335 336 // isRefreshHeld returns whether an auto-refresh is currently held back or not, 337 // as indicated by m.EffectiveRefreshHold(). 338 func (m *autoRefresh) isRefreshHeld(refreshSchedule []*timeutil.Schedule) (bool, time.Time, error) { 339 now := time.Now() 340 // should we hold back refreshes? 341 holdTime, err := m.EffectiveRefreshHold() 342 if err != nil { 343 return false, time.Time{}, err 344 } 345 if holdTime.After(now) { 346 return true, holdTime, nil 347 } 348 349 return false, holdTime, nil 350 } 351 352 func (m *autoRefresh) ensureLastRefreshAnchor() { 353 seedTime, _ := getTime(m.state, "seed-time") 354 if !seedTime.IsZero() { 355 return 356 } 357 358 // last core refresh 359 coreRefreshDate := snap.InstallDate("core") 360 if !coreRefreshDate.IsZero() { 361 m.state.Set("last-refresh", coreRefreshDate) 362 return 363 } 364 365 // fallback to executable time 366 st, err := os.Stat("/proc/self/exe") 367 if err == nil { 368 m.state.Set("last-refresh", st.ModTime()) 369 return 370 } 371 } 372 373 // refreshScheduleWithDefaultsFallback returns the current refresh schedule 374 // and refresh string. When an invalid refresh schedule is set by the user 375 // the refresh schedule is automatically reset to the default. 376 // 377 // TODO: we can remove the refreshSchedule reset because we have validation 378 // of the schedule now. 379 func (m *autoRefresh) refreshScheduleWithDefaultsFallback() (ts []*timeutil.Schedule, scheduleAsStr string, legacy bool, err error) { 380 managed, requested, legacy := refreshScheduleManaged(m.state) 381 if managed { 382 if m.lastRefreshSchedule != "managed" { 383 logger.Noticef("refresh is managed via the snapd-control interface") 384 m.lastRefreshSchedule = "managed" 385 } 386 m.managedDeniedLogged = false 387 return nil, "managed", legacy, nil 388 } else if requested { 389 // managed refresh schedule was denied 390 if !m.managedDeniedLogged { 391 logger.Noticef("managed refresh schedule denied, no properly configured snapd-control") 392 m.managedDeniedLogged = true 393 } 394 // fallback to default schedule 395 return refreshScheduleDefault() 396 } else { 397 m.managedDeniedLogged = false 398 } 399 400 tr := config.NewTransaction(m.state) 401 // try the new refresh.timer config option first 402 err = tr.Get("core", "refresh.timer", &scheduleAsStr) 403 if err != nil && !config.IsNoOption(err) { 404 return nil, "", false, err 405 } 406 if scheduleAsStr != "" { 407 ts, err = timeutil.ParseSchedule(scheduleAsStr) 408 if err != nil { 409 logger.Noticef("cannot use refresh.timer configuration: %s", err) 410 return refreshScheduleDefault() 411 } 412 return ts, scheduleAsStr, false, nil 413 } 414 415 // fallback to legacy refresh.schedule setting when the new 416 // config option is not set 417 err = tr.Get("core", "refresh.schedule", &scheduleAsStr) 418 if err != nil && !config.IsNoOption(err) { 419 return nil, "", false, err 420 } 421 if scheduleAsStr != "" { 422 ts, err = timeutil.ParseLegacySchedule(scheduleAsStr) 423 if err != nil { 424 logger.Noticef("cannot use refresh.schedule configuration: %s", err) 425 return refreshScheduleDefault() 426 } 427 return ts, scheduleAsStr, true, nil 428 } 429 430 return refreshScheduleDefault() 431 } 432 433 // launchAutoRefresh creates the auto-refresh taskset and a change for it. 434 func (m *autoRefresh) launchAutoRefresh(refreshSchedule []*timeutil.Schedule) error { 435 perfTimings := timings.New(map[string]string{"ensure": "auto-refresh"}) 436 tm := perfTimings.StartSpan("auto-refresh", "query store and setup auto-refresh change") 437 defer func() { 438 tm.Stop() 439 perfTimings.Save(m.state) 440 }() 441 442 m.lastRefreshAttempt = time.Now() 443 444 // NOTE: this will unlock and re-lock state for network ops 445 updated, tasksets, err := AutoRefresh(auth.EnsureContextTODO(), m.state) 446 447 // TODO: we should have some way to lock just creating and starting changes, 448 // as that would alleviate this race condition we are guarding against 449 // with this check and probably would eliminate other similar race 450 // conditions elsewhere 451 452 // re-check if the refresh is held because it could have been re-held and 453 // pushed back, in which case we need to abort the auto-refresh and wait 454 held, _, holdErr := m.isRefreshHeld(refreshSchedule) 455 if holdErr != nil { 456 return holdErr 457 } 458 459 if held { 460 // then a request came in that pushed the refresh out, so we will need 461 // to try again later 462 logger.Noticef("Auto-refresh was delayed mid-way through launching, aborting to try again later") 463 return nil 464 } 465 466 if _, ok := err.(*httputil.PersistentNetworkError); ok { 467 logger.Noticef("Cannot prepare auto-refresh change due to a permanent network error: %s", err) 468 return err 469 } 470 m.state.Set("last-refresh", time.Now()) 471 if err != nil { 472 logger.Noticef("Cannot prepare auto-refresh change: %s", err) 473 return err 474 } 475 476 var msg string 477 switch len(updated) { 478 case 0: 479 logger.Noticef(i18n.G("auto-refresh: all snaps are up-to-date")) 480 return nil 481 case 1: 482 msg = fmt.Sprintf(i18n.G("Auto-refresh snap %q"), updated[0]) 483 case 2, 3: 484 quoted := strutil.Quoted(updated) 485 // TRANSLATORS: the %s is a comma-separated list of quoted snap names 486 msg = fmt.Sprintf(i18n.G("Auto-refresh snaps %s"), quoted) 487 default: 488 msg = fmt.Sprintf(i18n.G("Auto-refresh %d snaps"), len(updated)) 489 } 490 491 chg := m.state.NewChange("auto-refresh", msg) 492 for _, ts := range tasksets { 493 chg.AddAll(ts) 494 } 495 chg.Set("snap-names", updated) 496 chg.Set("api-data", map[string]interface{}{"snap-names": updated}) 497 state.TagTimingsWithChange(perfTimings, chg) 498 499 return nil 500 } 501 502 func refreshScheduleDefault() (ts []*timeutil.Schedule, scheduleStr string, legacy bool, err error) { 503 refreshSchedule, err := timeutil.ParseSchedule(defaultRefreshSchedule) 504 if err != nil { 505 panic(fmt.Sprintf("defaultRefreshSchedule cannot be parsed: %s", err)) 506 } 507 508 return refreshSchedule, defaultRefreshSchedule, false, nil 509 } 510 511 func autoRefreshInFlight(st *state.State) bool { 512 for _, chg := range st.Changes() { 513 if chg.Kind() == "auto-refresh" && !chg.Status().Ready() { 514 return true 515 } 516 } 517 return false 518 } 519 520 // refreshScheduleManaged returns true if the refresh schedule of the 521 // device is managed by an external snap 522 func refreshScheduleManaged(st *state.State) (managed, requested, legacy bool) { 523 var confStr string 524 525 // this will only be "nil" if running in tests 526 if CanManageRefreshes == nil { 527 return false, false, legacy 528 } 529 530 // check new style timer first 531 tr := config.NewTransaction(st) 532 err := tr.Get("core", "refresh.timer", &confStr) 533 if err != nil && !config.IsNoOption(err) { 534 return false, false, legacy 535 } 536 // if not set, fallback to refresh.schedule 537 if confStr == "" { 538 if err := tr.Get("core", "refresh.schedule", &confStr); err != nil { 539 return false, false, legacy 540 } 541 legacy = true 542 } 543 544 if confStr != "managed" { 545 return false, false, legacy 546 } 547 548 return CanManageRefreshes(st), true, legacy 549 } 550 551 // getTime retrieves a time from a state value. 552 func getTime(st *state.State, timeKey string) (time.Time, error) { 553 var t1 time.Time 554 err := st.Get(timeKey, &t1) 555 if err != nil && err != state.ErrNoState { 556 return time.Time{}, err 557 } 558 return t1, nil 559 } 560 561 // asyncPendingRefreshNotification broadcasts desktop notification in a goroutine. 562 // 563 // This allows the, possibly slow, communication with each snapd session agent, 564 // to be performed without holding the snap state lock. 565 var asyncPendingRefreshNotification = func(context context.Context, client *userclient.Client, refreshInfo *userclient.PendingSnapRefreshInfo) { 566 go func() { 567 if err := client.PendingRefreshNotification(context, refreshInfo); err != nil { 568 logger.Noticef("Cannot send notification about pending refresh: %v", err) 569 } 570 }() 571 } 572 573 // inhibitRefresh returns an error if refresh is inhibited by running apps. 574 // 575 // Internally the snap state is updated to remember when the inhibition first 576 // took place. Apps can inhibit refreshes for up to "maxInhibition", beyond 577 // that period the refresh will go ahead despite application activity. 578 func inhibitRefresh(st *state.State, snapst *SnapState, info *snap.Info, checker func(*snap.Info) error) error { 579 checkerErr := checker(info) 580 if checkerErr == nil { 581 return nil 582 } 583 584 // Get pending refresh information from compatible errors or synthesize a new one. 585 var refreshInfo *userclient.PendingSnapRefreshInfo 586 if err, ok := checkerErr.(*BusySnapError); ok { 587 refreshInfo = err.PendingSnapRefreshInfo() 588 } else { 589 refreshInfo = &userclient.PendingSnapRefreshInfo{ 590 InstanceName: info.InstanceName(), 591 } 592 } 593 594 // Decide on what to do depending on the state of the snap and the remaining 595 // inhibition time. 596 now := time.Now() 597 switch { 598 case snapst.RefreshInhibitedTime == nil: 599 // If the snap did not have inhibited refresh yet then commence a new 600 // window, during which refreshes are postponed, by storing the current 601 // time in the snap state's RefreshInhibitedTime field. This field is 602 // reset to nil on successful refresh. 603 snapst.RefreshInhibitedTime = &now 604 refreshInfo.TimeRemaining = (maxInhibition - now.Sub(*snapst.RefreshInhibitedTime)).Truncate(time.Second) 605 Set(st, info.InstanceName(), snapst) 606 case now.Sub(*snapst.RefreshInhibitedTime) < maxInhibition: 607 // If we are still in the allowed window then just return the error but 608 // don't change the snap state again. 609 // TODO: as time left shrinks, send additional notifications with 610 // increasing frequency, allowing the user to understand the urgency. 611 refreshInfo.TimeRemaining = (maxInhibition - now.Sub(*snapst.RefreshInhibitedTime)).Truncate(time.Second) 612 default: 613 // If we run out of time then consume the error that would normally 614 // inhibit refresh and notify the user that the snap is refreshing right 615 // now, by not setting the TimeRemaining field of the refresh 616 // notification message. 617 checkerErr = nil 618 } 619 620 // Send the notification asynchronously to avoid holding the state lock. 621 asyncPendingRefreshNotification(context.TODO(), userclient.New(), refreshInfo) 622 return checkerErr 623 }