github.com/rigado/snapd@v2.42.5-go-mod+incompatible/store/store.go (about) 1 // -*- Mode: Go; indent-tabs-mode: t -*- 2 3 /* 4 * Copyright (C) 2014-2018 Canonical Ltd 5 * 6 * This program is free software: you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 3 as 8 * published by the Free Software Foundation. 9 * 10 * This program is distributed in the hope that it will be useful, 11 * but WITHOUT ANY WARRANTY; without even the implied warranty of 12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 13 * GNU General Public License for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program. If not, see <http://www.gnu.org/licenses/>. 17 * 18 */ 19 20 // Package store has support to use the Ubuntu Store for querying and downloading of snaps, and the related services. 21 package store 22 23 import ( 24 "bytes" 25 "context" 26 "crypto" 27 "encoding/base64" 28 "encoding/json" 29 "errors" 30 "fmt" 31 "io" 32 "net/http" 33 "net/url" 34 "os" 35 "os/exec" 36 "path" 37 "path/filepath" 38 "strconv" 39 "strings" 40 "sync" 41 "time" 42 43 "github.com/juju/ratelimit" 44 "gopkg.in/retry.v1" 45 46 "github.com/snapcore/snapd/arch" 47 "github.com/snapcore/snapd/asserts" 48 "github.com/snapcore/snapd/client" 49 "github.com/snapcore/snapd/cmd/cmdutil" 50 "github.com/snapcore/snapd/dirs" 51 "github.com/snapcore/snapd/httputil" 52 "github.com/snapcore/snapd/i18n" 53 "github.com/snapcore/snapd/jsonutil" 54 "github.com/snapcore/snapd/logger" 55 "github.com/snapcore/snapd/osutil" 56 "github.com/snapcore/snapd/overlord/auth" 57 "github.com/snapcore/snapd/progress" 58 "github.com/snapcore/snapd/release" 59 "github.com/snapcore/snapd/snap" 60 "github.com/snapcore/snapd/strutil" 61 ) 62 63 // TODO: better/shorter names are probably in order once fewer legacy places are using this 64 65 const ( 66 // halJsonContentType is the default accept value for store requests 67 halJsonContentType = "application/hal+json" 68 // jsonContentType is for store enpoints that don't support HAL 69 jsonContentType = "application/json" 70 // UbuntuCoreWireProtocol is the protocol level we support when 71 // communicating with the store. History: 72 // - "1": client supports squashfs snaps 73 UbuntuCoreWireProtocol = "1" 74 ) 75 76 type RefreshOptions struct { 77 // RefreshManaged indicates to the store that the refresh is 78 // managed via snapd-control. 79 RefreshManaged bool 80 IsAutoRefresh bool 81 82 PrivacyKey string 83 } 84 85 // the LimitTime should be slightly more than 3 times of our http.Client 86 // Timeout value 87 var defaultRetryStrategy = retry.LimitCount(6, retry.LimitTime(38*time.Second, 88 retry.Exponential{ 89 Initial: 350 * time.Millisecond, 90 Factor: 2.5, 91 }, 92 )) 93 94 var downloadRetryStrategy = retry.LimitCount(7, retry.LimitTime(90*time.Second, 95 retry.Exponential{ 96 Initial: 500 * time.Millisecond, 97 Factor: 2.5, 98 }, 99 )) 100 101 var connCheckStrategy = retry.LimitCount(3, retry.LimitTime(38*time.Second, 102 retry.Exponential{ 103 Initial: 900 * time.Millisecond, 104 Factor: 1.3, 105 }, 106 )) 107 108 // Config represents the configuration to access the snap store 109 type Config struct { 110 // Store API base URLs. The assertions url is only separate because it can 111 // be overridden by its own env var. 112 StoreBaseURL *url.URL 113 AssertionsBaseURL *url.URL 114 115 // StoreID is the store id used if we can't get one through the DeviceAndAuthContext. 116 StoreID string 117 118 Architecture string 119 Series string 120 121 DetailFields []string 122 InfoFields []string 123 DeltaFormat string 124 125 // CacheDownloads is the number of downloads that should be cached 126 CacheDownloads int 127 128 // Proxy returns the HTTP proxy to use when talking to the store 129 Proxy func(*http.Request) (*url.URL, error) 130 } 131 132 // setBaseURL updates the store API's base URL in the Config. Must not be used 133 // to change active config. 134 func (cfg *Config) setBaseURL(u *url.URL) error { 135 storeBaseURI, err := storeURL(u) 136 if err != nil { 137 return err 138 } 139 140 assertsBaseURI, err := assertsURL() 141 if err != nil { 142 return err 143 } 144 145 cfg.StoreBaseURL = storeBaseURI 146 cfg.AssertionsBaseURL = assertsBaseURI 147 148 return nil 149 } 150 151 // Store represents the ubuntu snap store 152 type Store struct { 153 cfg *Config 154 155 architecture string 156 series string 157 158 noCDN bool 159 160 fallbackStoreID string 161 162 detailFields []string 163 infoFields []string 164 deltaFormat string 165 // reused http client 166 client *http.Client 167 168 dauthCtx DeviceAndAuthContext 169 sessionMu sync.Mutex 170 171 mu sync.Mutex 172 suggestedCurrency string 173 174 cacher downloadCache 175 proxy func(*http.Request) (*url.URL, error) 176 } 177 178 var ErrTooManyRequests = errors.New("too many requests") 179 180 func respToError(resp *http.Response, msg string) error { 181 if resp.StatusCode == 429 { 182 return ErrTooManyRequests 183 } 184 185 tpl := "cannot %s: got unexpected HTTP status code %d via %s to %q" 186 if oops := resp.Header.Get("X-Oops-Id"); oops != "" { 187 tpl += " [%s]" 188 return fmt.Errorf(tpl, msg, resp.StatusCode, resp.Request.Method, resp.Request.URL, oops) 189 } 190 191 return fmt.Errorf(tpl, msg, resp.StatusCode, resp.Request.Method, resp.Request.URL) 192 } 193 194 // Deltas enabled by default on classic, but allow opting in or out on both classic and core. 195 func useDeltas() bool { 196 // only xdelta3 is supported for now, so check the binary exists here 197 // TODO: have a per-format checker instead 198 if _, err := getXdelta3Cmd(); err != nil { 199 return false 200 } 201 202 return osutil.GetenvBool("SNAPD_USE_DELTAS_EXPERIMENTAL", true) 203 } 204 205 func useStaging() bool { 206 return osutil.GetenvBool("SNAPPY_USE_STAGING_STORE") 207 } 208 209 // endpointURL clones a base URL and updates it with optional path and query. 210 func endpointURL(base *url.URL, path string, query url.Values) *url.URL { 211 u := *base 212 if path != "" { 213 u.Path = strings.TrimSuffix(u.Path, "/") + "/" + strings.TrimPrefix(path, "/") 214 u.RawQuery = "" 215 } 216 if len(query) != 0 { 217 u.RawQuery = query.Encode() 218 } 219 return &u 220 } 221 222 // apiURL returns the system default base API URL. 223 func apiURL() *url.URL { 224 s := "https://api.snapcraft.io/" 225 if useStaging() { 226 s = "https://api.staging.snapcraft.io/" 227 } 228 u, _ := url.Parse(s) 229 return u 230 } 231 232 // storeURL returns the base store URL, derived from either the given API URL 233 // or an env var override. 234 func storeURL(api *url.URL) (*url.URL, error) { 235 var override string 236 var overrideName string 237 // XXX: time to drop FORCE_CPI support 238 // XXX: Deprecated but present for backward-compatibility: this used 239 // to be "Click Package Index". Remove this once people have got 240 // used to SNAPPY_FORCE_API_URL instead. 241 if s := os.Getenv("SNAPPY_FORCE_CPI_URL"); s != "" && strings.HasSuffix(s, "api/v1/") { 242 overrideName = "SNAPPY_FORCE_CPI_URL" 243 override = strings.TrimSuffix(s, "api/v1/") 244 } else if s := os.Getenv("SNAPPY_FORCE_API_URL"); s != "" { 245 overrideName = "SNAPPY_FORCE_API_URL" 246 override = s 247 } 248 if override != "" { 249 u, err := url.Parse(override) 250 if err != nil { 251 return nil, fmt.Errorf("invalid %s: %s", overrideName, err) 252 } 253 return u, nil 254 } 255 return api, nil 256 } 257 258 func assertsURL() (*url.URL, error) { 259 if s := os.Getenv("SNAPPY_FORCE_SAS_URL"); s != "" { 260 u, err := url.Parse(s) 261 if err != nil { 262 return nil, fmt.Errorf("invalid SNAPPY_FORCE_SAS_URL: %s", err) 263 } 264 return u, nil 265 } 266 267 // nil means fallback to store base url 268 return nil, nil 269 } 270 271 func authLocation() string { 272 if useStaging() { 273 return "login.staging.ubuntu.com" 274 } 275 return "login.ubuntu.com" 276 } 277 278 func authURL() string { 279 if u := os.Getenv("SNAPPY_FORCE_SSO_URL"); u != "" { 280 return u 281 } 282 return "https://" + authLocation() + "/api/v2" 283 } 284 285 var defaultStoreDeveloperURL = "https://dashboard.snapcraft.io/" 286 287 func storeDeveloperURL() string { 288 if useStaging() { 289 return "https://dashboard.staging.snapcraft.io/" 290 } 291 return defaultStoreDeveloperURL 292 } 293 294 var defaultConfig = Config{} 295 296 // DefaultConfig returns a copy of the default configuration ready to be adapted. 297 func DefaultConfig() *Config { 298 cfg := defaultConfig 299 return &cfg 300 } 301 302 func init() { 303 storeBaseURI, err := storeURL(apiURL()) 304 if err != nil { 305 panic(err) 306 } 307 if storeBaseURI.RawQuery != "" { 308 panic("store API URL may not contain query string") 309 } 310 err = defaultConfig.setBaseURL(storeBaseURI) 311 if err != nil { 312 panic(err) 313 } 314 defaultConfig.DetailFields = jsonutil.StructFields((*snapDetails)(nil), "snap_yaml_raw") 315 defaultConfig.InfoFields = jsonutil.StructFields((*storeSnap)(nil), "snap-yaml") 316 } 317 318 type searchResults struct { 319 Payload struct { 320 Packages []*snapDetails `json:"clickindex:package"` 321 } `json:"_embedded"` 322 } 323 324 type sectionResults struct { 325 Payload struct { 326 Sections []struct{ Name string } `json:"clickindex:sections"` 327 } `json:"_embedded"` 328 } 329 330 // The default delta format if not configured. 331 var defaultSupportedDeltaFormat = "xdelta3" 332 333 // New creates a new Store with the given access configuration and for given the store id. 334 func New(cfg *Config, dauthCtx DeviceAndAuthContext) *Store { 335 if cfg == nil { 336 cfg = &defaultConfig 337 } 338 339 detailFields := cfg.DetailFields 340 if detailFields == nil { 341 detailFields = defaultConfig.DetailFields 342 } 343 344 infoFields := cfg.InfoFields 345 if infoFields == nil { 346 infoFields = defaultConfig.InfoFields 347 } 348 349 architecture := cfg.Architecture 350 if cfg.Architecture == "" { 351 architecture = arch.DpkgArchitecture() 352 } 353 354 series := cfg.Series 355 if cfg.Series == "" { 356 series = release.Series 357 } 358 359 deltaFormat := cfg.DeltaFormat 360 if deltaFormat == "" { 361 deltaFormat = defaultSupportedDeltaFormat 362 } 363 364 store := &Store{ 365 cfg: cfg, 366 series: series, 367 architecture: architecture, 368 noCDN: osutil.GetenvBool("SNAPPY_STORE_NO_CDN"), 369 fallbackStoreID: cfg.StoreID, 370 detailFields: detailFields, 371 infoFields: infoFields, 372 dauthCtx: dauthCtx, 373 deltaFormat: deltaFormat, 374 proxy: cfg.Proxy, 375 376 client: httputil.NewHTTPClient(&httputil.ClientOptions{ 377 Timeout: 10 * time.Second, 378 MayLogBody: true, 379 Proxy: cfg.Proxy, 380 }), 381 } 382 store.SetCacheDownloads(cfg.CacheDownloads) 383 384 return store 385 } 386 387 // API endpoint paths 388 const ( 389 // see https://wiki.ubuntu.com/AppStore/Interfaces/ClickPackageIndex 390 // XXX: Repeating "api/" here is cumbersome, but the next generation 391 // of store APIs will probably drop that prefix (since it now 392 // duplicates the hostname), and we may want to switch to v2 APIs 393 // one at a time; so it's better to consider that as part of 394 // individual endpoint paths. 395 searchEndpPath = "api/v1/snaps/search" 396 ordersEndpPath = "api/v1/snaps/purchases/orders" 397 buyEndpPath = "api/v1/snaps/purchases/buy" 398 customersMeEndpPath = "api/v1/snaps/purchases/customers/me" 399 sectionsEndpPath = "api/v1/snaps/sections" 400 commandsEndpPath = "api/v1/snaps/names" 401 // v2 402 snapActionEndpPath = "v2/snaps/refresh" 403 snapInfoEndpPath = "v2/snaps/info" 404 cohortsEndpPath = "v2/cohorts" 405 406 deviceNonceEndpPath = "api/v1/snaps/auth/nonces" 407 deviceSessionEndpPath = "api/v1/snaps/auth/sessions" 408 409 assertionsPath = "api/v1/snaps/assertions" 410 ) 411 412 func (s *Store) defaultSnapQuery() url.Values { 413 q := url.Values{} 414 if len(s.detailFields) != 0 { 415 q.Set("fields", strings.Join(s.detailFields, ",")) 416 } 417 return q 418 } 419 420 func (s *Store) baseURL(defaultURL *url.URL) *url.URL { 421 u := defaultURL 422 if s.dauthCtx != nil { 423 var err error 424 _, u, err = s.dauthCtx.ProxyStoreParams(defaultURL) 425 if err != nil { 426 logger.Debugf("cannot get proxy store parameters from state: %v", err) 427 } 428 } 429 if u != nil { 430 return u 431 } 432 return defaultURL 433 } 434 435 func (s *Store) endpointURL(p string, query url.Values) *url.URL { 436 return endpointURL(s.baseURL(s.cfg.StoreBaseURL), p, query) 437 } 438 439 func (s *Store) assertionsEndpointURL(p string, query url.Values) *url.URL { 440 defBaseURL := s.cfg.StoreBaseURL 441 // can be overridden separately! 442 if s.cfg.AssertionsBaseURL != nil { 443 defBaseURL = s.cfg.AssertionsBaseURL 444 } 445 return endpointURL(s.baseURL(defBaseURL), path.Join(assertionsPath, p), query) 446 } 447 448 // LoginUser logs user in the store and returns the authentication macaroons. 449 func (s *Store) LoginUser(username, password, otp string) (string, string, error) { 450 macaroon, err := requestStoreMacaroon(s.client) 451 if err != nil { 452 return "", "", err 453 } 454 deserializedMacaroon, err := auth.MacaroonDeserialize(macaroon) 455 if err != nil { 456 return "", "", err 457 } 458 459 // get SSO 3rd party caveat, and request discharge 460 loginCaveat, err := loginCaveatID(deserializedMacaroon) 461 if err != nil { 462 return "", "", err 463 } 464 465 discharge, err := dischargeAuthCaveat(s.client, loginCaveat, username, password, otp) 466 if err != nil { 467 return "", "", err 468 } 469 470 return macaroon, discharge, nil 471 } 472 473 // authAvailable returns true if there is a user and/or device session setup 474 func (s *Store) authAvailable(user *auth.UserState) (bool, error) { 475 if user.HasStoreAuth() { 476 return true, nil 477 } else { 478 var device *auth.DeviceState 479 var err error 480 if s.dauthCtx != nil { 481 device, err = s.dauthCtx.Device() 482 if err != nil { 483 return false, err 484 } 485 } 486 return device != nil && device.SessionMacaroon != "", nil 487 } 488 } 489 490 // authenticateUser will add the store expected Macaroon Authorization header for user 491 func authenticateUser(r *http.Request, user *auth.UserState) { 492 var buf bytes.Buffer 493 fmt.Fprintf(&buf, `Macaroon root="%s"`, user.StoreMacaroon) 494 495 // deserialize root macaroon (we need its signature to do the discharge binding) 496 root, err := auth.MacaroonDeserialize(user.StoreMacaroon) 497 if err != nil { 498 logger.Debugf("cannot deserialize root macaroon: %v", err) 499 return 500 } 501 502 for _, d := range user.StoreDischarges { 503 // prepare discharge for request 504 discharge, err := auth.MacaroonDeserialize(d) 505 if err != nil { 506 logger.Debugf("cannot deserialize discharge macaroon: %v", err) 507 return 508 } 509 discharge.Bind(root.Signature()) 510 511 serializedDischarge, err := auth.MacaroonSerialize(discharge) 512 if err != nil { 513 logger.Debugf("cannot re-serialize discharge macaroon: %v", err) 514 return 515 } 516 fmt.Fprintf(&buf, `, discharge="%s"`, serializedDischarge) 517 } 518 r.Header.Set("Authorization", buf.String()) 519 } 520 521 // refreshDischarges will request refreshed discharge macaroons for the user 522 func refreshDischarges(httpClient *http.Client, user *auth.UserState) ([]string, error) { 523 newDischarges := make([]string, len(user.StoreDischarges)) 524 for i, d := range user.StoreDischarges { 525 discharge, err := auth.MacaroonDeserialize(d) 526 if err != nil { 527 return nil, err 528 } 529 if discharge.Location() != UbuntuoneLocation { 530 newDischarges[i] = d 531 continue 532 } 533 534 refreshedDischarge, err := refreshDischargeMacaroon(httpClient, d) 535 if err != nil { 536 return nil, err 537 } 538 newDischarges[i] = refreshedDischarge 539 } 540 return newDischarges, nil 541 } 542 543 // refreshUser will refresh user discharge macaroon and update state 544 func (s *Store) refreshUser(user *auth.UserState) error { 545 if s.dauthCtx == nil { 546 return fmt.Errorf("user credentials need to be refreshed but update in place only supported in snapd") 547 } 548 newDischarges, err := refreshDischarges(s.client, user) 549 if err != nil { 550 return err 551 } 552 553 curUser, err := s.dauthCtx.UpdateUserAuth(user, newDischarges) 554 if err != nil { 555 return err 556 } 557 // update in place 558 *user = *curUser 559 560 return nil 561 } 562 563 // refreshDeviceSession will set or refresh the device session in the state 564 func (s *Store) refreshDeviceSession(device *auth.DeviceState) error { 565 if s.dauthCtx == nil { 566 return fmt.Errorf("internal error: no device and auth context") 567 } 568 569 s.sessionMu.Lock() 570 defer s.sessionMu.Unlock() 571 // check that no other goroutine has already got a new session etc... 572 device1, err := s.dauthCtx.Device() 573 if err != nil { 574 return err 575 } 576 // We can replace device with "device1" here because Device 577 // and UpdateDeviceAuth (and the underlying SetDevice) 578 // require/use the global state lock, so the reading/setting 579 // values have a total order, and device1 cannot come before 580 // device in that order. See also: 581 // https://github.com/snapcore/snapd/pull/6716#discussion_r277025834 582 if *device1 != *device { 583 // nothing to do 584 *device = *device1 585 return nil 586 } 587 588 nonce, err := requestStoreDeviceNonce(s.client, s.endpointURL(deviceNonceEndpPath, nil).String()) 589 if err != nil { 590 return err 591 } 592 593 devSessReqParams, err := s.dauthCtx.DeviceSessionRequestParams(nonce) 594 if err != nil { 595 return err 596 } 597 598 session, err := requestDeviceSession(s.client, s.endpointURL(deviceSessionEndpPath, nil).String(), devSessReqParams, device.SessionMacaroon) 599 if err != nil { 600 return err 601 } 602 603 curDevice, err := s.dauthCtx.UpdateDeviceAuth(device, session) 604 if err != nil { 605 return err 606 } 607 // update in place 608 *device = *curDevice 609 return nil 610 } 611 612 // EnsureDeviceSession makes sure the store has a device session available. 613 // Expects the store to have an AuthContext. 614 func (s *Store) EnsureDeviceSession() (*auth.DeviceState, error) { 615 if s.dauthCtx == nil { 616 return nil, fmt.Errorf("internal error: no authContext") 617 } 618 619 device, err := s.dauthCtx.Device() 620 if err != nil { 621 return nil, err 622 } 623 624 if device.SessionMacaroon != "" { 625 return device, nil 626 } 627 if device.Serial == "" { 628 return nil, ErrNoSerial 629 } 630 // we don't have a session yet but have a serial, try 631 // to get a session 632 err = s.refreshDeviceSession(device) 633 if err != nil { 634 return nil, err 635 } 636 return device, err 637 } 638 639 // authenticateDevice will add the store expected Macaroon X-Device-Authorization header for device 640 func authenticateDevice(r *http.Request, device *auth.DeviceState, apiLevel apiLevel) { 641 if device != nil && device.SessionMacaroon != "" { 642 r.Header.Set(hdrSnapDeviceAuthorization[apiLevel], fmt.Sprintf(`Macaroon root="%s"`, device.SessionMacaroon)) 643 } 644 } 645 646 func (s *Store) setStoreID(r *http.Request, apiLevel apiLevel) (customStore bool) { 647 storeID := s.fallbackStoreID 648 if s.dauthCtx != nil { 649 cand, err := s.dauthCtx.StoreID(storeID) 650 if err != nil { 651 logger.Debugf("cannot get store ID from state: %v", err) 652 } else { 653 storeID = cand 654 } 655 } 656 if storeID != "" { 657 r.Header.Set(hdrSnapDeviceStore[apiLevel], storeID) 658 return true 659 } 660 return false 661 } 662 663 type apiLevel int 664 665 const ( 666 apiV1Endps apiLevel = 0 // api/v1 endpoints 667 apiV2Endps apiLevel = 1 // v2 endpoints 668 ) 669 670 var ( 671 hdrSnapDeviceAuthorization = []string{"X-Device-Authorization", "Snap-Device-Authorization"} 672 hdrSnapDeviceStore = []string{"X-Ubuntu-Store", "Snap-Device-Store"} 673 hdrSnapDeviceSeries = []string{"X-Ubuntu-Series", "Snap-Device-Series"} 674 hdrSnapDeviceArchitecture = []string{"X-Ubuntu-Architecture", "Snap-Device-Architecture"} 675 hdrSnapClassic = []string{"X-Ubuntu-Classic", "Snap-Classic"} 676 ) 677 678 type deviceAuthNeed int 679 680 const ( 681 deviceAuthPreferred deviceAuthNeed = iota 682 deviceAuthCustomStoreOnly 683 ) 684 685 // requestOptions specifies parameters for store requests. 686 type requestOptions struct { 687 Method string 688 URL *url.URL 689 Accept string 690 ContentType string 691 APILevel apiLevel 692 ExtraHeaders map[string]string 693 Data []byte 694 695 // DeviceAuthNeed indicates the level of need to supply device 696 // authorization for this request, can be: 697 // - deviceAuthPreferred: should be provided if available 698 // - deviceAuthCustomStoreOnly: should be provided only in case 699 // of a custom store 700 DeviceAuthNeed deviceAuthNeed 701 } 702 703 func (r *requestOptions) addHeader(k, v string) { 704 if r.ExtraHeaders == nil { 705 r.ExtraHeaders = make(map[string]string) 706 } 707 r.ExtraHeaders[k] = v 708 } 709 710 func cancelled(ctx context.Context) bool { 711 select { 712 case <-ctx.Done(): 713 return true 714 default: 715 return false 716 } 717 } 718 719 var expectedCatalogPreamble = []interface{}{ 720 json.Delim('{'), 721 "_embedded", 722 json.Delim('{'), 723 "clickindex:package", 724 json.Delim('['), 725 } 726 727 type alias struct { 728 Name string `json:"name"` 729 } 730 731 type catalogItem struct { 732 Name string `json:"package_name"` 733 Version string `json:"version"` 734 Summary string `json:"summary"` 735 Aliases []alias `json:"aliases"` 736 Apps []string `json:"apps"` 737 } 738 739 type SnapAdder interface { 740 AddSnap(snapName, version, summary string, commands []string) error 741 } 742 743 func decodeCatalog(resp *http.Response, names io.Writer, db SnapAdder) error { 744 const what = "decode new commands catalog" 745 if resp.StatusCode != 200 { 746 return respToError(resp, what) 747 } 748 dec := json.NewDecoder(resp.Body) 749 for _, expectedToken := range expectedCatalogPreamble { 750 token, err := dec.Token() 751 if err != nil { 752 return err 753 } 754 if token != expectedToken { 755 return fmt.Errorf(what+": bad catalog preamble: expected %#v, got %#v", expectedToken, token) 756 } 757 } 758 759 for dec.More() { 760 var v catalogItem 761 if err := dec.Decode(&v); err != nil { 762 return fmt.Errorf(what+": %v", err) 763 } 764 if v.Name == "" { 765 continue 766 } 767 fmt.Fprintln(names, v.Name) 768 if len(v.Apps) == 0 { 769 continue 770 } 771 772 commands := make([]string, 0, len(v.Aliases)+len(v.Apps)) 773 774 for _, alias := range v.Aliases { 775 commands = append(commands, alias.Name) 776 } 777 for _, app := range v.Apps { 778 commands = append(commands, snap.JoinSnapApp(v.Name, app)) 779 } 780 781 if err := db.AddSnap(v.Name, v.Version, v.Summary, commands); err != nil { 782 return err 783 } 784 } 785 786 return nil 787 } 788 789 func decodeJSONBody(resp *http.Response, success interface{}, failure interface{}) error { 790 ok := (resp.StatusCode == 200 || resp.StatusCode == 201) 791 // always decode on success; decode failures only if body is not empty 792 if !ok && resp.ContentLength == 0 { 793 return nil 794 } 795 result := success 796 if !ok { 797 result = failure 798 } 799 if result != nil { 800 return json.NewDecoder(resp.Body).Decode(result) 801 } 802 return nil 803 } 804 805 // retryRequestDecodeJSON calls retryRequest and decodes the response into either success or failure. 806 func (s *Store) retryRequestDecodeJSON(ctx context.Context, reqOptions *requestOptions, user *auth.UserState, success interface{}, failure interface{}) (resp *http.Response, err error) { 807 return httputil.RetryRequest(reqOptions.URL.String(), func() (*http.Response, error) { 808 return s.doRequest(ctx, s.client, reqOptions, user) 809 }, func(resp *http.Response) error { 810 return decodeJSONBody(resp, success, failure) 811 }, defaultRetryStrategy) 812 } 813 814 // doRequest does an authenticated request to the store handling a potential macaroon refresh required if needed 815 func (s *Store) doRequest(ctx context.Context, client *http.Client, reqOptions *requestOptions, user *auth.UserState) (*http.Response, error) { 816 authRefreshes := 0 817 for { 818 req, err := s.newRequest(ctx, reqOptions, user) 819 if err != nil { 820 return nil, err 821 } 822 if ctx != nil { 823 req = req.WithContext(ctx) 824 } 825 826 resp, err := client.Do(req) 827 if err != nil { 828 return nil, err 829 } 830 831 wwwAuth := resp.Header.Get("WWW-Authenticate") 832 if resp.StatusCode == 401 && authRefreshes < 4 { 833 // 4 tries: 2 tries for each in case both user 834 // and device need refreshing 835 var refreshNeed authRefreshNeed 836 if user != nil && strings.Contains(wwwAuth, "needs_refresh=1") { 837 // refresh user 838 refreshNeed.user = true 839 } 840 if strings.Contains(wwwAuth, "refresh_device_session=1") { 841 // refresh device session 842 refreshNeed.device = true 843 } 844 if refreshNeed.needed() { 845 err := s.refreshAuth(user, refreshNeed) 846 if err != nil { 847 return nil, err 848 } 849 // close previous response and retry 850 resp.Body.Close() 851 authRefreshes++ 852 continue 853 } 854 } 855 856 return resp, err 857 } 858 } 859 860 type authRefreshNeed struct { 861 device bool 862 user bool 863 } 864 865 func (rn *authRefreshNeed) needed() bool { 866 return rn.device || rn.user 867 } 868 869 func (s *Store) refreshAuth(user *auth.UserState, need authRefreshNeed) error { 870 if need.user { 871 // refresh user 872 err := s.refreshUser(user) 873 if err != nil { 874 return err 875 } 876 } 877 if need.device { 878 // refresh device session 879 if s.dauthCtx == nil { 880 return fmt.Errorf("internal error: no device and auth context") 881 } 882 device, err := s.dauthCtx.Device() 883 if err != nil { 884 return err 885 } 886 887 err = s.refreshDeviceSession(device) 888 if err != nil { 889 return err 890 } 891 } 892 return nil 893 } 894 895 // build a new http.Request with headers for the store 896 func (s *Store) newRequest(ctx context.Context, reqOptions *requestOptions, user *auth.UserState) (*http.Request, error) { 897 var body io.Reader 898 if reqOptions.Data != nil { 899 body = bytes.NewBuffer(reqOptions.Data) 900 } 901 902 req, err := http.NewRequest(reqOptions.Method, reqOptions.URL.String(), body) 903 if err != nil { 904 return nil, err 905 } 906 907 customStore := s.setStoreID(req, reqOptions.APILevel) 908 909 if s.dauthCtx != nil && (customStore || reqOptions.DeviceAuthNeed != deviceAuthCustomStoreOnly) { 910 device, err := s.EnsureDeviceSession() 911 if err != nil && err != ErrNoSerial { 912 return nil, err 913 } 914 if err == ErrNoSerial { 915 // missing serial assertion, log and continue without device authentication 916 logger.Debugf("cannot set device session: %v", err) 917 } else { 918 authenticateDevice(req, device, reqOptions.APILevel) 919 } 920 } 921 922 // only set user authentication if user logged in to the store 923 if user.HasStoreAuth() { 924 authenticateUser(req, user) 925 } 926 927 req.Header.Set("User-Agent", httputil.UserAgent()) 928 req.Header.Set("Accept", reqOptions.Accept) 929 req.Header.Set(hdrSnapDeviceArchitecture[reqOptions.APILevel], s.architecture) 930 req.Header.Set(hdrSnapDeviceSeries[reqOptions.APILevel], s.series) 931 req.Header.Set(hdrSnapClassic[reqOptions.APILevel], strconv.FormatBool(release.OnClassic)) 932 if cua := ClientUserAgent(ctx); cua != "" { 933 req.Header.Set("Snap-Client-User-Agent", cua) 934 } 935 if reqOptions.APILevel == apiV1Endps { 936 req.Header.Set("X-Ubuntu-Wire-Protocol", UbuntuCoreWireProtocol) 937 } 938 939 if reqOptions.ContentType != "" { 940 req.Header.Set("Content-Type", reqOptions.ContentType) 941 } 942 943 for header, value := range reqOptions.ExtraHeaders { 944 req.Header.Set(header, value) 945 } 946 947 return req, nil 948 } 949 950 func (s *Store) cdnHeader() (string, error) { 951 if s.noCDN { 952 return "none", nil 953 } 954 955 if s.dauthCtx == nil { 956 return "", nil 957 } 958 959 // set Snap-CDN from cloud instance information 960 // if available 961 962 // TODO: do we want a more complex retry strategy 963 // where we first to send this header and if the 964 // operation fails that way to even get the connection 965 // then we retry without sending this? 966 967 cloudInfo, err := s.dauthCtx.CloudInfo() 968 if err != nil { 969 return "", err 970 } 971 972 if cloudInfo != nil { 973 cdnParams := []string{fmt.Sprintf("cloud-name=%q", cloudInfo.Name)} 974 if cloudInfo.Region != "" { 975 cdnParams = append(cdnParams, fmt.Sprintf("region=%q", cloudInfo.Region)) 976 } 977 if cloudInfo.AvailabilityZone != "" { 978 cdnParams = append(cdnParams, fmt.Sprintf("availability-zone=%q", cloudInfo.AvailabilityZone)) 979 } 980 981 return strings.Join(cdnParams, " "), nil 982 } 983 984 return "", nil 985 } 986 987 func (s *Store) extractSuggestedCurrency(resp *http.Response) { 988 suggestedCurrency := resp.Header.Get("X-Suggested-Currency") 989 990 if suggestedCurrency != "" { 991 s.mu.Lock() 992 s.suggestedCurrency = suggestedCurrency 993 s.mu.Unlock() 994 } 995 } 996 997 // ordersResult encapsulates the order data sent to us from the software center agent. 998 // 999 // { 1000 // "orders": [ 1001 // { 1002 // "snap_id": "abcd1234efgh5678ijkl9012", 1003 // "currency": "USD", 1004 // "amount": "2.99", 1005 // "state": "Complete", 1006 // "refundable_until": null, 1007 // "purchase_date": "2016-09-20T15:00:00+00:00" 1008 // }, 1009 // { 1010 // "snap_id": "abcd1234efgh5678ijkl9012", 1011 // "currency": null, 1012 // "amount": null, 1013 // "state": "Complete", 1014 // "refundable_until": null, 1015 // "purchase_date": "2016-09-20T15:00:00+00:00" 1016 // } 1017 // ] 1018 // } 1019 type ordersResult struct { 1020 Orders []*order `json:"orders"` 1021 } 1022 1023 type order struct { 1024 SnapID string `json:"snap_id"` 1025 Currency string `json:"currency"` 1026 Amount string `json:"amount"` 1027 State string `json:"state"` 1028 RefundableUntil string `json:"refundable_until"` 1029 PurchaseDate string `json:"purchase_date"` 1030 } 1031 1032 // decorateOrders sets the MustBuy property of each snap in the given list according to the user's known orders. 1033 func (s *Store) decorateOrders(snaps []*snap.Info, user *auth.UserState) error { 1034 // Mark every non-free snap as must buy until we know better. 1035 hasPriced := false 1036 for _, info := range snaps { 1037 if info.Paid { 1038 info.MustBuy = true 1039 hasPriced = true 1040 } 1041 } 1042 1043 if user == nil { 1044 return nil 1045 } 1046 1047 if !hasPriced { 1048 return nil 1049 } 1050 1051 var err error 1052 1053 reqOptions := &requestOptions{ 1054 Method: "GET", 1055 URL: s.endpointURL(ordersEndpPath, nil), 1056 Accept: jsonContentType, 1057 } 1058 var result ordersResult 1059 resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, &result, nil) 1060 if err != nil { 1061 return err 1062 } 1063 1064 if resp.StatusCode == 401 { 1065 // TODO handle token expiry and refresh 1066 return ErrInvalidCredentials 1067 } 1068 if resp.StatusCode != 200 { 1069 return respToError(resp, "obtain known orders from store") 1070 } 1071 1072 // Make a map of the IDs of bought snaps 1073 bought := make(map[string]bool) 1074 for _, order := range result.Orders { 1075 bought[order.SnapID] = true 1076 } 1077 1078 for _, info := range snaps { 1079 info.MustBuy = mustBuy(info.Paid, bought[info.SnapID]) 1080 } 1081 1082 return nil 1083 } 1084 1085 // mustBuy determines if a snap requires a payment, based on if it is non-free and if the user has already bought it 1086 func mustBuy(paid bool, bought bool) bool { 1087 if !paid { 1088 // If the snap is free, then it doesn't need buying 1089 return false 1090 } 1091 1092 return !bought 1093 } 1094 1095 // A SnapSpec describes a single snap wanted from SnapInfo 1096 type SnapSpec struct { 1097 Name string 1098 } 1099 1100 // SnapInfo returns the snap.Info for the store-hosted snap matching the given spec, or an error. 1101 func (s *Store) SnapInfo(ctx context.Context, snapSpec SnapSpec, user *auth.UserState) (*snap.Info, error) { 1102 query := url.Values{} 1103 query.Set("fields", strings.Join(s.infoFields, ",")) 1104 query.Set("architecture", s.architecture) 1105 1106 u := s.endpointURL(path.Join(snapInfoEndpPath, snapSpec.Name), query) 1107 reqOptions := &requestOptions{ 1108 Method: "GET", 1109 URL: u, 1110 APILevel: apiV2Endps, 1111 } 1112 1113 var remote storeInfo 1114 resp, err := s.retryRequestDecodeJSON(ctx, reqOptions, user, &remote, nil) 1115 if err != nil { 1116 return nil, err 1117 } 1118 1119 // check statusCode 1120 switch resp.StatusCode { 1121 case 200: 1122 // OK 1123 case 404: 1124 return nil, ErrSnapNotFound 1125 default: 1126 msg := fmt.Sprintf("get details for snap %q", snapSpec.Name) 1127 return nil, respToError(resp, msg) 1128 } 1129 1130 info, err := infoFromStoreInfo(&remote) 1131 if err != nil { 1132 return nil, err 1133 } 1134 1135 err = s.decorateOrders([]*snap.Info{info}, user) 1136 if err != nil { 1137 logger.Noticef("cannot get user orders: %v", err) 1138 } 1139 1140 s.extractSuggestedCurrency(resp) 1141 1142 return info, nil 1143 } 1144 1145 // A Search is what you do in order to Find something 1146 type Search struct { 1147 // Query is a term to search by or a prefix (if Prefix is true) 1148 Query string 1149 Prefix bool 1150 1151 CommonID string 1152 1153 Section string 1154 Private bool 1155 Scope string 1156 } 1157 1158 // Find finds (installable) snaps from the store, matching the 1159 // given Search. 1160 func (s *Store) Find(ctx context.Context, search *Search, user *auth.UserState) ([]*snap.Info, error) { 1161 if search.Private && user == nil { 1162 return nil, ErrUnauthenticated 1163 } 1164 1165 searchTerm := strings.TrimSpace(search.Query) 1166 1167 // these characters might have special meaning on the search 1168 // server, and don't form part of a reasonable search, so 1169 // abort if they're included. 1170 // 1171 // "-" might also be special on the server, but it's also a 1172 // valid part of a package name, so we let it pass 1173 if strings.ContainsAny(searchTerm, `+=&|><!(){}[]^"~*?:\/`) { 1174 return nil, ErrBadQuery 1175 } 1176 1177 q := s.defaultSnapQuery() 1178 1179 if search.Private { 1180 q.Set("private", "true") 1181 } 1182 1183 if search.Prefix { 1184 q.Set("name", searchTerm) 1185 } else { 1186 if search.CommonID != "" { 1187 q.Set("common_id", search.CommonID) 1188 } 1189 if searchTerm != "" { 1190 q.Set("q", searchTerm) 1191 } 1192 } 1193 if search.Section != "" { 1194 q.Set("section", search.Section) 1195 } 1196 if search.Scope != "" { 1197 q.Set("scope", search.Scope) 1198 } 1199 1200 if release.OnClassic { 1201 q.Set("confinement", "strict,classic") 1202 } else { 1203 q.Set("confinement", "strict") 1204 } 1205 1206 u := s.endpointURL(searchEndpPath, q) 1207 reqOptions := &requestOptions{ 1208 Method: "GET", 1209 URL: u, 1210 Accept: halJsonContentType, 1211 } 1212 1213 var searchData searchResults 1214 resp, err := s.retryRequestDecodeJSON(ctx, reqOptions, user, &searchData, nil) 1215 if err != nil { 1216 return nil, err 1217 } 1218 1219 if resp.StatusCode != 200 { 1220 return nil, respToError(resp, "search") 1221 } 1222 1223 if ct := resp.Header.Get("Content-Type"); ct != halJsonContentType { 1224 return nil, fmt.Errorf("received an unexpected content type (%q) when trying to search via %q", ct, resp.Request.URL) 1225 } 1226 1227 snaps := make([]*snap.Info, len(searchData.Payload.Packages)) 1228 for i, pkg := range searchData.Payload.Packages { 1229 snaps[i] = infoFromRemote(pkg) 1230 } 1231 1232 err = s.decorateOrders(snaps, user) 1233 if err != nil { 1234 logger.Noticef("cannot get user orders: %v", err) 1235 } 1236 1237 s.extractSuggestedCurrency(resp) 1238 1239 return snaps, nil 1240 } 1241 1242 // Sections retrieves the list of available store sections. 1243 func (s *Store) Sections(ctx context.Context, user *auth.UserState) ([]string, error) { 1244 reqOptions := &requestOptions{ 1245 Method: "GET", 1246 URL: s.endpointURL(sectionsEndpPath, nil), 1247 Accept: halJsonContentType, 1248 DeviceAuthNeed: deviceAuthCustomStoreOnly, 1249 } 1250 1251 var sectionData sectionResults 1252 resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, §ionData, nil) 1253 if err != nil { 1254 return nil, err 1255 } 1256 1257 if resp.StatusCode != 200 { 1258 return nil, respToError(resp, "sections") 1259 } 1260 1261 if ct := resp.Header.Get("Content-Type"); ct != halJsonContentType { 1262 return nil, fmt.Errorf("received an unexpected content type (%q) when trying to retrieve the sections via %q", ct, resp.Request.URL) 1263 } 1264 1265 var sectionNames []string 1266 for _, s := range sectionData.Payload.Sections { 1267 sectionNames = append(sectionNames, s.Name) 1268 } 1269 1270 return sectionNames, nil 1271 } 1272 1273 // WriteCatalogs queries the "commands" endpoint and writes the 1274 // command names into the given io.Writer. 1275 func (s *Store) WriteCatalogs(ctx context.Context, names io.Writer, adder SnapAdder) error { 1276 u := *s.endpointURL(commandsEndpPath, nil) 1277 1278 q := u.Query() 1279 if release.OnClassic { 1280 q.Set("confinement", "strict,classic") 1281 } else { 1282 q.Set("confinement", "strict") 1283 } 1284 1285 u.RawQuery = q.Encode() 1286 reqOptions := &requestOptions{ 1287 Method: "GET", 1288 URL: &u, 1289 Accept: halJsonContentType, 1290 DeviceAuthNeed: deviceAuthCustomStoreOnly, 1291 } 1292 1293 // do not log body for catalog updates (its huge) 1294 client := httputil.NewHTTPClient(&httputil.ClientOptions{ 1295 MayLogBody: false, 1296 Timeout: 10 * time.Second, 1297 Proxy: s.proxy, 1298 }) 1299 doRequest := func() (*http.Response, error) { 1300 return s.doRequest(ctx, client, reqOptions, nil) 1301 } 1302 readResponse := func(resp *http.Response) error { 1303 return decodeCatalog(resp, names, adder) 1304 } 1305 1306 resp, err := httputil.RetryRequest(u.String(), doRequest, readResponse, defaultRetryStrategy) 1307 if err != nil { 1308 return err 1309 } 1310 if resp.StatusCode != 200 { 1311 return respToError(resp, "refresh commands catalog") 1312 } 1313 1314 return nil 1315 } 1316 1317 func findRev(needle snap.Revision, haystack []snap.Revision) bool { 1318 for _, r := range haystack { 1319 if needle == r { 1320 return true 1321 } 1322 } 1323 return false 1324 } 1325 1326 type HashError struct { 1327 name string 1328 sha3_384 string 1329 targetSha3_384 string 1330 } 1331 1332 func (e HashError) Error() string { 1333 return fmt.Sprintf("sha3-384 mismatch for %q: got %s but expected %s", e.name, e.sha3_384, e.targetSha3_384) 1334 } 1335 1336 type DownloadOptions struct { 1337 RateLimit int64 1338 IsAutoRefresh bool 1339 LeavePartialOnError bool 1340 } 1341 1342 // Download downloads the snap addressed by download info and returns its 1343 // filename. 1344 // The file is saved in temporary storage, and should be removed 1345 // after use to prevent the disk from running out of space. 1346 func (s *Store) Download(ctx context.Context, name string, targetPath string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState, dlOpts *DownloadOptions) error { 1347 if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { 1348 return err 1349 } 1350 1351 if err := s.cacher.Get(downloadInfo.Sha3_384, targetPath); err == nil { 1352 logger.Debugf("Cache hit for SHA3_384 …%.5s.", downloadInfo.Sha3_384) 1353 return nil 1354 } 1355 1356 if useDeltas() { 1357 logger.Debugf("Available deltas returned by store: %v", downloadInfo.Deltas) 1358 1359 if len(downloadInfo.Deltas) == 1 { 1360 err := s.downloadAndApplyDelta(name, targetPath, downloadInfo, pbar, user, dlOpts) 1361 if err == nil { 1362 return nil 1363 } 1364 // We revert to normal downloads if there is any error. 1365 logger.Noticef("Cannot download or apply deltas for %s: %v", name, err) 1366 } 1367 } 1368 1369 partialPath := targetPath + ".partial" 1370 w, err := os.OpenFile(partialPath, os.O_RDWR|os.O_CREATE, 0600) 1371 if err != nil { 1372 return err 1373 } 1374 resume, err := w.Seek(0, os.SEEK_END) 1375 if err != nil { 1376 return err 1377 } 1378 defer func() { 1379 fi, _ := w.Stat() 1380 if cerr := w.Close(); cerr != nil && err == nil { 1381 err = cerr 1382 } 1383 if err == nil { 1384 return 1385 } 1386 if dlOpts == nil || !dlOpts.LeavePartialOnError || fi == nil || fi.Size() == 0 { 1387 os.Remove(w.Name()) 1388 } 1389 }() 1390 if resume > 0 { 1391 logger.Debugf("Resuming download of %q at %d.", partialPath, resume) 1392 } else { 1393 logger.Debugf("Starting download of %q.", partialPath) 1394 } 1395 1396 authAvail, err := s.authAvailable(user) 1397 if err != nil { 1398 return err 1399 } 1400 1401 url := downloadInfo.AnonDownloadURL 1402 if url == "" || authAvail { 1403 url = downloadInfo.DownloadURL 1404 } 1405 1406 if downloadInfo.Size == 0 || resume < downloadInfo.Size { 1407 err = download(ctx, name, downloadInfo.Sha3_384, url, user, s, w, resume, pbar, dlOpts) 1408 if err != nil { 1409 logger.Debugf("download of %q failed: %#v", url, err) 1410 } 1411 } else { 1412 // we're done! check the hash though 1413 h := crypto.SHA3_384.New() 1414 if _, err := w.Seek(0, os.SEEK_SET); err != nil { 1415 return err 1416 } 1417 if _, err := io.Copy(h, w); err != nil { 1418 return err 1419 } 1420 actualSha3 := fmt.Sprintf("%x", h.Sum(nil)) 1421 if downloadInfo.Sha3_384 != actualSha3 { 1422 err = HashError{name, actualSha3, downloadInfo.Sha3_384} 1423 } 1424 } 1425 // If hashsum is incorrect retry once 1426 if _, ok := err.(HashError); ok { 1427 logger.Debugf("Hashsum error on download: %v", err.Error()) 1428 logger.Debugf("Truncating and trying again from scratch.") 1429 err = w.Truncate(0) 1430 if err != nil { 1431 return err 1432 } 1433 _, err = w.Seek(0, os.SEEK_SET) 1434 if err != nil { 1435 return err 1436 } 1437 err = download(ctx, name, downloadInfo.Sha3_384, url, user, s, w, 0, pbar, nil) 1438 if err != nil { 1439 logger.Debugf("download of %q failed: %#v", url, err) 1440 } 1441 } 1442 1443 if err != nil { 1444 return err 1445 } 1446 1447 if err := os.Rename(w.Name(), targetPath); err != nil { 1448 return err 1449 } 1450 1451 if err := w.Sync(); err != nil { 1452 return err 1453 } 1454 1455 return s.cacher.Put(downloadInfo.Sha3_384, targetPath) 1456 } 1457 1458 func downloadReqOpts(storeURL *url.URL, cdnHeader string, opts *DownloadOptions) *requestOptions { 1459 reqOptions := requestOptions{ 1460 Method: "GET", 1461 URL: storeURL, 1462 ExtraHeaders: map[string]string{}, 1463 // FIXME: use the new headers? with 1464 // APILevel: apiV2Endps, 1465 } 1466 if cdnHeader != "" { 1467 reqOptions.ExtraHeaders["Snap-CDN"] = cdnHeader 1468 } 1469 if opts != nil && opts.IsAutoRefresh { 1470 reqOptions.ExtraHeaders["Snap-Refresh-Reason"] = "scheduled" 1471 } 1472 1473 return &reqOptions 1474 } 1475 1476 var ratelimitReader = ratelimit.Reader 1477 1478 var download = downloadImpl 1479 1480 // download writes an http.Request showing a progress.Meter 1481 func downloadImpl(ctx context.Context, name, sha3_384, downloadURL string, user *auth.UserState, s *Store, w io.ReadWriteSeeker, resume int64, pbar progress.Meter, dlOpts *DownloadOptions) error { 1482 if dlOpts == nil { 1483 dlOpts = &DownloadOptions{} 1484 } 1485 1486 storeURL, err := url.Parse(downloadURL) 1487 if err != nil { 1488 return err 1489 } 1490 1491 cdnHeader, err := s.cdnHeader() 1492 if err != nil { 1493 return err 1494 } 1495 1496 var finalErr error 1497 var dlSize float64 1498 startTime := time.Now() 1499 for attempt := retry.Start(downloadRetryStrategy, nil); attempt.Next(); { 1500 reqOptions := downloadReqOpts(storeURL, cdnHeader, dlOpts) 1501 1502 httputil.MaybeLogRetryAttempt(reqOptions.URL.String(), attempt, startTime) 1503 1504 h := crypto.SHA3_384.New() 1505 1506 if resume > 0 { 1507 reqOptions.ExtraHeaders["Range"] = fmt.Sprintf("bytes=%d-", resume) 1508 // seed the sha3 with the already local file 1509 if _, err := w.Seek(0, os.SEEK_SET); err != nil { 1510 return err 1511 } 1512 n, err := io.Copy(h, w) 1513 if err != nil { 1514 return err 1515 } 1516 if n != resume { 1517 return fmt.Errorf("resume offset wrong: %d != %d", resume, n) 1518 } 1519 } 1520 1521 if cancelled(ctx) { 1522 return fmt.Errorf("The download has been cancelled: %s", ctx.Err()) 1523 } 1524 var resp *http.Response 1525 resp, finalErr = s.doRequest(ctx, httputil.NewHTTPClient(&httputil.ClientOptions{Proxy: s.proxy}), reqOptions, user) 1526 1527 if cancelled(ctx) { 1528 return fmt.Errorf("The download has been cancelled: %s", ctx.Err()) 1529 } 1530 if finalErr != nil { 1531 if httputil.ShouldRetryError(attempt, finalErr) { 1532 continue 1533 } 1534 break 1535 } 1536 1537 if httputil.ShouldRetryHttpResponse(attempt, resp) { 1538 resp.Body.Close() 1539 continue 1540 } 1541 1542 defer resp.Body.Close() 1543 1544 switch resp.StatusCode { 1545 case 200, 206: // OK, Partial Content 1546 case 402: // Payment Required 1547 1548 return fmt.Errorf("please buy %s before installing it.", name) 1549 default: 1550 return &DownloadError{Code: resp.StatusCode, URL: resp.Request.URL} 1551 } 1552 1553 if pbar == nil { 1554 pbar = progress.Null 1555 } 1556 dlSize = float64(resp.ContentLength) 1557 pbar.Start(name, dlSize) 1558 mw := io.MultiWriter(w, h, pbar) 1559 var limiter io.Reader 1560 limiter = resp.Body 1561 if limit := dlOpts.RateLimit; limit > 0 { 1562 bucket := ratelimit.NewBucketWithRate(float64(limit), 2*limit) 1563 limiter = ratelimitReader(resp.Body, bucket) 1564 } 1565 _, finalErr = io.Copy(mw, limiter) 1566 pbar.Finished() 1567 if finalErr != nil { 1568 if httputil.ShouldRetryError(attempt, finalErr) { 1569 // error while downloading should resume 1570 var seekerr error 1571 resume, seekerr = w.Seek(0, os.SEEK_END) 1572 if seekerr == nil { 1573 continue 1574 } 1575 // if seek failed, then don't retry end return the original error 1576 } 1577 break 1578 } 1579 1580 if cancelled(ctx) { 1581 return fmt.Errorf("The download has been cancelled: %s", ctx.Err()) 1582 } 1583 1584 actualSha3 := fmt.Sprintf("%x", h.Sum(nil)) 1585 if sha3_384 != "" && sha3_384 != actualSha3 { 1586 finalErr = HashError{name, actualSha3, sha3_384} 1587 } 1588 break 1589 } 1590 if finalErr == nil { 1591 // not using quantity.FormatFoo as this is just for debug 1592 dt := time.Since(startTime) 1593 r := dlSize / dt.Seconds() 1594 var p rune 1595 for _, p = range " kMGTPEZY" { 1596 if r < 1000 { 1597 break 1598 } 1599 r /= 1000 1600 } 1601 1602 logger.Debugf("Download succeeded in %.03fs (%.0f%cB/s).", dt.Seconds(), r, p) 1603 } 1604 return finalErr 1605 } 1606 1607 // DownloadStream will copy the snap from the request to the io.Reader 1608 func (s *Store) DownloadStream(ctx context.Context, name string, downloadInfo *snap.DownloadInfo, user *auth.UserState) (io.ReadCloser, error) { 1609 if path := s.cacher.GetPath(downloadInfo.Sha3_384); path != "" { 1610 logger.Debugf("Cache hit for SHA3_384 …%.5s.", downloadInfo.Sha3_384) 1611 file, err := os.OpenFile(path, os.O_RDONLY, 0600) 1612 if err != nil { 1613 return nil, err 1614 } 1615 return file, nil 1616 } 1617 1618 authAvail, err := s.authAvailable(user) 1619 if err != nil { 1620 return nil, err 1621 } 1622 1623 downloadURL := downloadInfo.AnonDownloadURL 1624 if downloadURL == "" || authAvail { 1625 downloadURL = downloadInfo.DownloadURL 1626 } 1627 1628 storeURL, err := url.Parse(downloadURL) 1629 if err != nil { 1630 return nil, err 1631 } 1632 1633 cdnHeader, err := s.cdnHeader() 1634 if err != nil { 1635 return nil, err 1636 } 1637 1638 resp, err := doDownloadReq(ctx, storeURL, cdnHeader, s, user) 1639 if err != nil { 1640 return nil, err 1641 } 1642 return resp.Body, nil 1643 } 1644 1645 var doDownloadReq = doDowloadReqImpl 1646 1647 func doDowloadReqImpl(ctx context.Context, storeURL *url.URL, cdnHeader string, s *Store, user *auth.UserState) (*http.Response, error) { 1648 reqOptions := downloadReqOpts(storeURL, cdnHeader, nil) 1649 return s.doRequest(ctx, httputil.NewHTTPClient(&httputil.ClientOptions{Proxy: s.proxy}), reqOptions, user) 1650 } 1651 1652 // downloadDelta downloads the delta for the preferred format, returning the path. 1653 func (s *Store) downloadDelta(deltaName string, downloadInfo *snap.DownloadInfo, w io.ReadWriteSeeker, pbar progress.Meter, user *auth.UserState, dlOpts *DownloadOptions) error { 1654 1655 if len(downloadInfo.Deltas) != 1 { 1656 return errors.New("store returned more than one download delta") 1657 } 1658 1659 deltaInfo := downloadInfo.Deltas[0] 1660 1661 if deltaInfo.Format != s.deltaFormat { 1662 return fmt.Errorf("store returned unsupported delta format %q (only xdelta3 currently)", deltaInfo.Format) 1663 } 1664 1665 authAvail, err := s.authAvailable(user) 1666 if err != nil { 1667 return err 1668 } 1669 1670 url := deltaInfo.AnonDownloadURL 1671 if url == "" || authAvail { 1672 url = deltaInfo.DownloadURL 1673 } 1674 1675 return download(context.TODO(), deltaName, deltaInfo.Sha3_384, url, user, s, w, 0, pbar, dlOpts) 1676 } 1677 1678 func getXdelta3Cmd(args ...string) (*exec.Cmd, error) { 1679 switch { 1680 case osutil.ExecutableExists("xdelta3"): 1681 return exec.Command("xdelta3", args...), nil 1682 case osutil.FileExists(filepath.Join(dirs.SnapMountDir, "/core/current/usr/bin/xdelta3")): 1683 return cmdutil.CommandFromSystemSnap("/usr/bin/xdelta3", args...) 1684 } 1685 return nil, fmt.Errorf("cannot find xdelta3 binary in PATH or core snap") 1686 } 1687 1688 // applyDelta generates a target snap from a previously downloaded snap and a downloaded delta. 1689 var applyDelta = func(name string, deltaPath string, deltaInfo *snap.DeltaInfo, targetPath string, targetSha3_384 string) error { 1690 snapBase := fmt.Sprintf("%s_%d.snap", name, deltaInfo.FromRevision) 1691 snapPath := filepath.Join(dirs.SnapBlobDir, snapBase) 1692 1693 if !osutil.FileExists(snapPath) { 1694 return fmt.Errorf("snap %q revision %d not found at %s", name, deltaInfo.FromRevision, snapPath) 1695 } 1696 1697 if deltaInfo.Format != "xdelta3" { 1698 return fmt.Errorf("cannot apply unsupported delta format %q (only xdelta3 currently)", deltaInfo.Format) 1699 } 1700 1701 partialTargetPath := targetPath + ".partial" 1702 1703 xdelta3Args := []string{"-d", "-s", snapPath, deltaPath, partialTargetPath} 1704 cmd, err := getXdelta3Cmd(xdelta3Args...) 1705 if err != nil { 1706 return err 1707 } 1708 1709 if err := cmd.Run(); err != nil { 1710 if err := os.Remove(partialTargetPath); err != nil { 1711 logger.Noticef("failed to remove partial delta target %q: %s", partialTargetPath, err) 1712 } 1713 return err 1714 } 1715 1716 if err := os.Chmod(partialTargetPath, 0600); err != nil { 1717 return err 1718 } 1719 1720 bsha3_384, _, err := osutil.FileDigest(partialTargetPath, crypto.SHA3_384) 1721 if err != nil { 1722 return err 1723 } 1724 sha3_384 := fmt.Sprintf("%x", bsha3_384) 1725 if targetSha3_384 != "" && sha3_384 != targetSha3_384 { 1726 if err := os.Remove(partialTargetPath); err != nil { 1727 logger.Noticef("failed to remove partial delta target %q: %s", partialTargetPath, err) 1728 } 1729 return HashError{name, sha3_384, targetSha3_384} 1730 } 1731 1732 if err := os.Rename(partialTargetPath, targetPath); err != nil { 1733 return osutil.CopyFile(partialTargetPath, targetPath, 0) 1734 } 1735 1736 return nil 1737 } 1738 1739 // downloadAndApplyDelta downloads and then applies the delta to the current snap. 1740 func (s *Store) downloadAndApplyDelta(name, targetPath string, downloadInfo *snap.DownloadInfo, pbar progress.Meter, user *auth.UserState, dlOpts *DownloadOptions) error { 1741 deltaInfo := &downloadInfo.Deltas[0] 1742 1743 deltaPath := fmt.Sprintf("%s.%s-%d-to-%d.partial", targetPath, deltaInfo.Format, deltaInfo.FromRevision, deltaInfo.ToRevision) 1744 deltaName := fmt.Sprintf(i18n.G("%s (delta)"), name) 1745 1746 w, err := os.OpenFile(deltaPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0600) 1747 if err != nil { 1748 return err 1749 } 1750 defer func() { 1751 if cerr := w.Close(); cerr != nil && err == nil { 1752 err = cerr 1753 } 1754 os.Remove(deltaPath) 1755 }() 1756 1757 err = s.downloadDelta(deltaName, downloadInfo, w, pbar, user, dlOpts) 1758 if err != nil { 1759 return err 1760 } 1761 1762 logger.Debugf("Successfully downloaded delta for %q at %s", name, deltaPath) 1763 if err := applyDelta(name, deltaPath, deltaInfo, targetPath, downloadInfo.Sha3_384); err != nil { 1764 return err 1765 } 1766 1767 logger.Debugf("Successfully applied delta for %q at %s, saving %d bytes.", name, deltaPath, downloadInfo.Size-deltaInfo.Size) 1768 return nil 1769 } 1770 1771 type assertionSvcError struct { 1772 Status int `json:"status"` 1773 Type string `json:"type"` 1774 Title string `json:"title"` 1775 Detail string `json:"detail"` 1776 } 1777 1778 // Assertion retrivies the assertion for the given type and primary key. 1779 func (s *Store) Assertion(assertType *asserts.AssertionType, primaryKey []string, user *auth.UserState) (asserts.Assertion, error) { 1780 v := url.Values{} 1781 v.Set("max-format", strconv.Itoa(assertType.MaxSupportedFormat())) 1782 u := s.assertionsEndpointURL(path.Join(assertType.Name, path.Join(primaryKey...)), v) 1783 1784 reqOptions := &requestOptions{ 1785 Method: "GET", 1786 URL: u, 1787 Accept: asserts.MediaType, 1788 } 1789 1790 var asrt asserts.Assertion 1791 1792 resp, err := httputil.RetryRequest(reqOptions.URL.String(), func() (*http.Response, error) { 1793 return s.doRequest(context.TODO(), s.client, reqOptions, user) 1794 }, func(resp *http.Response) error { 1795 var e error 1796 if resp.StatusCode == 200 { 1797 // decode assertion 1798 dec := asserts.NewDecoder(resp.Body) 1799 asrt, e = dec.Decode() 1800 } else { 1801 contentType := resp.Header.Get("Content-Type") 1802 if contentType == jsonContentType || contentType == "application/problem+json" { 1803 var svcErr assertionSvcError 1804 dec := json.NewDecoder(resp.Body) 1805 if e = dec.Decode(&svcErr); e != nil { 1806 return fmt.Errorf("cannot decode assertion service error with HTTP status code %d: %v", resp.StatusCode, e) 1807 } 1808 if svcErr.Status == 404 { 1809 // best-effort 1810 headers, _ := asserts.HeadersFromPrimaryKey(assertType, primaryKey) 1811 return &asserts.NotFoundError{ 1812 Type: assertType, 1813 Headers: headers, 1814 } 1815 } 1816 return fmt.Errorf("assertion service error: [%s] %q", svcErr.Title, svcErr.Detail) 1817 } 1818 } 1819 return e 1820 }, defaultRetryStrategy) 1821 1822 if err != nil { 1823 return nil, err 1824 } 1825 1826 if resp.StatusCode != 200 { 1827 return nil, respToError(resp, "fetch assertion") 1828 } 1829 1830 return asrt, err 1831 } 1832 1833 // SuggestedCurrency retrieves the cached value for the store's suggested currency 1834 func (s *Store) SuggestedCurrency() string { 1835 s.mu.Lock() 1836 defer s.mu.Unlock() 1837 1838 if s.suggestedCurrency == "" { 1839 return "USD" 1840 } 1841 return s.suggestedCurrency 1842 } 1843 1844 // orderInstruction holds data sent to the store for orders. 1845 type orderInstruction struct { 1846 SnapID string `json:"snap_id"` 1847 Amount string `json:"amount,omitempty"` 1848 Currency string `json:"currency,omitempty"` 1849 } 1850 1851 type storeError struct { 1852 Code string `json:"code"` 1853 Message string `json:"message"` 1854 } 1855 1856 func (s *storeError) Error() string { 1857 return s.Message 1858 } 1859 1860 type storeErrors struct { 1861 Errors []*storeError `json:"error_list"` 1862 } 1863 1864 func (s *storeErrors) Code() string { 1865 if len(s.Errors) == 0 { 1866 return "" 1867 } 1868 return s.Errors[0].Code 1869 } 1870 1871 func (s *storeErrors) Error() string { 1872 if len(s.Errors) == 0 { 1873 return "internal error: empty store error used as an actual error" 1874 } 1875 return s.Errors[0].Error() 1876 } 1877 1878 func buyOptionError(message string) (*client.BuyResult, error) { 1879 return nil, fmt.Errorf("cannot buy snap: %s", message) 1880 } 1881 1882 // Buy sends a buy request for the specified snap. 1883 // Returns the state of the order: Complete, Cancelled. 1884 func (s *Store) Buy(options *client.BuyOptions, user *auth.UserState) (*client.BuyResult, error) { 1885 if options.SnapID == "" { 1886 return buyOptionError("snap ID missing") 1887 } 1888 if options.Price <= 0 { 1889 return buyOptionError("invalid expected price") 1890 } 1891 if options.Currency == "" { 1892 return buyOptionError("currency missing") 1893 } 1894 if user == nil { 1895 return nil, ErrUnauthenticated 1896 } 1897 1898 instruction := orderInstruction{ 1899 SnapID: options.SnapID, 1900 Amount: fmt.Sprintf("%.2f", options.Price), 1901 Currency: options.Currency, 1902 } 1903 1904 jsonData, err := json.Marshal(instruction) 1905 if err != nil { 1906 return nil, err 1907 } 1908 1909 reqOptions := &requestOptions{ 1910 Method: "POST", 1911 URL: s.endpointURL(buyEndpPath, nil), 1912 Accept: jsonContentType, 1913 ContentType: jsonContentType, 1914 Data: jsonData, 1915 } 1916 1917 var orderDetails order 1918 var errorInfo storeErrors 1919 resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, &orderDetails, &errorInfo) 1920 if err != nil { 1921 return nil, err 1922 } 1923 1924 switch resp.StatusCode { 1925 case 200, 201: 1926 // user already ordered or order successful 1927 if orderDetails.State == "Cancelled" { 1928 return buyOptionError("payment cancelled") 1929 } 1930 1931 return &client.BuyResult{ 1932 State: orderDetails.State, 1933 }, nil 1934 case 400: 1935 // Invalid price was specified, etc. 1936 return buyOptionError(fmt.Sprintf("bad request: %v", errorInfo.Error())) 1937 case 403: 1938 // Customer account not set up for purchases. 1939 switch errorInfo.Code() { 1940 case "no-payment-methods": 1941 return nil, ErrNoPaymentMethods 1942 case "tos-not-accepted": 1943 return nil, ErrTOSNotAccepted 1944 } 1945 return buyOptionError(fmt.Sprintf("permission denied: %v", errorInfo.Error())) 1946 case 404: 1947 // Likely because customer account or snap ID doesn't exist. 1948 return buyOptionError(fmt.Sprintf("server says not found: %v", errorInfo.Error())) 1949 case 402: // Payment Required 1950 // Payment failed for some reason. 1951 return nil, ErrPaymentDeclined 1952 case 401: 1953 // TODO handle token expiry and refresh 1954 return nil, ErrInvalidCredentials 1955 default: 1956 return nil, respToError(resp, fmt.Sprintf("buy snap: %v", errorInfo)) 1957 } 1958 } 1959 1960 type storeCustomer struct { 1961 LatestTOSDate string `json:"latest_tos_date"` 1962 AcceptedTOSDate string `json:"accepted_tos_date"` 1963 LatestTOSAccepted bool `json:"latest_tos_accepted"` 1964 HasPaymentMethod bool `json:"has_payment_method"` 1965 } 1966 1967 // ReadyToBuy returns nil if the user's account has accepted T&Cs and has a payment method registered, and an error otherwise 1968 func (s *Store) ReadyToBuy(user *auth.UserState) error { 1969 if user == nil { 1970 return ErrUnauthenticated 1971 } 1972 1973 reqOptions := &requestOptions{ 1974 Method: "GET", 1975 URL: s.endpointURL(customersMeEndpPath, nil), 1976 Accept: jsonContentType, 1977 } 1978 1979 var customer storeCustomer 1980 var errors storeErrors 1981 resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, &customer, &errors) 1982 if err != nil { 1983 return err 1984 } 1985 1986 switch resp.StatusCode { 1987 case 200: 1988 if !customer.HasPaymentMethod { 1989 return ErrNoPaymentMethods 1990 } 1991 if !customer.LatestTOSAccepted { 1992 return ErrTOSNotAccepted 1993 } 1994 return nil 1995 case 404: 1996 // Likely because user has no account registered on the pay server 1997 return fmt.Errorf("cannot get customer details: server says no account exists") 1998 case 401: 1999 return ErrInvalidCredentials 2000 default: 2001 if len(errors.Errors) == 0 { 2002 return fmt.Errorf("cannot get customer details: unexpected HTTP code %d", resp.StatusCode) 2003 } 2004 return &errors 2005 } 2006 } 2007 2008 func (s *Store) CacheDownloads() int { 2009 return s.cfg.CacheDownloads 2010 } 2011 2012 func (s *Store) SetCacheDownloads(fileCount int) { 2013 s.cfg.CacheDownloads = fileCount 2014 if fileCount > 0 { 2015 s.cacher = NewCacheManager(dirs.SnapDownloadCacheDir, fileCount) 2016 } else { 2017 s.cacher = &nullCache{} 2018 } 2019 } 2020 2021 // snap action: install/refresh 2022 2023 type CurrentSnap struct { 2024 InstanceName string 2025 SnapID string 2026 Revision snap.Revision 2027 TrackingChannel string 2028 RefreshedDate time.Time 2029 IgnoreValidation bool 2030 Block []snap.Revision 2031 Epoch snap.Epoch 2032 CohortKey string 2033 } 2034 2035 type currentSnapV2JSON struct { 2036 SnapID string `json:"snap-id"` 2037 InstanceKey string `json:"instance-key"` 2038 Revision int `json:"revision"` 2039 TrackingChannel string `json:"tracking-channel"` 2040 Epoch snap.Epoch `json:"epoch"` 2041 RefreshedDate *time.Time `json:"refreshed-date,omitempty"` 2042 IgnoreValidation bool `json:"ignore-validation,omitempty"` 2043 CohortKey string `json:"cohort-key,omitempty"` 2044 } 2045 2046 type SnapActionFlags int 2047 2048 const ( 2049 SnapActionIgnoreValidation SnapActionFlags = 1 << iota 2050 SnapActionEnforceValidation 2051 ) 2052 2053 type SnapAction struct { 2054 Action string 2055 InstanceName string 2056 SnapID string 2057 Channel string 2058 Revision snap.Revision 2059 CohortKey string 2060 Flags SnapActionFlags 2061 Epoch snap.Epoch 2062 } 2063 2064 func isValidAction(action string) bool { 2065 switch action { 2066 case "download", "install", "refresh": 2067 return true 2068 default: 2069 return false 2070 } 2071 } 2072 2073 type snapActionJSON struct { 2074 Action string `json:"action"` 2075 InstanceKey string `json:"instance-key"` 2076 Name string `json:"name,omitempty"` 2077 SnapID string `json:"snap-id,omitempty"` 2078 Channel string `json:"channel,omitempty"` 2079 Revision int `json:"revision,omitempty"` 2080 CohortKey string `json:"cohort-key,omitempty"` 2081 IgnoreValidation *bool `json:"ignore-validation,omitempty"` 2082 2083 // NOTE the store needs an epoch (even if null) for the "install" and "download" 2084 // actions, to know the client handles epochs at all. "refresh" actions should 2085 // send nothing, not even null -- the snap in the context should have the epoch 2086 // already. We achieve this by making Epoch be an `interface{}` with omitempty, 2087 // and then setting it to a (possibly nil) epoch for install and download. As a 2088 // nil epoch is not an empty interface{}, you'll get the null in the json. 2089 Epoch interface{} `json:"epoch,omitempty"` 2090 } 2091 2092 type snapRelease struct { 2093 Architecture string `json:"architecture"` 2094 Channel string `json:"channel"` 2095 } 2096 2097 type snapActionResult struct { 2098 Result string `json:"result"` 2099 InstanceKey string `json:"instance-key"` 2100 SnapID string `json:"snap-id,omitempy"` 2101 Name string `json:"name,omitempty"` 2102 Snap storeSnap `json:"snap"` 2103 EffectiveChannel string `json:"effective-channel,omitempty"` 2104 Error struct { 2105 Code string `json:"code"` 2106 Message string `json:"message"` 2107 Extra struct { 2108 Releases []snapRelease `json:"releases"` 2109 } `json:"extra"` 2110 } `json:"error"` 2111 } 2112 2113 type snapActionRequest struct { 2114 Context []*currentSnapV2JSON `json:"context"` 2115 Actions []*snapActionJSON `json:"actions"` 2116 Fields []string `json:"fields"` 2117 } 2118 2119 type snapActionResultList struct { 2120 Results []*snapActionResult `json:"results"` 2121 ErrorList []struct { 2122 Code string `json:"code"` 2123 Message string `json:"message"` 2124 } `json:"error-list"` 2125 } 2126 2127 var snapActionFields = jsonutil.StructFields((*storeSnap)(nil)) 2128 2129 // SnapAction queries the store for snap information for the given 2130 // install/refresh actions, given the context information about 2131 // current installed snaps in currentSnaps. If the request was overall 2132 // successul (200) but there were reported errors it will return both 2133 // the snap infos and an SnapActionError. 2134 func (s *Store) SnapAction(ctx context.Context, currentSnaps []*CurrentSnap, actions []*SnapAction, user *auth.UserState, opts *RefreshOptions) ([]*snap.Info, error) { 2135 if opts == nil { 2136 opts = &RefreshOptions{} 2137 } 2138 2139 if len(currentSnaps) == 0 && len(actions) == 0 { 2140 // nothing to do 2141 return nil, &SnapActionError{NoResults: true} 2142 } 2143 2144 authRefreshes := 0 2145 for { 2146 snaps, err := s.snapAction(ctx, currentSnaps, actions, user, opts) 2147 2148 if saErr, ok := err.(*SnapActionError); ok && authRefreshes < 2 && len(saErr.Other) > 0 { 2149 // do we need to try to refresh auths?, 2 tries 2150 var refreshNeed authRefreshNeed 2151 for _, otherErr := range saErr.Other { 2152 switch otherErr { 2153 case errUserAuthorizationNeedsRefresh: 2154 refreshNeed.user = true 2155 case errDeviceAuthorizationNeedsRefresh: 2156 refreshNeed.device = true 2157 } 2158 } 2159 if refreshNeed.needed() { 2160 err := s.refreshAuth(user, refreshNeed) 2161 if err != nil { 2162 // best effort 2163 logger.Noticef("cannot refresh soft-expired authorisation: %v", err) 2164 } 2165 authRefreshes++ 2166 // TODO: we could avoid retrying here 2167 // if refreshAuth gave no error we got 2168 // as many non-error results from the 2169 // store as actions anyway 2170 continue 2171 } 2172 } 2173 2174 return snaps, err 2175 } 2176 } 2177 2178 func genInstanceKey(curSnap *CurrentSnap, salt string) (string, error) { 2179 _, snapInstanceKey := snap.SplitInstanceName(curSnap.InstanceName) 2180 2181 if snapInstanceKey == "" { 2182 return curSnap.SnapID, nil 2183 } 2184 2185 if salt == "" { 2186 return "", fmt.Errorf("internal error: request salt not provided") 2187 } 2188 2189 // due to privacy concerns, avoid sending the local names to the 2190 // backend, instead hash the snap ID and instance key together 2191 h := crypto.SHA256.New() 2192 h.Write([]byte(curSnap.SnapID)) 2193 h.Write([]byte(snapInstanceKey)) 2194 h.Write([]byte(salt)) 2195 enc := base64.RawURLEncoding.EncodeToString(h.Sum(nil)) 2196 return fmt.Sprintf("%s:%s", curSnap.SnapID, enc), nil 2197 } 2198 2199 func (s *Store) snapAction(ctx context.Context, currentSnaps []*CurrentSnap, actions []*SnapAction, user *auth.UserState, opts *RefreshOptions) ([]*snap.Info, error) { 2200 2201 // TODO: the store already requires instance-key but doesn't 2202 // yet support repeating in context or sending actions for the 2203 // same snap-id, for now we keep instance-key handling internal 2204 2205 requestSalt := "" 2206 if opts != nil { 2207 requestSalt = opts.PrivacyKey 2208 } 2209 curSnaps := make(map[string]*CurrentSnap, len(currentSnaps)) 2210 curSnapJSONs := make([]*currentSnapV2JSON, len(currentSnaps)) 2211 instanceNameToKey := make(map[string]string, len(currentSnaps)) 2212 for i, curSnap := range currentSnaps { 2213 if curSnap.SnapID == "" || curSnap.InstanceName == "" || curSnap.Revision.Unset() { 2214 return nil, fmt.Errorf("internal error: invalid current snap information") 2215 } 2216 instanceKey, err := genInstanceKey(curSnap, requestSalt) 2217 if err != nil { 2218 return nil, err 2219 } 2220 curSnaps[instanceKey] = curSnap 2221 instanceNameToKey[curSnap.InstanceName] = instanceKey 2222 2223 channel := curSnap.TrackingChannel 2224 if channel == "" { 2225 channel = "stable" 2226 } 2227 var refreshedDate *time.Time 2228 if !curSnap.RefreshedDate.IsZero() { 2229 refreshedDate = &curSnap.RefreshedDate 2230 } 2231 curSnapJSONs[i] = ¤tSnapV2JSON{ 2232 SnapID: curSnap.SnapID, 2233 InstanceKey: instanceKey, 2234 Revision: curSnap.Revision.N, 2235 TrackingChannel: channel, 2236 IgnoreValidation: curSnap.IgnoreValidation, 2237 RefreshedDate: refreshedDate, 2238 Epoch: curSnap.Epoch, 2239 CohortKey: curSnap.CohortKey, 2240 } 2241 } 2242 2243 downloadNum := 0 2244 installNum := 0 2245 installs := make(map[string]*SnapAction, len(actions)) 2246 downloads := make(map[string]*SnapAction, len(actions)) 2247 refreshes := make(map[string]*SnapAction, len(actions)) 2248 actionJSONs := make([]*snapActionJSON, len(actions)) 2249 for i, a := range actions { 2250 if !isValidAction(a.Action) { 2251 return nil, fmt.Errorf("internal error: unsupported action %q", a.Action) 2252 } 2253 if a.InstanceName == "" { 2254 return nil, fmt.Errorf("internal error: action without instance name") 2255 } 2256 var ignoreValidation *bool 2257 if a.Flags&SnapActionIgnoreValidation != 0 { 2258 var t = true 2259 ignoreValidation = &t 2260 } else if a.Flags&SnapActionEnforceValidation != 0 { 2261 var f = false 2262 ignoreValidation = &f 2263 } 2264 2265 var instanceKey string 2266 aJSON := &snapActionJSON{ 2267 Action: a.Action, 2268 SnapID: a.SnapID, 2269 Channel: a.Channel, 2270 Revision: a.Revision.N, 2271 CohortKey: a.CohortKey, 2272 IgnoreValidation: ignoreValidation, 2273 } 2274 if !a.Revision.Unset() { 2275 a.Channel = "" 2276 } 2277 2278 if a.Action == "install" { 2279 installNum++ 2280 instanceKey = fmt.Sprintf("install-%d", installNum) 2281 installs[instanceKey] = a 2282 } else if a.Action == "download" { 2283 downloadNum++ 2284 instanceKey = fmt.Sprintf("download-%d", downloadNum) 2285 downloads[instanceKey] = a 2286 if _, key := snap.SplitInstanceName(a.InstanceName); key != "" { 2287 return nil, fmt.Errorf("internal error: unsupported download with instance name %q", a.InstanceName) 2288 } 2289 } else { 2290 instanceKey = instanceNameToKey[a.InstanceName] 2291 refreshes[instanceKey] = a 2292 } 2293 2294 if a.Action != "refresh" { 2295 aJSON.Name = snap.InstanceSnap(a.InstanceName) 2296 if a.Epoch.IsZero() { 2297 // Let the store know we can handle epochs, by sending the `epoch` 2298 // field in the request. A nil epoch is not an empty interface{}, 2299 // you'll get the null in the json. See comment in snapActionJSON. 2300 aJSON.Epoch = (*snap.Epoch)(nil) 2301 } else { 2302 // this is the amend case 2303 aJSON.Epoch = &a.Epoch 2304 } 2305 } 2306 2307 aJSON.InstanceKey = instanceKey 2308 2309 actionJSONs[i] = aJSON 2310 } 2311 2312 // build input for the install/refresh endpoint 2313 jsonData, err := json.Marshal(snapActionRequest{ 2314 Context: curSnapJSONs, 2315 Actions: actionJSONs, 2316 Fields: snapActionFields, 2317 }) 2318 if err != nil { 2319 return nil, err 2320 } 2321 2322 reqOptions := &requestOptions{ 2323 Method: "POST", 2324 URL: s.endpointURL(snapActionEndpPath, nil), 2325 Accept: jsonContentType, 2326 ContentType: jsonContentType, 2327 Data: jsonData, 2328 APILevel: apiV2Endps, 2329 } 2330 2331 if opts.IsAutoRefresh { 2332 logger.Debugf("Auto-refresh; adding header Snap-Refresh-Reason: scheduled") 2333 reqOptions.addHeader("Snap-Refresh-Reason", "scheduled") 2334 } 2335 2336 if useDeltas() { 2337 logger.Debugf("Deltas enabled. Adding header Snap-Accept-Delta-Format: %v", s.deltaFormat) 2338 reqOptions.addHeader("Snap-Accept-Delta-Format", s.deltaFormat) 2339 } 2340 if opts.RefreshManaged { 2341 reqOptions.addHeader("Snap-Refresh-Managed", "true") 2342 } 2343 2344 var results snapActionResultList 2345 resp, err := s.retryRequestDecodeJSON(ctx, reqOptions, user, &results, nil) 2346 if err != nil { 2347 return nil, err 2348 } 2349 2350 if resp.StatusCode != 200 { 2351 return nil, respToError(resp, "query the store for updates") 2352 } 2353 2354 s.extractSuggestedCurrency(resp) 2355 2356 refreshErrors := make(map[string]error) 2357 installErrors := make(map[string]error) 2358 downloadErrors := make(map[string]error) 2359 var otherErrors []error 2360 2361 var snaps []*snap.Info 2362 for _, res := range results.Results { 2363 if res.Result == "error" { 2364 if a := installs[res.InstanceKey]; a != nil { 2365 if res.Name != "" { 2366 installErrors[a.InstanceName] = translateSnapActionError("install", a.Channel, res.Error.Code, res.Error.Message, res.Error.Extra.Releases) 2367 continue 2368 } 2369 } else if a := downloads[res.InstanceKey]; a != nil { 2370 if res.Name != "" { 2371 downloadErrors[res.Name] = translateSnapActionError("download", a.Channel, res.Error.Code, res.Error.Message, res.Error.Extra.Releases) 2372 continue 2373 } 2374 } else { 2375 if cur := curSnaps[res.InstanceKey]; cur != nil { 2376 a := refreshes[res.InstanceKey] 2377 if a == nil { 2378 // got an error for a snap that was not part of an 'action' 2379 otherErrors = append(otherErrors, translateSnapActionError("", "", res.Error.Code, fmt.Sprintf("snap %q: %s", cur.InstanceName, res.Error.Message), nil)) 2380 logger.Debugf("Unexpected error for snap %q, instance key %v: [%v] %v", cur.InstanceName, res.InstanceKey, res.Error.Code, res.Error.Message) 2381 continue 2382 } 2383 channel := a.Channel 2384 if channel == "" && a.Revision.Unset() { 2385 channel = cur.TrackingChannel 2386 } 2387 refreshErrors[cur.InstanceName] = translateSnapActionError("refresh", channel, res.Error.Code, res.Error.Message, res.Error.Extra.Releases) 2388 continue 2389 } 2390 } 2391 otherErrors = append(otherErrors, translateSnapActionError("", "", res.Error.Code, res.Error.Message, nil)) 2392 continue 2393 } 2394 snapInfo, err := infoFromStoreSnap(&res.Snap) 2395 if err != nil { 2396 return nil, fmt.Errorf("unexpected invalid install/refresh API result: %v", err) 2397 } 2398 2399 snapInfo.Channel = res.EffectiveChannel 2400 2401 var instanceName string 2402 if res.Result == "refresh" { 2403 cur := curSnaps[res.InstanceKey] 2404 if cur == nil { 2405 return nil, fmt.Errorf("unexpected invalid install/refresh API result: unexpected refresh") 2406 } 2407 rrev := snap.R(res.Snap.Revision) 2408 if rrev == cur.Revision || findRev(rrev, cur.Block) { 2409 refreshErrors[cur.InstanceName] = ErrNoUpdateAvailable 2410 continue 2411 } 2412 instanceName = cur.InstanceName 2413 } else if res.Result == "install" { 2414 if action := installs[res.InstanceKey]; action != nil { 2415 instanceName = action.InstanceName 2416 } 2417 } 2418 2419 if res.Result != "download" && instanceName == "" { 2420 return nil, fmt.Errorf("unexpected invalid install/refresh API result: unexpected instance-key %q", res.InstanceKey) 2421 } 2422 2423 _, instanceKey := snap.SplitInstanceName(instanceName) 2424 snapInfo.InstanceKey = instanceKey 2425 2426 snaps = append(snaps, snapInfo) 2427 } 2428 2429 for _, errObj := range results.ErrorList { 2430 otherErrors = append(otherErrors, translateSnapActionError("", "", errObj.Code, errObj.Message, nil)) 2431 } 2432 2433 if len(refreshErrors)+len(installErrors)+len(downloadErrors) != 0 || len(results.Results) == 0 || len(otherErrors) != 0 { 2434 // normalize empty maps 2435 if len(refreshErrors) == 0 { 2436 refreshErrors = nil 2437 } 2438 if len(installErrors) == 0 { 2439 installErrors = nil 2440 } 2441 if len(downloadErrors) == 0 { 2442 downloadErrors = nil 2443 } 2444 return snaps, &SnapActionError{ 2445 NoResults: len(results.Results) == 0, 2446 Refresh: refreshErrors, 2447 Install: installErrors, 2448 Download: downloadErrors, 2449 Other: otherErrors, 2450 } 2451 } 2452 2453 return snaps, nil 2454 } 2455 2456 // abbreviated info structs just for the download info 2457 type storeInfoChannelAbbrev struct { 2458 Download storeSnapDownload `json:"download"` 2459 } 2460 2461 type storeInfoAbbrev struct { 2462 // discard anything beyond the first entry 2463 ChannelMap [1]storeInfoChannelAbbrev `json:"channel-map"` 2464 } 2465 2466 var errUnexpectedConnCheckResponse = errors.New("unexpected response during connection check") 2467 2468 func (s *Store) snapConnCheck() ([]string, error) { 2469 var hosts []string 2470 // NOTE: "core" is possibly the only snap that's sure to be in all stores 2471 // when we drop "core" in the move to snapd/core18/etc, change this 2472 infoURL := s.endpointURL(path.Join(snapInfoEndpPath, "core"), url.Values{ 2473 // we only want the download URL 2474 "fields": {"download"}, 2475 // we only need *one* (but can't filter by channel ... yet) 2476 "architecture": {s.architecture}, 2477 }) 2478 hosts = append(hosts, infoURL.Host) 2479 2480 var result storeInfoAbbrev 2481 resp, err := httputil.RetryRequest(infoURL.String(), func() (*http.Response, error) { 2482 return s.doRequest(context.TODO(), s.client, &requestOptions{ 2483 Method: "GET", 2484 URL: infoURL, 2485 APILevel: apiV2Endps, 2486 }, nil) 2487 }, func(resp *http.Response) error { 2488 return decodeJSONBody(resp, &result, nil) 2489 }, connCheckStrategy) 2490 2491 if err != nil { 2492 return hosts, err 2493 } 2494 resp.Body.Close() 2495 2496 dlURLraw := result.ChannelMap[0].Download.URL 2497 dlURL, err := url.ParseRequestURI(dlURLraw) 2498 if err != nil { 2499 return hosts, err 2500 } 2501 hosts = append(hosts, dlURL.Host) 2502 2503 cdnHeader, err := s.cdnHeader() 2504 if err != nil { 2505 return hosts, err 2506 } 2507 2508 reqOptions := downloadReqOpts(dlURL, cdnHeader, nil) 2509 reqOptions.Method = "HEAD" // not actually a download 2510 2511 // TODO: We need the HEAD here so that we get redirected to the 2512 // right CDN machine. Consider just doing a "net.Dial" 2513 // after the redirect here. Suggested in 2514 // https://github.com/snapcore/snapd/pull/5176#discussion_r193437230 2515 resp, err = httputil.RetryRequest(dlURLraw, func() (*http.Response, error) { 2516 return s.doRequest(context.TODO(), s.client, reqOptions, nil) 2517 }, func(resp *http.Response) error { 2518 // account for redirect 2519 hosts[len(hosts)-1] = resp.Request.URL.Host 2520 return nil 2521 }, connCheckStrategy) 2522 if err != nil { 2523 return hosts, err 2524 } 2525 resp.Body.Close() 2526 2527 if resp.StatusCode != 200 { 2528 return hosts, errUnexpectedConnCheckResponse 2529 } 2530 2531 return hosts, nil 2532 } 2533 2534 func (s *Store) ConnectivityCheck() (status map[string]bool, err error) { 2535 status = make(map[string]bool) 2536 2537 checkers := []func() ([]string, error){ 2538 s.snapConnCheck, 2539 } 2540 2541 for _, checker := range checkers { 2542 hosts, err := checker() 2543 for _, host := range hosts { 2544 status[host] = (err == nil) 2545 } 2546 } 2547 2548 return status, nil 2549 } 2550 2551 func (s *Store) CreateCohorts(ctx context.Context, snaps []string) (map[string]string, error) { 2552 jsonData, err := json.Marshal(map[string][]string{"snaps": snaps}) 2553 if err != nil { 2554 return nil, err 2555 } 2556 2557 u := s.endpointURL(cohortsEndpPath, nil) 2558 reqOptions := &requestOptions{ 2559 Method: "POST", 2560 URL: u, 2561 APILevel: apiV2Endps, 2562 Data: jsonData, 2563 } 2564 2565 var remote struct { 2566 CohortKeys map[string]string `json:"cohort-keys"` 2567 } 2568 resp, err := s.retryRequestDecodeJSON(ctx, reqOptions, nil, &remote, nil) 2569 if err != nil { 2570 return nil, err 2571 } 2572 switch resp.StatusCode { 2573 case 200: 2574 // OK 2575 case 404: 2576 return nil, ErrSnapNotFound 2577 default: 2578 return nil, respToError(resp, fmt.Sprintf("create cohorts for %s", strutil.Quoted(snaps))) 2579 } 2580 2581 return remote.CohortKeys, nil 2582 }