github.com/freetocompute/snapd@v0.0.0-20210618182524-2fb355d72fd9/store/store.go (about)

     1  // -*- Mode: Go; indent-tabs-mode: t -*-
     2  
     3  /*
     4   * Copyright (C) 2014-2021 Canonical Ltd
     5   *
     6   * This program is free software: you can redistribute it and/or modify
     7   * it under the terms of the GNU General Public License version 3 as
     8   * published by the Free Software Foundation.
     9   *
    10   * This program is distributed in the hope that it will be useful,
    11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13   * GNU General Public License for more details.
    14   *
    15   * You should have received a copy of the GNU General Public License
    16   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17   *
    18   */
    19  
    20  // Package store has support to use the Ubuntu Store for querying and downloading of snaps, and the related services.
    21  package store
    22  
    23  import (
    24  	"bytes"
    25  	"context"
    26  	"encoding/json"
    27  	"errors"
    28  	"fmt"
    29  	"io"
    30  	"net/http"
    31  	"net/url"
    32  	"os"
    33  	"path"
    34  	"strconv"
    35  	"strings"
    36  	"sync"
    37  	"time"
    38  
    39  	"gopkg.in/retry.v1"
    40  
    41  	"github.com/snapcore/snapd/arch"
    42  	"github.com/snapcore/snapd/client"
    43  	"github.com/snapcore/snapd/dirs"
    44  	"github.com/snapcore/snapd/httputil"
    45  	"github.com/snapcore/snapd/jsonutil"
    46  	"github.com/snapcore/snapd/logger"
    47  	"github.com/snapcore/snapd/osutil"
    48  	"github.com/snapcore/snapd/overlord/auth"
    49  	"github.com/snapcore/snapd/release"
    50  	"github.com/snapcore/snapd/snap"
    51  	"github.com/snapcore/snapd/snap/channel"
    52  	"github.com/snapcore/snapd/snap/naming"
    53  	"github.com/snapcore/snapd/snapdenv"
    54  	"github.com/snapcore/snapd/strutil"
    55  )
    56  
    57  // TODO: better/shorter names are probably in order once fewer legacy places are using this
    58  
    59  const (
    60  	// halJsonContentType is the default accept value for store requests
    61  	halJsonContentType = "application/hal+json"
    62  	// jsonContentType is for store enpoints that don't support HAL
    63  	jsonContentType = "application/json"
    64  	// UbuntuCoreWireProtocol is the protocol level we support when
    65  	// communicating with the store. History:
    66  	//  - "1": client supports squashfs snaps
    67  	UbuntuCoreWireProtocol = "1"
    68  )
    69  
    70  // the LimitTime should be slightly more than 3 times of our http.Client
    71  // Timeout value
    72  var defaultRetryStrategy = retry.LimitCount(6, retry.LimitTime(38*time.Second,
    73  	retry.Exponential{
    74  		Initial: 500 * time.Millisecond,
    75  		Factor:  2.5,
    76  	},
    77  ))
    78  
    79  var connCheckStrategy = retry.LimitCount(3, retry.LimitTime(38*time.Second,
    80  	retry.Exponential{
    81  		Initial: 900 * time.Millisecond,
    82  		Factor:  1.3,
    83  	},
    84  ))
    85  
    86  // Config represents the configuration to access the snap store
    87  type Config struct {
    88  	// Store API base URLs. The assertions url is only separate because it can
    89  	// be overridden by its own env var.
    90  	StoreBaseURL      *url.URL
    91  	AssertionsBaseURL *url.URL
    92  
    93  	// StoreID is the store id used if we can't get one through the DeviceAndAuthContext.
    94  	StoreID string
    95  
    96  	Architecture string
    97  	Series       string
    98  
    99  	DetailFields []string
   100  	InfoFields   []string
   101  	// search v2 fields
   102  	FindFields  []string
   103  	DeltaFormat string
   104  
   105  	// CacheDownloads is the number of downloads that should be cached
   106  	CacheDownloads int
   107  
   108  	// Proxy returns the HTTP proxy to use when talking to the store
   109  	Proxy func(*http.Request) (*url.URL, error)
   110  }
   111  
   112  // setBaseURL updates the store API's base URL in the Config. Must not be used
   113  // to change active config.
   114  func (cfg *Config) setBaseURL(u *url.URL) error {
   115  	storeBaseURI, err := storeURL(u)
   116  	if err != nil {
   117  		return err
   118  	}
   119  
   120  	assertsBaseURI, err := assertsURL()
   121  	if err != nil {
   122  		return err
   123  	}
   124  
   125  	cfg.StoreBaseURL = storeBaseURI
   126  	cfg.AssertionsBaseURL = assertsBaseURI
   127  
   128  	return nil
   129  }
   130  
   131  // Store represents the ubuntu snap store
   132  type Store struct {
   133  	cfg *Config
   134  
   135  	architecture string
   136  	series       string
   137  
   138  	noCDN bool
   139  
   140  	fallbackStoreID string
   141  
   142  	detailFields []string
   143  	infoFields   []string
   144  	findFields   []string
   145  	deltaFormat  string
   146  	// reused http client
   147  	client *http.Client
   148  
   149  	dauthCtx  DeviceAndAuthContext
   150  	sessionMu sync.Mutex
   151  
   152  	mu                sync.Mutex
   153  	suggestedCurrency string
   154  
   155  	cacher downloadCache
   156  
   157  	proxy              func(*http.Request) (*url.URL, error)
   158  	proxyConnectHeader http.Header
   159  
   160  	userAgent string
   161  }
   162  
   163  var ErrTooManyRequests = errors.New("too many requests")
   164  
   165  // UnexpectedHTTPStatusError represents an error where the store
   166  // returned an unexpected HTTP status code, i.e. a status code that
   167  // doesn't represent success nor an expected error condition with
   168  // known handling (e.g. a 404 when instead presence is always
   169  // expected).
   170  type UnexpectedHTTPStatusError struct {
   171  	OpSummary  string
   172  	StatusCode int
   173  	Method     string
   174  	URL        *url.URL
   175  	OopsID     string
   176  }
   177  
   178  func (e *UnexpectedHTTPStatusError) Error() string {
   179  	tpl := "cannot %s: got unexpected HTTP status code %d via %s to %q"
   180  	if e.OopsID != "" {
   181  		tpl += " [%s]"
   182  		return fmt.Sprintf(tpl, e.OpSummary, e.StatusCode, e.Method, e.URL, e.OopsID)
   183  	}
   184  	return fmt.Sprintf(tpl, e.OpSummary, e.StatusCode, e.Method, e.URL)
   185  }
   186  
   187  func respToError(resp *http.Response, opSummary string) error {
   188  	if resp.StatusCode == 429 {
   189  		return ErrTooManyRequests
   190  	}
   191  	return &UnexpectedHTTPStatusError{
   192  		OpSummary:  opSummary,
   193  		StatusCode: resp.StatusCode,
   194  		Method:     resp.Request.Method,
   195  		URL:        resp.Request.URL,
   196  		OopsID:     resp.Header.Get("X-Oops-Id"),
   197  	}
   198  }
   199  
   200  // endpointURL clones a base URL and updates it with optional path and query.
   201  func endpointURL(base *url.URL, path string, query url.Values) *url.URL {
   202  	u := *base
   203  	if path != "" {
   204  		u.Path = strings.TrimSuffix(u.Path, "/") + "/" + strings.TrimPrefix(path, "/")
   205  		u.RawQuery = ""
   206  	}
   207  	if len(query) != 0 {
   208  		u.RawQuery = query.Encode()
   209  	}
   210  	return &u
   211  }
   212  
   213  // apiURL returns the system default base API URL.
   214  func apiURL() *url.URL {
   215  	s := "https://api.snapcraft.io/"
   216  	if snapdenv.UseStagingStore() {
   217  		s = "https://api.staging.snapcraft.io/"
   218  	}
   219  	u, _ := url.Parse(s)
   220  	return u
   221  }
   222  
   223  // storeURL returns the base store URL, derived from either the given API URL
   224  // or an env var override.
   225  func storeURL(api *url.URL) (*url.URL, error) {
   226  	var override string
   227  	var overrideName string
   228  	// XXX: time to drop FORCE_CPI support
   229  	// XXX: Deprecated but present for backward-compatibility: this used
   230  	// to be "Click Package Index".  Remove this once people have got
   231  	// used to SNAPPY_FORCE_API_URL instead.
   232  	if s := os.Getenv("SNAPPY_FORCE_CPI_URL"); s != "" && strings.HasSuffix(s, "api/v1/") {
   233  		overrideName = "SNAPPY_FORCE_CPI_URL"
   234  		override = strings.TrimSuffix(s, "api/v1/")
   235  	} else if s := os.Getenv("SNAPPY_FORCE_API_URL"); s != "" {
   236  		overrideName = "SNAPPY_FORCE_API_URL"
   237  		override = s
   238  	}
   239  	if override != "" {
   240  		u, err := url.Parse(override)
   241  		if err != nil {
   242  			return nil, fmt.Errorf("invalid %s: %s", overrideName, err)
   243  		}
   244  		return u, nil
   245  	}
   246  	return api, nil
   247  }
   248  
   249  func assertsURL() (*url.URL, error) {
   250  	if s := os.Getenv("SNAPPY_FORCE_SAS_URL"); s != "" {
   251  		u, err := url.Parse(s)
   252  		if err != nil {
   253  			return nil, fmt.Errorf("invalid SNAPPY_FORCE_SAS_URL: %s", err)
   254  		}
   255  		return u, nil
   256  	}
   257  
   258  	// nil means fallback to store base url
   259  	return nil, nil
   260  }
   261  
   262  func authLocation() string {
   263  	if snapdenv.UseStagingStore() {
   264  		return "login.staging.ubuntu.com"
   265  	}
   266  	return "login.ubuntu.com"
   267  }
   268  
   269  func authURL() string {
   270  	if u := os.Getenv("SNAPPY_FORCE_SSO_URL"); u != "" {
   271  		return u
   272  	}
   273  	return "https://" + authLocation() + "/api/v2"
   274  }
   275  
   276  var defaultStoreDeveloperURL = "https://dashboard.snapcraft.io/"
   277  
   278  func storeDeveloperURL() string {
   279  	if snapdenv.UseStagingStore() {
   280  		return "https://dashboard.staging.snapcraft.io/"
   281  	}
   282  	return defaultStoreDeveloperURL
   283  }
   284  
   285  var defaultConfig = Config{}
   286  
   287  // DefaultConfig returns a copy of the default configuration ready to be adapted.
   288  func DefaultConfig() *Config {
   289  	cfg := defaultConfig
   290  	return &cfg
   291  }
   292  
   293  func init() {
   294  	storeBaseURI, err := storeURL(apiURL())
   295  	if err != nil {
   296  		panic(err)
   297  	}
   298  	if storeBaseURI.RawQuery != "" {
   299  		panic("store API URL may not contain query string")
   300  	}
   301  	err = defaultConfig.setBaseURL(storeBaseURI)
   302  	if err != nil {
   303  		panic(err)
   304  	}
   305  	defaultConfig.DetailFields = jsonutil.StructFields((*snapDetails)(nil), "snap_yaml_raw")
   306  	defaultConfig.InfoFields = jsonutil.StructFields((*storeSnap)(nil), "snap-yaml")
   307  	defaultConfig.FindFields = append(jsonutil.StructFields((*storeSnap)(nil),
   308  		"architectures", "created-at", "epoch", "name", "snap-id", "snap-yaml"),
   309  		"channel")
   310  }
   311  
   312  type searchV2Results struct {
   313  	Results   []*storeSearchResult `json:"results"`
   314  	ErrorList []struct {
   315  		Code    string `json:"code"`
   316  		Message string `json:"message"`
   317  	} `json:"error-list"`
   318  }
   319  
   320  type searchResults struct {
   321  	Payload struct {
   322  		Packages []*snapDetails `json:"clickindex:package"`
   323  	} `json:"_embedded"`
   324  }
   325  
   326  type sectionResults struct {
   327  	Payload struct {
   328  		Sections []struct{ Name string } `json:"clickindex:sections"`
   329  	} `json:"_embedded"`
   330  }
   331  
   332  // The default delta format if not configured.
   333  var defaultSupportedDeltaFormat = "xdelta3"
   334  
   335  // New creates a new Store with the given access configuration and for given the store id.
   336  func New(cfg *Config, dauthCtx DeviceAndAuthContext) *Store {
   337  	if cfg == nil {
   338  		cfg = &defaultConfig
   339  	}
   340  
   341  	detailFields := cfg.DetailFields
   342  	if detailFields == nil {
   343  		detailFields = defaultConfig.DetailFields
   344  	}
   345  
   346  	infoFields := cfg.InfoFields
   347  	if infoFields == nil {
   348  		infoFields = defaultConfig.InfoFields
   349  	}
   350  
   351  	findFields := cfg.FindFields
   352  	if findFields == nil {
   353  		findFields = defaultConfig.FindFields
   354  	}
   355  
   356  	architecture := cfg.Architecture
   357  	if cfg.Architecture == "" {
   358  		architecture = arch.DpkgArchitecture()
   359  	}
   360  
   361  	series := cfg.Series
   362  	if cfg.Series == "" {
   363  		series = release.Series
   364  	}
   365  
   366  	deltaFormat := cfg.DeltaFormat
   367  	if deltaFormat == "" {
   368  		deltaFormat = defaultSupportedDeltaFormat
   369  	}
   370  
   371  	userAgent := snapdenv.UserAgent()
   372  	proxyConnectHeader := http.Header{"User-Agent": []string{userAgent}}
   373  
   374  	store := &Store{
   375  		cfg:                cfg,
   376  		series:             series,
   377  		architecture:       architecture,
   378  		noCDN:              osutil.GetenvBool("SNAPPY_STORE_NO_CDN"),
   379  		fallbackStoreID:    cfg.StoreID,
   380  		detailFields:       detailFields,
   381  		infoFields:         infoFields,
   382  		findFields:         findFields,
   383  		dauthCtx:           dauthCtx,
   384  		deltaFormat:        deltaFormat,
   385  		proxy:              cfg.Proxy,
   386  		proxyConnectHeader: proxyConnectHeader,
   387  		userAgent:          userAgent,
   388  	}
   389  	store.client = store.newHTTPClient(&httputil.ClientOptions{
   390  		Timeout:    10 * time.Second,
   391  		MayLogBody: true,
   392  	})
   393  	store.SetCacheDownloads(cfg.CacheDownloads)
   394  
   395  	return store
   396  }
   397  
   398  // API endpoint paths
   399  const (
   400  	// see https://dashboard.snapcraft.io/docs/
   401  	// XXX: Repeating "api/" here is cumbersome, but the next generation
   402  	// of store APIs will probably drop that prefix (since it now
   403  	// duplicates the hostname), and we may want to switch to v2 APIs
   404  	// one at a time; so it's better to consider that as part of
   405  	// individual endpoint paths.
   406  	searchEndpPath      = "api/v1/snaps/search"
   407  	ordersEndpPath      = "api/v1/snaps/purchases/orders"
   408  	buyEndpPath         = "api/v1/snaps/purchases/buy"
   409  	customersMeEndpPath = "api/v1/snaps/purchases/customers/me"
   410  	sectionsEndpPath    = "api/v1/snaps/sections"
   411  	commandsEndpPath    = "api/v1/snaps/names"
   412  	// v2
   413  	snapActionEndpPath = "v2/snaps/refresh"
   414  	snapInfoEndpPath   = "v2/snaps/info"
   415  	cohortsEndpPath    = "v2/cohorts"
   416  	findEndpPath       = "v2/snaps/find"
   417  
   418  	deviceNonceEndpPath   = "api/v1/snaps/auth/nonces"
   419  	deviceSessionEndpPath = "api/v1/snaps/auth/sessions"
   420  
   421  	assertionsPath = "v2/assertions"
   422  )
   423  
   424  func (s *Store) newHTTPClient(opts *httputil.ClientOptions) *http.Client {
   425  	if opts == nil {
   426  		opts = &httputil.ClientOptions{}
   427  	}
   428  	opts.Proxy = s.cfg.Proxy
   429  	opts.ProxyConnectHeader = s.proxyConnectHeader
   430  	opts.ExtraSSLCerts = &httputil.ExtraSSLCertsFromDir{
   431  		Dir: dirs.SnapdStoreSSLCertsDir,
   432  	}
   433  	return httputil.NewHTTPClient(opts)
   434  }
   435  
   436  func (s *Store) defaultSnapQuery() url.Values {
   437  	q := url.Values{}
   438  	if len(s.detailFields) != 0 {
   439  		q.Set("fields", strings.Join(s.detailFields, ","))
   440  	}
   441  	return q
   442  }
   443  
   444  func (s *Store) baseURL(defaultURL *url.URL) *url.URL {
   445  	u := defaultURL
   446  	if s.dauthCtx != nil {
   447  		var err error
   448  		_, u, err = s.dauthCtx.ProxyStoreParams(defaultURL)
   449  		if err != nil {
   450  			logger.Debugf("cannot get proxy store parameters from state: %v", err)
   451  		}
   452  	}
   453  	if u != nil {
   454  		return u
   455  	}
   456  	return defaultURL
   457  }
   458  
   459  func (s *Store) endpointURL(p string, query url.Values) *url.URL {
   460  	return endpointURL(s.baseURL(s.cfg.StoreBaseURL), p, query)
   461  }
   462  
   463  // LoginUser logs user in the store and returns the authentication macaroons.
   464  func (s *Store) LoginUser(username, password, otp string) (string, string, error) {
   465  	macaroon, err := requestStoreMacaroon(s.client)
   466  	if err != nil {
   467  		return "", "", err
   468  	}
   469  	deserializedMacaroon, err := auth.MacaroonDeserialize(macaroon)
   470  	if err != nil {
   471  		return "", "", err
   472  	}
   473  
   474  	// get SSO 3rd party caveat, and request discharge
   475  	loginCaveat, err := loginCaveatID(deserializedMacaroon)
   476  	if err != nil {
   477  		return "", "", err
   478  	}
   479  
   480  	discharge, err := dischargeAuthCaveat(s.client, loginCaveat, username, password, otp)
   481  	if err != nil {
   482  		return "", "", err
   483  	}
   484  
   485  	return macaroon, discharge, nil
   486  }
   487  
   488  // authAvailable returns true if there is a user and/or device session setup
   489  func (s *Store) authAvailable(user *auth.UserState) (bool, error) {
   490  	if user.HasStoreAuth() {
   491  		return true, nil
   492  	} else {
   493  		var device *auth.DeviceState
   494  		var err error
   495  		if s.dauthCtx != nil {
   496  			device, err = s.dauthCtx.Device()
   497  			if err != nil {
   498  				return false, err
   499  			}
   500  		}
   501  		return device != nil && device.SessionMacaroon != "", nil
   502  	}
   503  }
   504  
   505  // authenticateUser will add the store expected Macaroon Authorization header for user
   506  func authenticateUser(r *http.Request, user *auth.UserState) {
   507  	var buf bytes.Buffer
   508  	fmt.Fprintf(&buf, `Macaroon root="%s"`, user.StoreMacaroon)
   509  
   510  	// deserialize root macaroon (we need its signature to do the discharge binding)
   511  	root, err := auth.MacaroonDeserialize(user.StoreMacaroon)
   512  	if err != nil {
   513  		logger.Debugf("cannot deserialize root macaroon: %v", err)
   514  		return
   515  	}
   516  
   517  	for _, d := range user.StoreDischarges {
   518  		// prepare discharge for request
   519  		discharge, err := auth.MacaroonDeserialize(d)
   520  		if err != nil {
   521  			logger.Debugf("cannot deserialize discharge macaroon: %v", err)
   522  			return
   523  		}
   524  		discharge.Bind(root.Signature())
   525  
   526  		serializedDischarge, err := auth.MacaroonSerialize(discharge)
   527  		if err != nil {
   528  			logger.Debugf("cannot re-serialize discharge macaroon: %v", err)
   529  			return
   530  		}
   531  		fmt.Fprintf(&buf, `, discharge="%s"`, serializedDischarge)
   532  	}
   533  	r.Header.Set("Authorization", buf.String())
   534  }
   535  
   536  // refreshDischarges will request refreshed discharge macaroons for the user
   537  func refreshDischarges(httpClient *http.Client, user *auth.UserState) ([]string, error) {
   538  	newDischarges := make([]string, len(user.StoreDischarges))
   539  	for i, d := range user.StoreDischarges {
   540  		discharge, err := auth.MacaroonDeserialize(d)
   541  		if err != nil {
   542  			return nil, err
   543  		}
   544  		if discharge.Location() != UbuntuoneLocation {
   545  			newDischarges[i] = d
   546  			continue
   547  		}
   548  
   549  		refreshedDischarge, err := refreshDischargeMacaroon(httpClient, d)
   550  		if err != nil {
   551  			return nil, err
   552  		}
   553  		newDischarges[i] = refreshedDischarge
   554  	}
   555  	return newDischarges, nil
   556  }
   557  
   558  // refreshUser will refresh user discharge macaroon and update state
   559  func (s *Store) refreshUser(user *auth.UserState) error {
   560  	if s.dauthCtx == nil {
   561  		return fmt.Errorf("user credentials need to be refreshed but update in place only supported in snapd")
   562  	}
   563  	newDischarges, err := refreshDischarges(s.client, user)
   564  	if err != nil {
   565  		return err
   566  	}
   567  
   568  	curUser, err := s.dauthCtx.UpdateUserAuth(user, newDischarges)
   569  	if err != nil {
   570  		return err
   571  	}
   572  	// update in place
   573  	*user = *curUser
   574  
   575  	return nil
   576  }
   577  
   578  // refreshDeviceSession will set or refresh the device session in the state
   579  func (s *Store) refreshDeviceSession(device *auth.DeviceState) error {
   580  	if s.dauthCtx == nil {
   581  		return fmt.Errorf("internal error: no device and auth context")
   582  	}
   583  
   584  	s.sessionMu.Lock()
   585  	defer s.sessionMu.Unlock()
   586  	// check that no other goroutine has already got a new session etc...
   587  	device1, err := s.dauthCtx.Device()
   588  	if err != nil {
   589  		return err
   590  	}
   591  	// We can replace device with "device1" here because Device
   592  	// and UpdateDeviceAuth (and the underlying SetDevice)
   593  	// require/use the global state lock, so the reading/setting
   594  	// values have a total order, and device1 cannot come before
   595  	// device in that order. See also:
   596  	// https://github.com/snapcore/snapd/pull/6716#discussion_r277025834
   597  	if *device1 != *device {
   598  		// nothing to do
   599  		*device = *device1
   600  		return nil
   601  	}
   602  
   603  	nonce, err := requestStoreDeviceNonce(s.client, s.endpointURL(deviceNonceEndpPath, nil).String())
   604  	if err != nil {
   605  		return err
   606  	}
   607  
   608  	devSessReqParams, err := s.dauthCtx.DeviceSessionRequestParams(nonce)
   609  	if err != nil {
   610  		return err
   611  	}
   612  
   613  	session, err := requestDeviceSession(s.client, s.endpointURL(deviceSessionEndpPath, nil).String(), devSessReqParams, device.SessionMacaroon)
   614  	if err != nil {
   615  		return err
   616  	}
   617  
   618  	curDevice, err := s.dauthCtx.UpdateDeviceAuth(device, session)
   619  	if err != nil {
   620  		return err
   621  	}
   622  	// update in place
   623  	*device = *curDevice
   624  	return nil
   625  }
   626  
   627  // EnsureDeviceSession makes sure the store has a device session available.
   628  // Expects the store to have an AuthContext.
   629  func (s *Store) EnsureDeviceSession() (*auth.DeviceState, error) {
   630  	if s.dauthCtx == nil {
   631  		return nil, fmt.Errorf("internal error: no authContext")
   632  	}
   633  
   634  	device, err := s.dauthCtx.Device()
   635  	if err != nil {
   636  		return nil, err
   637  	}
   638  
   639  	if device.SessionMacaroon != "" {
   640  		return device, nil
   641  	}
   642  	if device.Serial == "" {
   643  		return nil, ErrNoSerial
   644  	}
   645  	// we don't have a session yet but have a serial, try
   646  	// to get a session
   647  	err = s.refreshDeviceSession(device)
   648  	if err != nil {
   649  		return nil, err
   650  	}
   651  	return device, err
   652  }
   653  
   654  // authenticateDevice will add the store expected Macaroon X-Device-Authorization header for device
   655  func authenticateDevice(r *http.Request, device *auth.DeviceState, apiLevel apiLevel) {
   656  	if device != nil && device.SessionMacaroon != "" {
   657  		r.Header.Set(hdrSnapDeviceAuthorization[apiLevel], fmt.Sprintf(`Macaroon root="%s"`, device.SessionMacaroon))
   658  	}
   659  }
   660  
   661  func (s *Store) setStoreID(r *http.Request, apiLevel apiLevel) (customStore bool) {
   662  	storeID := s.fallbackStoreID
   663  	if s.dauthCtx != nil {
   664  		cand, err := s.dauthCtx.StoreID(storeID)
   665  		if err != nil {
   666  			logger.Debugf("cannot get store ID from state: %v", err)
   667  		} else {
   668  			storeID = cand
   669  		}
   670  	}
   671  	if storeID != "" {
   672  		r.Header.Set(hdrSnapDeviceStore[apiLevel], storeID)
   673  		return true
   674  	}
   675  	return false
   676  }
   677  
   678  type apiLevel int
   679  
   680  const (
   681  	apiV1Endps apiLevel = 0 // api/v1 endpoints
   682  	apiV2Endps apiLevel = 1 // v2 endpoints
   683  )
   684  
   685  var (
   686  	hdrSnapDeviceAuthorization = []string{"X-Device-Authorization", "Snap-Device-Authorization"}
   687  	hdrSnapDeviceStore         = []string{"X-Ubuntu-Store", "Snap-Device-Store"}
   688  	hdrSnapDeviceSeries        = []string{"X-Ubuntu-Series", "Snap-Device-Series"}
   689  	hdrSnapDeviceArchitecture  = []string{"X-Ubuntu-Architecture", "Snap-Device-Architecture"}
   690  	hdrSnapClassic             = []string{"X-Ubuntu-Classic", "Snap-Classic"}
   691  )
   692  
   693  type deviceAuthNeed int
   694  
   695  const (
   696  	deviceAuthPreferred deviceAuthNeed = iota
   697  	deviceAuthCustomStoreOnly
   698  )
   699  
   700  // requestOptions specifies parameters for store requests.
   701  type requestOptions struct {
   702  	Method       string
   703  	URL          *url.URL
   704  	Accept       string
   705  	ContentType  string
   706  	APILevel     apiLevel
   707  	ExtraHeaders map[string]string
   708  	Data         []byte
   709  
   710  	// DeviceAuthNeed indicates the level of need to supply device
   711  	// authorization for this request, can be:
   712  	//  - deviceAuthPreferred: should be provided if available
   713  	//  - deviceAuthCustomStoreOnly: should be provided only in case
   714  	//    of a custom store
   715  	DeviceAuthNeed deviceAuthNeed
   716  }
   717  
   718  func (r *requestOptions) addHeader(k, v string) {
   719  	if r.ExtraHeaders == nil {
   720  		r.ExtraHeaders = make(map[string]string)
   721  	}
   722  	r.ExtraHeaders[k] = v
   723  }
   724  
   725  func cancelled(ctx context.Context) bool {
   726  	select {
   727  	case <-ctx.Done():
   728  		return true
   729  	default:
   730  		return false
   731  	}
   732  }
   733  
   734  var expectedCatalogPreamble = []interface{}{
   735  	json.Delim('{'),
   736  	"_embedded",
   737  	json.Delim('{'),
   738  	"clickindex:package",
   739  	json.Delim('['),
   740  }
   741  
   742  type alias struct {
   743  	Name string `json:"name"`
   744  }
   745  
   746  type catalogItem struct {
   747  	Name    string   `json:"package_name"`
   748  	Version string   `json:"version"`
   749  	Summary string   `json:"summary"`
   750  	Aliases []alias  `json:"aliases"`
   751  	Apps    []string `json:"apps"`
   752  }
   753  
   754  type SnapAdder interface {
   755  	AddSnap(snapName, version, summary string, commands []string) error
   756  }
   757  
   758  func decodeCatalog(resp *http.Response, names io.Writer, db SnapAdder) error {
   759  	const what = "decode new commands catalog"
   760  	if resp.StatusCode != 200 {
   761  		return respToError(resp, what)
   762  	}
   763  	dec := json.NewDecoder(resp.Body)
   764  	for _, expectedToken := range expectedCatalogPreamble {
   765  		token, err := dec.Token()
   766  		if err != nil {
   767  			return err
   768  		}
   769  		if token != expectedToken {
   770  			return fmt.Errorf(what+": bad catalog preamble: expected %#v, got %#v", expectedToken, token)
   771  		}
   772  	}
   773  
   774  	for dec.More() {
   775  		var v catalogItem
   776  		if err := dec.Decode(&v); err != nil {
   777  			return fmt.Errorf(what+": %v", err)
   778  		}
   779  		if v.Name == "" {
   780  			continue
   781  		}
   782  		fmt.Fprintln(names, v.Name)
   783  		if len(v.Apps) == 0 {
   784  			continue
   785  		}
   786  
   787  		commands := make([]string, 0, len(v.Aliases)+len(v.Apps))
   788  
   789  		for _, alias := range v.Aliases {
   790  			commands = append(commands, alias.Name)
   791  		}
   792  		for _, app := range v.Apps {
   793  			commands = append(commands, snap.JoinSnapApp(v.Name, app))
   794  		}
   795  
   796  		if err := db.AddSnap(v.Name, v.Version, v.Summary, commands); err != nil {
   797  			return err
   798  		}
   799  	}
   800  
   801  	return nil
   802  }
   803  
   804  func decodeJSONBody(resp *http.Response, success interface{}, failure interface{}) error {
   805  	ok := (resp.StatusCode == 200 || resp.StatusCode == 201)
   806  	// always decode on success; decode failures only if body is not empty
   807  	if !ok && resp.ContentLength == 0 {
   808  		return nil
   809  	}
   810  	result := success
   811  	if !ok {
   812  		result = failure
   813  	}
   814  	if result != nil {
   815  		return json.NewDecoder(resp.Body).Decode(result)
   816  	}
   817  	return nil
   818  }
   819  
   820  // retryRequestDecodeJSON calls retryRequest and decodes the response into either success or failure.
   821  func (s *Store) retryRequestDecodeJSON(ctx context.Context, reqOptions *requestOptions, user *auth.UserState, success interface{}, failure interface{}) (resp *http.Response, err error) {
   822  	return httputil.RetryRequest(reqOptions.URL.String(), func() (*http.Response, error) {
   823  		return s.doRequest(ctx, s.client, reqOptions, user)
   824  	}, func(resp *http.Response) error {
   825  		return decodeJSONBody(resp, success, failure)
   826  	}, defaultRetryStrategy)
   827  }
   828  
   829  // doRequest does an authenticated request to the store handling a potential macaroon refresh required if needed
   830  func (s *Store) doRequest(ctx context.Context, client *http.Client, reqOptions *requestOptions, user *auth.UserState) (*http.Response, error) {
   831  	authRefreshes := 0
   832  	for {
   833  		req, err := s.newRequest(ctx, reqOptions, user)
   834  		if err != nil {
   835  			return nil, err
   836  		}
   837  		if ctx != nil {
   838  			req = req.WithContext(ctx)
   839  		}
   840  
   841  		resp, err := client.Do(req)
   842  		if err != nil {
   843  			return nil, err
   844  		}
   845  
   846  		wwwAuth := resp.Header.Get("WWW-Authenticate")
   847  		if resp.StatusCode == 401 && authRefreshes < 4 {
   848  			// 4 tries: 2 tries for each in case both user
   849  			// and device need refreshing
   850  			var refreshNeed authRefreshNeed
   851  			if user != nil && strings.Contains(wwwAuth, "needs_refresh=1") {
   852  				// refresh user
   853  				refreshNeed.user = true
   854  			}
   855  			if strings.Contains(wwwAuth, "refresh_device_session=1") {
   856  				// refresh device session
   857  				refreshNeed.device = true
   858  			}
   859  			if refreshNeed.needed() {
   860  				err := s.refreshAuth(user, refreshNeed)
   861  				if err != nil {
   862  					return nil, err
   863  				}
   864  				// close previous response and retry
   865  				resp.Body.Close()
   866  				authRefreshes++
   867  				continue
   868  			}
   869  		}
   870  
   871  		return resp, err
   872  	}
   873  }
   874  
   875  type authRefreshNeed struct {
   876  	device bool
   877  	user   bool
   878  }
   879  
   880  func (rn *authRefreshNeed) needed() bool {
   881  	return rn.device || rn.user
   882  }
   883  
   884  func (s *Store) refreshAuth(user *auth.UserState, need authRefreshNeed) error {
   885  	if need.user {
   886  		// refresh user
   887  		err := s.refreshUser(user)
   888  		if err != nil {
   889  			return err
   890  		}
   891  	}
   892  	if need.device {
   893  		// refresh device session
   894  		if s.dauthCtx == nil {
   895  			return fmt.Errorf("internal error: no device and auth context")
   896  		}
   897  		device, err := s.dauthCtx.Device()
   898  		if err != nil {
   899  			return err
   900  		}
   901  
   902  		err = s.refreshDeviceSession(device)
   903  		if err != nil {
   904  			return err
   905  		}
   906  	}
   907  	return nil
   908  }
   909  
   910  // build a new http.Request with headers for the store
   911  func (s *Store) newRequest(ctx context.Context, reqOptions *requestOptions, user *auth.UserState) (*http.Request, error) {
   912  	var body io.Reader
   913  	if reqOptions.Data != nil {
   914  		body = bytes.NewBuffer(reqOptions.Data)
   915  	}
   916  
   917  	req, err := http.NewRequest(reqOptions.Method, reqOptions.URL.String(), body)
   918  	if err != nil {
   919  		return nil, err
   920  	}
   921  
   922  	customStore := s.setStoreID(req, reqOptions.APILevel)
   923  
   924  	if s.dauthCtx != nil && (customStore || reqOptions.DeviceAuthNeed != deviceAuthCustomStoreOnly) {
   925  		device, err := s.EnsureDeviceSession()
   926  		if err != nil && err != ErrNoSerial {
   927  			return nil, err
   928  		}
   929  		if err == ErrNoSerial {
   930  			// missing serial assertion, log and continue without device authentication
   931  			logger.Debugf("cannot set device session: %v", err)
   932  		} else {
   933  			authenticateDevice(req, device, reqOptions.APILevel)
   934  		}
   935  	}
   936  
   937  	// only set user authentication if user logged in to the store
   938  	if user.HasStoreAuth() {
   939  		authenticateUser(req, user)
   940  	}
   941  
   942  	req.Header.Set("User-Agent", s.userAgent)
   943  	req.Header.Set("Accept", reqOptions.Accept)
   944  	req.Header.Set(hdrSnapDeviceArchitecture[reqOptions.APILevel], s.architecture)
   945  	req.Header.Set(hdrSnapDeviceSeries[reqOptions.APILevel], s.series)
   946  	req.Header.Set(hdrSnapClassic[reqOptions.APILevel], strconv.FormatBool(release.OnClassic))
   947  	req.Header.Set("Snap-Device-Capabilities", "default-tracks")
   948  	if cua := ClientUserAgent(ctx); cua != "" {
   949  		req.Header.Set("Snap-Client-User-Agent", cua)
   950  	}
   951  	if reqOptions.APILevel == apiV1Endps {
   952  		req.Header.Set("X-Ubuntu-Wire-Protocol", UbuntuCoreWireProtocol)
   953  	}
   954  
   955  	if reqOptions.ContentType != "" {
   956  		req.Header.Set("Content-Type", reqOptions.ContentType)
   957  	}
   958  
   959  	for header, value := range reqOptions.ExtraHeaders {
   960  		req.Header.Set(header, value)
   961  	}
   962  
   963  	return req, nil
   964  }
   965  
   966  func (s *Store) extractSuggestedCurrency(resp *http.Response) {
   967  	suggestedCurrency := resp.Header.Get("X-Suggested-Currency")
   968  
   969  	if suggestedCurrency != "" {
   970  		s.mu.Lock()
   971  		s.suggestedCurrency = suggestedCurrency
   972  		s.mu.Unlock()
   973  	}
   974  }
   975  
   976  // ordersResult encapsulates the order data sent to us from the software center agent.
   977  //
   978  // {
   979  //   "orders": [
   980  //     {
   981  //       "snap_id": "abcd1234efgh5678ijkl9012",
   982  //       "currency": "USD",
   983  //       "amount": "2.99",
   984  //       "state": "Complete",
   985  //       "refundable_until": null,
   986  //       "purchase_date": "2016-09-20T15:00:00+00:00"
   987  //     },
   988  //     {
   989  //       "snap_id": "abcd1234efgh5678ijkl9012",
   990  //       "currency": null,
   991  //       "amount": null,
   992  //       "state": "Complete",
   993  //       "refundable_until": null,
   994  //       "purchase_date": "2016-09-20T15:00:00+00:00"
   995  //     }
   996  //   ]
   997  // }
   998  type ordersResult struct {
   999  	Orders []*order `json:"orders"`
  1000  }
  1001  
  1002  type order struct {
  1003  	SnapID          string `json:"snap_id"`
  1004  	Currency        string `json:"currency"`
  1005  	Amount          string `json:"amount"`
  1006  	State           string `json:"state"`
  1007  	RefundableUntil string `json:"refundable_until"`
  1008  	PurchaseDate    string `json:"purchase_date"`
  1009  }
  1010  
  1011  // decorateOrders sets the MustBuy property of each snap in the given list according to the user's known orders.
  1012  func (s *Store) decorateOrders(snaps []*snap.Info, user *auth.UserState) error {
  1013  	// Mark every non-free snap as must buy until we know better.
  1014  	hasPriced := false
  1015  	for _, info := range snaps {
  1016  		if info.Paid {
  1017  			info.MustBuy = true
  1018  			hasPriced = true
  1019  		}
  1020  	}
  1021  
  1022  	if user == nil {
  1023  		return nil
  1024  	}
  1025  
  1026  	if !hasPriced {
  1027  		return nil
  1028  	}
  1029  
  1030  	var err error
  1031  
  1032  	reqOptions := &requestOptions{
  1033  		Method: "GET",
  1034  		URL:    s.endpointURL(ordersEndpPath, nil),
  1035  		Accept: jsonContentType,
  1036  	}
  1037  	var result ordersResult
  1038  	resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, &result, nil)
  1039  	if err != nil {
  1040  		return err
  1041  	}
  1042  
  1043  	if resp.StatusCode == 401 {
  1044  		// TODO handle token expiry and refresh
  1045  		return ErrInvalidCredentials
  1046  	}
  1047  	if resp.StatusCode != 200 {
  1048  		return respToError(resp, "obtain known orders from store")
  1049  	}
  1050  
  1051  	// Make a map of the IDs of bought snaps
  1052  	bought := make(map[string]bool)
  1053  	for _, order := range result.Orders {
  1054  		bought[order.SnapID] = true
  1055  	}
  1056  
  1057  	for _, info := range snaps {
  1058  		info.MustBuy = mustBuy(info.Paid, bought[info.SnapID])
  1059  	}
  1060  
  1061  	return nil
  1062  }
  1063  
  1064  // mustBuy determines if a snap requires a payment, based on if it is non-free and if the user has already bought it
  1065  func mustBuy(paid bool, bought bool) bool {
  1066  	if !paid {
  1067  		// If the snap is free, then it doesn't need buying
  1068  		return false
  1069  	}
  1070  
  1071  	return !bought
  1072  }
  1073  
  1074  // A SnapSpec describes a single snap wanted from SnapInfo
  1075  type SnapSpec struct {
  1076  	Name string
  1077  }
  1078  
  1079  // SnapInfo returns the snap.Info for the store-hosted snap matching the given spec, or an error.
  1080  func (s *Store) SnapInfo(ctx context.Context, snapSpec SnapSpec, user *auth.UserState) (*snap.Info, error) {
  1081  	fields := strings.Join(s.infoFields, ",")
  1082  
  1083  	si, resp, err := s.snapInfo(ctx, snapSpec.Name, fields, user)
  1084  	if err != nil {
  1085  		return nil, err
  1086  	}
  1087  
  1088  	info, err := infoFromStoreInfo(si)
  1089  	if err != nil {
  1090  		return nil, err
  1091  	}
  1092  
  1093  	err = s.decorateOrders([]*snap.Info{info}, user)
  1094  	if err != nil {
  1095  		logger.Noticef("cannot get user orders: %v", err)
  1096  	}
  1097  
  1098  	s.extractSuggestedCurrency(resp)
  1099  
  1100  	return info, nil
  1101  }
  1102  
  1103  func (s *Store) snapInfo(ctx context.Context, snapName string, fields string, user *auth.UserState) (*storeInfo, *http.Response, error) {
  1104  	query := url.Values{}
  1105  	query.Set("fields", fields)
  1106  	query.Set("architecture", s.architecture)
  1107  
  1108  	u := s.endpointURL(path.Join(snapInfoEndpPath, snapName), query)
  1109  	reqOptions := &requestOptions{
  1110  		Method:   "GET",
  1111  		URL:      u,
  1112  		APILevel: apiV2Endps,
  1113  	}
  1114  
  1115  	var remote storeInfo
  1116  	resp, err := s.retryRequestDecodeJSON(ctx, reqOptions, user, &remote, nil)
  1117  	if err != nil {
  1118  		return nil, nil, err
  1119  	}
  1120  
  1121  	// check statusCode
  1122  	switch resp.StatusCode {
  1123  	case 200:
  1124  		// OK
  1125  	case 404:
  1126  		return nil, nil, ErrSnapNotFound
  1127  	default:
  1128  		msg := fmt.Sprintf("get details for snap %q", snapName)
  1129  		return nil, nil, respToError(resp, msg)
  1130  	}
  1131  
  1132  	return &remote, resp, err
  1133  }
  1134  
  1135  // SnapInfo checks whether the store-hosted snap matching the given spec exists and returns a reference with it name and snap-id and default channel, or an error.
  1136  func (s *Store) SnapExists(ctx context.Context, snapSpec SnapSpec, user *auth.UserState) (naming.SnapRef, *channel.Channel, error) {
  1137  	// request the minimal amount information
  1138  	fields := "channel-map"
  1139  
  1140  	si, _, err := s.snapInfo(ctx, snapSpec.Name, fields, user)
  1141  	if err != nil {
  1142  		return nil, nil, err
  1143  	}
  1144  
  1145  	return minimalFromStoreInfo(si)
  1146  }
  1147  
  1148  // A Search is what you do in order to Find something
  1149  type Search struct {
  1150  	// Query is a term to search by or a prefix (if Prefix is true)
  1151  	Query  string
  1152  	Prefix bool
  1153  
  1154  	CommonID string
  1155  
  1156  	// category is "section" in search v1
  1157  	Category string
  1158  	Private  bool
  1159  	Scope    string
  1160  }
  1161  
  1162  // Find finds  (installable) snaps from the store, matching the
  1163  // given Search.
  1164  func (s *Store) Find(ctx context.Context, search *Search, user *auth.UserState) ([]*snap.Info, error) {
  1165  	if search.Private && user == nil {
  1166  		return nil, ErrUnauthenticated
  1167  	}
  1168  
  1169  	searchTerm := strings.TrimSpace(search.Query)
  1170  
  1171  	// these characters might have special meaning on the search
  1172  	// server, and don't form part of a reasonable search, so
  1173  	// abort if they're included.
  1174  	//
  1175  	// "-" might also be special on the server, but it's also a
  1176  	// valid part of a package name, so we let it pass
  1177  	if strings.ContainsAny(searchTerm, `+=&|><!(){}[]^"~*?:\/`) {
  1178  		return nil, ErrBadQuery
  1179  	}
  1180  
  1181  	q := url.Values{}
  1182  	q.Set("fields", strings.Join(s.findFields, ","))
  1183  	q.Set("architecture", s.architecture)
  1184  
  1185  	if search.Private {
  1186  		q.Set("private", "true")
  1187  	}
  1188  
  1189  	if search.Prefix {
  1190  		q.Set("name", searchTerm)
  1191  	} else {
  1192  		if search.CommonID != "" {
  1193  			q.Set("common-id", search.CommonID)
  1194  		}
  1195  		if searchTerm != "" {
  1196  			q.Set("q", searchTerm)
  1197  		}
  1198  	}
  1199  
  1200  	if search.Category != "" {
  1201  		q.Set("category", search.Category)
  1202  	}
  1203  
  1204  	// with search v2 all risks are searched by default (same as scope=wide
  1205  	// with v1) so we need to restrict channel if scope is not passed.
  1206  	if search.Scope == "" {
  1207  		q.Set("channel", "stable")
  1208  	} else if search.Scope != "wide" {
  1209  		return nil, ErrInvalidScope
  1210  	}
  1211  
  1212  	if release.OnClassic {
  1213  		q.Set("confinement", "strict,classic")
  1214  	} else {
  1215  		q.Set("confinement", "strict")
  1216  	}
  1217  
  1218  	u := s.endpointURL(findEndpPath, q)
  1219  	reqOptions := &requestOptions{
  1220  		Method:   "GET",
  1221  		URL:      u,
  1222  		Accept:   jsonContentType,
  1223  		APILevel: apiV2Endps,
  1224  	}
  1225  
  1226  	var searchData searchV2Results
  1227  
  1228  	// TODO: use retryRequestDecodeJSON (may require content-type check there,
  1229  	// requires checking other handlers, their tests and store).
  1230  	doRequest := func() (*http.Response, error) {
  1231  		return s.doRequest(ctx, s.client, reqOptions, user)
  1232  	}
  1233  	readResponse := func(resp *http.Response) error {
  1234  		ok := (resp.StatusCode == 200 || resp.StatusCode == 201)
  1235  		ct := resp.Header.Get("Content-Type")
  1236  		// always decode on success; decode failures only if body is not empty
  1237  		if !ok && (resp.ContentLength == 0 || ct != jsonContentType) {
  1238  			return nil
  1239  		}
  1240  		return json.NewDecoder(resp.Body).Decode(&searchData)
  1241  	}
  1242  	resp, err := httputil.RetryRequest(u.String(), doRequest, readResponse, defaultRetryStrategy)
  1243  	if err != nil {
  1244  		return nil, err
  1245  	}
  1246  
  1247  	if resp.StatusCode != 200 {
  1248  		// fallback to search v1; v2 may not be available on some proxies
  1249  		if resp.StatusCode == 404 {
  1250  			verstr := resp.Header.Get("Snap-Store-Version")
  1251  			ver, err := strconv.Atoi(verstr)
  1252  			if err != nil {
  1253  				logger.Debugf("Bogus Snap-Store-Version header %q.", verstr)
  1254  			} else if ver < 20 {
  1255  				return s.findV1(ctx, search, user)
  1256  			}
  1257  		}
  1258  		if len(searchData.ErrorList) > 0 {
  1259  			if len(searchData.ErrorList) > 1 {
  1260  				logger.Noticef("unexpected number of errors (%d) when trying to search via %q", len(searchData.ErrorList), resp.Request.URL)
  1261  			}
  1262  			return nil, translateSnapActionError("", "", searchData.ErrorList[0].Code, searchData.ErrorList[0].Message, nil)
  1263  		}
  1264  		return nil, respToError(resp, "search")
  1265  	}
  1266  
  1267  	if ct := resp.Header.Get("Content-Type"); ct != jsonContentType {
  1268  		return nil, fmt.Errorf("received an unexpected content type (%q) when trying to search via %q", ct, resp.Request.URL)
  1269  	}
  1270  
  1271  	snaps := make([]*snap.Info, len(searchData.Results))
  1272  	for i, res := range searchData.Results {
  1273  		info, err := infoFromStoreSearchResult(res)
  1274  		if err != nil {
  1275  			return nil, err
  1276  		}
  1277  		snaps[i] = info
  1278  	}
  1279  
  1280  	err = s.decorateOrders(snaps, user)
  1281  	if err != nil {
  1282  		logger.Noticef("cannot get user orders: %v", err)
  1283  	}
  1284  
  1285  	s.extractSuggestedCurrency(resp)
  1286  
  1287  	return snaps, nil
  1288  }
  1289  
  1290  func (s *Store) findV1(ctx context.Context, search *Search, user *auth.UserState) ([]*snap.Info, error) {
  1291  	// search.Query is already verified for illegal characters by Find()
  1292  	searchTerm := strings.TrimSpace(search.Query)
  1293  	q := s.defaultSnapQuery()
  1294  
  1295  	if search.Private {
  1296  		q.Set("private", "true")
  1297  	}
  1298  
  1299  	if search.Prefix {
  1300  		q.Set("name", searchTerm)
  1301  	} else {
  1302  		if search.CommonID != "" {
  1303  			q.Set("common_id", search.CommonID)
  1304  		}
  1305  		if searchTerm != "" {
  1306  			q.Set("q", searchTerm)
  1307  		}
  1308  	}
  1309  
  1310  	// category was "section" in search v1
  1311  	if search.Category != "" {
  1312  		q.Set("section", search.Category)
  1313  	}
  1314  	if search.Scope != "" {
  1315  		q.Set("scope", search.Scope)
  1316  	}
  1317  
  1318  	if release.OnClassic {
  1319  		q.Set("confinement", "strict,classic")
  1320  	} else {
  1321  		q.Set("confinement", "strict")
  1322  	}
  1323  
  1324  	u := s.endpointURL(searchEndpPath, q)
  1325  	reqOptions := &requestOptions{
  1326  		Method: "GET",
  1327  		URL:    u,
  1328  		Accept: halJsonContentType,
  1329  	}
  1330  
  1331  	var searchData searchResults
  1332  	resp, err := s.retryRequestDecodeJSON(ctx, reqOptions, user, &searchData, nil)
  1333  	if err != nil {
  1334  		return nil, err
  1335  	}
  1336  
  1337  	if resp.StatusCode != 200 {
  1338  		return nil, respToError(resp, "search")
  1339  	}
  1340  
  1341  	if ct := resp.Header.Get("Content-Type"); ct != halJsonContentType {
  1342  		return nil, fmt.Errorf("received an unexpected content type (%q) when trying to search via %q", ct, resp.Request.URL)
  1343  	}
  1344  
  1345  	snaps := make([]*snap.Info, len(searchData.Payload.Packages))
  1346  	for i, pkg := range searchData.Payload.Packages {
  1347  		snaps[i] = infoFromRemote(pkg)
  1348  	}
  1349  
  1350  	err = s.decorateOrders(snaps, user)
  1351  	if err != nil {
  1352  		logger.Noticef("cannot get user orders: %v", err)
  1353  	}
  1354  
  1355  	s.extractSuggestedCurrency(resp)
  1356  
  1357  	return snaps, nil
  1358  }
  1359  
  1360  // Sections retrieves the list of available store sections.
  1361  func (s *Store) Sections(ctx context.Context, user *auth.UserState) ([]string, error) {
  1362  	reqOptions := &requestOptions{
  1363  		Method:         "GET",
  1364  		URL:            s.endpointURL(sectionsEndpPath, nil),
  1365  		Accept:         halJsonContentType,
  1366  		DeviceAuthNeed: deviceAuthCustomStoreOnly,
  1367  	}
  1368  
  1369  	var sectionData sectionResults
  1370  	resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, &sectionData, nil)
  1371  	if err != nil {
  1372  		return nil, err
  1373  	}
  1374  
  1375  	if resp.StatusCode != 200 {
  1376  		return nil, respToError(resp, "sections")
  1377  	}
  1378  
  1379  	if ct := resp.Header.Get("Content-Type"); ct != halJsonContentType {
  1380  		return nil, fmt.Errorf("received an unexpected content type (%q) when trying to retrieve the sections via %q", ct, resp.Request.URL)
  1381  	}
  1382  
  1383  	var sectionNames []string
  1384  	for _, s := range sectionData.Payload.Sections {
  1385  		sectionNames = append(sectionNames, s.Name)
  1386  	}
  1387  
  1388  	return sectionNames, nil
  1389  }
  1390  
  1391  // WriteCatalogs queries the "commands" endpoint and writes the
  1392  // command names into the given io.Writer.
  1393  func (s *Store) WriteCatalogs(ctx context.Context, names io.Writer, adder SnapAdder) error {
  1394  	u := *s.endpointURL(commandsEndpPath, nil)
  1395  
  1396  	q := u.Query()
  1397  	if release.OnClassic {
  1398  		q.Set("confinement", "strict,classic")
  1399  	} else {
  1400  		q.Set("confinement", "strict")
  1401  	}
  1402  
  1403  	u.RawQuery = q.Encode()
  1404  	reqOptions := &requestOptions{
  1405  		Method:         "GET",
  1406  		URL:            &u,
  1407  		Accept:         halJsonContentType,
  1408  		DeviceAuthNeed: deviceAuthCustomStoreOnly,
  1409  	}
  1410  
  1411  	// do not log body for catalog updates (its huge)
  1412  	client := s.newHTTPClient(&httputil.ClientOptions{
  1413  		MayLogBody: false,
  1414  		Timeout:    10 * time.Second,
  1415  	})
  1416  	doRequest := func() (*http.Response, error) {
  1417  		return s.doRequest(ctx, client, reqOptions, nil)
  1418  	}
  1419  	readResponse := func(resp *http.Response) error {
  1420  		return decodeCatalog(resp, names, adder)
  1421  	}
  1422  
  1423  	resp, err := httputil.RetryRequest(u.String(), doRequest, readResponse, defaultRetryStrategy)
  1424  	if err != nil {
  1425  		return err
  1426  	}
  1427  	if resp.StatusCode != 200 {
  1428  		return respToError(resp, "refresh commands catalog")
  1429  	}
  1430  
  1431  	return nil
  1432  }
  1433  
  1434  // SuggestedCurrency retrieves the cached value for the store's suggested currency
  1435  func (s *Store) SuggestedCurrency() string {
  1436  	s.mu.Lock()
  1437  	defer s.mu.Unlock()
  1438  
  1439  	if s.suggestedCurrency == "" {
  1440  		return "USD"
  1441  	}
  1442  	return s.suggestedCurrency
  1443  }
  1444  
  1445  // orderInstruction holds data sent to the store for orders.
  1446  type orderInstruction struct {
  1447  	SnapID   string `json:"snap_id"`
  1448  	Amount   string `json:"amount,omitempty"`
  1449  	Currency string `json:"currency,omitempty"`
  1450  }
  1451  
  1452  type storeError struct {
  1453  	Code    string `json:"code"`
  1454  	Message string `json:"message"`
  1455  }
  1456  
  1457  func (s *storeError) Error() string {
  1458  	return s.Message
  1459  }
  1460  
  1461  type storeErrors struct {
  1462  	Errors []*storeError `json:"error_list"`
  1463  }
  1464  
  1465  func (s *storeErrors) Code() string {
  1466  	if len(s.Errors) == 0 {
  1467  		return ""
  1468  	}
  1469  	return s.Errors[0].Code
  1470  }
  1471  
  1472  func (s *storeErrors) Error() string {
  1473  	if len(s.Errors) == 0 {
  1474  		return "internal error: empty store error used as an actual error"
  1475  	}
  1476  	return s.Errors[0].Error()
  1477  }
  1478  
  1479  func buyOptionError(message string) (*client.BuyResult, error) {
  1480  	return nil, fmt.Errorf("cannot buy snap: %s", message)
  1481  }
  1482  
  1483  // Buy sends a buy request for the specified snap.
  1484  // Returns the state of the order: Complete, Cancelled.
  1485  func (s *Store) Buy(options *client.BuyOptions, user *auth.UserState) (*client.BuyResult, error) {
  1486  	if options.SnapID == "" {
  1487  		return buyOptionError("snap ID missing")
  1488  	}
  1489  	if options.Price <= 0 {
  1490  		return buyOptionError("invalid expected price")
  1491  	}
  1492  	if options.Currency == "" {
  1493  		return buyOptionError("currency missing")
  1494  	}
  1495  	if user == nil {
  1496  		return nil, ErrUnauthenticated
  1497  	}
  1498  
  1499  	instruction := orderInstruction{
  1500  		SnapID:   options.SnapID,
  1501  		Amount:   fmt.Sprintf("%.2f", options.Price),
  1502  		Currency: options.Currency,
  1503  	}
  1504  
  1505  	jsonData, err := json.Marshal(instruction)
  1506  	if err != nil {
  1507  		return nil, err
  1508  	}
  1509  
  1510  	reqOptions := &requestOptions{
  1511  		Method:      "POST",
  1512  		URL:         s.endpointURL(buyEndpPath, nil),
  1513  		Accept:      jsonContentType,
  1514  		ContentType: jsonContentType,
  1515  		Data:        jsonData,
  1516  	}
  1517  
  1518  	var orderDetails order
  1519  	var errorInfo storeErrors
  1520  	resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, &orderDetails, &errorInfo)
  1521  	if err != nil {
  1522  		return nil, err
  1523  	}
  1524  
  1525  	switch resp.StatusCode {
  1526  	case 200, 201:
  1527  		// user already ordered or order successful
  1528  		if orderDetails.State == "Cancelled" {
  1529  			return buyOptionError("payment cancelled")
  1530  		}
  1531  
  1532  		return &client.BuyResult{
  1533  			State: orderDetails.State,
  1534  		}, nil
  1535  	case 400:
  1536  		// Invalid price was specified, etc.
  1537  		return buyOptionError(fmt.Sprintf("bad request: %v", errorInfo.Error()))
  1538  	case 403:
  1539  		// Customer account not set up for purchases.
  1540  		switch errorInfo.Code() {
  1541  		case "no-payment-methods":
  1542  			return nil, ErrNoPaymentMethods
  1543  		case "tos-not-accepted":
  1544  			return nil, ErrTOSNotAccepted
  1545  		}
  1546  		return buyOptionError(fmt.Sprintf("permission denied: %v", errorInfo.Error()))
  1547  	case 404:
  1548  		// Likely because customer account or snap ID doesn't exist.
  1549  		return buyOptionError(fmt.Sprintf("server says not found: %v", errorInfo.Error()))
  1550  	case 402: // Payment Required
  1551  		// Payment failed for some reason.
  1552  		return nil, ErrPaymentDeclined
  1553  	case 401:
  1554  		// TODO handle token expiry and refresh
  1555  		return nil, ErrInvalidCredentials
  1556  	default:
  1557  		return nil, respToError(resp, fmt.Sprintf("buy snap: %v", errorInfo))
  1558  	}
  1559  }
  1560  
  1561  type storeCustomer struct {
  1562  	LatestTOSDate     string `json:"latest_tos_date"`
  1563  	AcceptedTOSDate   string `json:"accepted_tos_date"`
  1564  	LatestTOSAccepted bool   `json:"latest_tos_accepted"`
  1565  	HasPaymentMethod  bool   `json:"has_payment_method"`
  1566  }
  1567  
  1568  // ReadyToBuy returns nil if the user's account has accepted T&Cs and has a payment method registered, and an error otherwise
  1569  func (s *Store) ReadyToBuy(user *auth.UserState) error {
  1570  	if user == nil {
  1571  		return ErrUnauthenticated
  1572  	}
  1573  
  1574  	reqOptions := &requestOptions{
  1575  		Method: "GET",
  1576  		URL:    s.endpointURL(customersMeEndpPath, nil),
  1577  		Accept: jsonContentType,
  1578  	}
  1579  
  1580  	var customer storeCustomer
  1581  	var errors storeErrors
  1582  	resp, err := s.retryRequestDecodeJSON(context.TODO(), reqOptions, user, &customer, &errors)
  1583  	if err != nil {
  1584  		return err
  1585  	}
  1586  
  1587  	switch resp.StatusCode {
  1588  	case 200:
  1589  		if !customer.HasPaymentMethod {
  1590  			return ErrNoPaymentMethods
  1591  		}
  1592  		if !customer.LatestTOSAccepted {
  1593  			return ErrTOSNotAccepted
  1594  		}
  1595  		return nil
  1596  	case 404:
  1597  		// Likely because user has no account registered on the pay server
  1598  		return fmt.Errorf("cannot get customer details: server says no account exists")
  1599  	case 401:
  1600  		return ErrInvalidCredentials
  1601  	default:
  1602  		if len(errors.Errors) == 0 {
  1603  			return fmt.Errorf("cannot get customer details: unexpected HTTP code %d", resp.StatusCode)
  1604  		}
  1605  		return &errors
  1606  	}
  1607  }
  1608  
  1609  // abbreviated info structs just for the download info
  1610  type storeInfoChannelAbbrev struct {
  1611  	Download storeSnapDownload `json:"download"`
  1612  }
  1613  
  1614  type storeInfoAbbrev struct {
  1615  	// discard anything beyond the first entry
  1616  	ChannelMap [1]storeInfoChannelAbbrev `json:"channel-map"`
  1617  }
  1618  
  1619  var errUnexpectedConnCheckResponse = errors.New("unexpected response during connection check")
  1620  
  1621  func (s *Store) snapConnCheck() ([]string, error) {
  1622  	var hosts []string
  1623  	// NOTE: "core" is possibly the only snap that's sure to be in all stores
  1624  	//       when we drop "core" in the move to snapd/core18/etc, change this
  1625  	infoURL := s.endpointURL(path.Join(snapInfoEndpPath, "core"), url.Values{
  1626  		// we only want the download URL
  1627  		"fields": {"download"},
  1628  		// we only need *one* (but can't filter by channel ... yet)
  1629  		"architecture": {s.architecture},
  1630  	})
  1631  	hosts = append(hosts, infoURL.Host)
  1632  
  1633  	var result storeInfoAbbrev
  1634  	resp, err := httputil.RetryRequest(infoURL.String(), func() (*http.Response, error) {
  1635  		return s.doRequest(context.TODO(), s.client, &requestOptions{
  1636  			Method:   "GET",
  1637  			URL:      infoURL,
  1638  			APILevel: apiV2Endps,
  1639  		}, nil)
  1640  	}, func(resp *http.Response) error {
  1641  		return decodeJSONBody(resp, &result, nil)
  1642  	}, connCheckStrategy)
  1643  
  1644  	if err != nil {
  1645  		return hosts, err
  1646  	}
  1647  	resp.Body.Close()
  1648  
  1649  	dlURLraw := result.ChannelMap[0].Download.URL
  1650  	dlURL, err := url.ParseRequestURI(dlURLraw)
  1651  	if err != nil {
  1652  		return hosts, err
  1653  	}
  1654  	hosts = append(hosts, dlURL.Host)
  1655  
  1656  	cdnHeader, err := s.cdnHeader()
  1657  	if err != nil {
  1658  		return hosts, err
  1659  	}
  1660  
  1661  	reqOptions := downloadReqOpts(dlURL, cdnHeader, nil)
  1662  	reqOptions.Method = "HEAD" // not actually a download
  1663  
  1664  	// TODO: We need the HEAD here so that we get redirected to the
  1665  	//       right CDN machine. Consider just doing a "net.Dial"
  1666  	//       after the redirect here. Suggested in
  1667  	// https://github.com/snapcore/snapd/pull/5176#discussion_r193437230
  1668  	resp, err = httputil.RetryRequest(dlURLraw, func() (*http.Response, error) {
  1669  		return s.doRequest(context.TODO(), s.client, reqOptions, nil)
  1670  	}, func(resp *http.Response) error {
  1671  		// account for redirect
  1672  		hosts[len(hosts)-1] = resp.Request.URL.Host
  1673  		return nil
  1674  	}, connCheckStrategy)
  1675  	if err != nil {
  1676  		return hosts, err
  1677  	}
  1678  	resp.Body.Close()
  1679  
  1680  	if resp.StatusCode != 200 {
  1681  		return hosts, errUnexpectedConnCheckResponse
  1682  	}
  1683  
  1684  	return hosts, nil
  1685  }
  1686  
  1687  func (s *Store) ConnectivityCheck() (status map[string]bool, err error) {
  1688  	status = make(map[string]bool)
  1689  
  1690  	checkers := []func() ([]string, error){
  1691  		s.snapConnCheck,
  1692  	}
  1693  
  1694  	for _, checker := range checkers {
  1695  		hosts, err := checker()
  1696  		for _, host := range hosts {
  1697  			status[host] = (err == nil)
  1698  		}
  1699  	}
  1700  
  1701  	return status, nil
  1702  }
  1703  
  1704  func (s *Store) CreateCohorts(ctx context.Context, snaps []string) (map[string]string, error) {
  1705  	jsonData, err := json.Marshal(map[string][]string{"snaps": snaps})
  1706  	if err != nil {
  1707  		return nil, err
  1708  	}
  1709  
  1710  	u := s.endpointURL(cohortsEndpPath, nil)
  1711  	reqOptions := &requestOptions{
  1712  		Method:   "POST",
  1713  		URL:      u,
  1714  		APILevel: apiV2Endps,
  1715  		Data:     jsonData,
  1716  	}
  1717  
  1718  	var remote struct {
  1719  		CohortKeys map[string]string `json:"cohort-keys"`
  1720  	}
  1721  	resp, err := s.retryRequestDecodeJSON(ctx, reqOptions, nil, &remote, nil)
  1722  	if err != nil {
  1723  		return nil, err
  1724  	}
  1725  	switch resp.StatusCode {
  1726  	case 200:
  1727  		// OK
  1728  	case 404:
  1729  		return nil, ErrSnapNotFound
  1730  	default:
  1731  		return nil, respToError(resp, fmt.Sprintf("create cohorts for %s", strutil.Quoted(snaps)))
  1732  	}
  1733  
  1734  	return remote.CohortKeys, nil
  1735  }