github.com/freetocompute/snapd@v0.0.0-20210618182524-2fb355d72fd9/overlord/assertstate/bulk.go (about)

     1  // -*- Mode: Go; indent-tabs-mode: t -*-
     2  
     3  /*
     4   * Copyright (C) 2020 Canonical Ltd
     5   *
     6   * This program is free software: you can redistribute it and/or modify
     7   * it under the terms of the GNU General Public License version 3 as
     8   * published by the Free Software Foundation.
     9   *
    10   * This program is distributed in the hope that it will be useful,
    11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
    12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    13   * GNU General Public License for more details.
    14   *
    15   * You should have received a copy of the GNU General Public License
    16   * along with this program.  If not, see <http://www.gnu.org/licenses/>.
    17   *
    18   */
    19  
    20  package assertstate
    21  
    22  import (
    23  	"context"
    24  	"fmt"
    25  	"sort"
    26  	"strings"
    27  
    28  	"github.com/snapcore/snapd/asserts"
    29  	"github.com/snapcore/snapd/overlord/snapstate"
    30  	"github.com/snapcore/snapd/overlord/state"
    31  	"github.com/snapcore/snapd/release"
    32  	"github.com/snapcore/snapd/store"
    33  )
    34  
    35  const storeGroup = "store assertion"
    36  
    37  // maxGroups is the maximum number of assertion groups we set with the
    38  // asserts.Pool used to refresh snap assertions, it corresponds
    39  // roughly to for how many snaps we will request assertions in
    40  // in one /v2/snaps/refresh request.
    41  // Given that requesting assertions for ~500 snaps together with no
    42  // updates can take around 900ms-1s, conservatively set it to half of
    43  // that. Most systems should be done in one request anyway.
    44  var maxGroups = 256
    45  
    46  func bulkRefreshSnapDeclarations(s *state.State, snapStates map[string]*snapstate.SnapState, userID int, deviceCtx snapstate.DeviceContext) error {
    47  	db := cachedDB(s)
    48  
    49  	pool := asserts.NewPool(db, maxGroups)
    50  
    51  	var mergedRPErr *resolvePoolError
    52  	tryResolvePool := func() error {
    53  		err := resolvePool(s, pool, userID, deviceCtx)
    54  		if rpe, ok := err.(*resolvePoolError); ok {
    55  			if mergedRPErr == nil {
    56  				mergedRPErr = rpe
    57  			} else {
    58  				mergedRPErr.merge(rpe)
    59  			}
    60  			return nil
    61  		}
    62  		return err
    63  	}
    64  
    65  	c := 0
    66  	for instanceName, snapst := range snapStates {
    67  		sideInfo := snapst.CurrentSideInfo()
    68  		if sideInfo.SnapID == "" {
    69  			continue
    70  		}
    71  
    72  		declRef := &asserts.Ref{
    73  			Type:       asserts.SnapDeclarationType,
    74  			PrimaryKey: []string{release.Series, sideInfo.SnapID},
    75  		}
    76  		// update snap-declaration (and prereqs) for the snap,
    77  		// they were originally added at install time
    78  		if err := pool.AddToUpdate(declRef, instanceName); err != nil {
    79  			return fmt.Errorf("cannot prepare snap-declaration refresh for snap %q: %v", instanceName, err)
    80  		}
    81  
    82  		c++
    83  		if c%maxGroups == 0 {
    84  			// we have exhausted max groups, resolve
    85  			// what we setup so far and then clear groups
    86  			// to reuse the pool
    87  			if err := tryResolvePool(); err != nil {
    88  				return err
    89  			}
    90  			if err := pool.ClearGroups(); err != nil {
    91  				// this shouldn't happen but if it
    92  				// does fallback
    93  				return &bulkAssertionFallbackError{err}
    94  			}
    95  		}
    96  	}
    97  
    98  	modelAs := deviceCtx.Model()
    99  
   100  	// fetch store assertion if available
   101  	if modelAs.Store() != "" {
   102  		storeRef := asserts.Ref{
   103  			Type:       asserts.StoreType,
   104  			PrimaryKey: []string{modelAs.Store()},
   105  		}
   106  		if err := pool.AddToUpdate(&storeRef, storeGroup); err != nil {
   107  			if !asserts.IsNotFound(err) {
   108  				return fmt.Errorf("cannot prepare store assertion refresh: %v", err)
   109  			}
   110  			// assertion is not present in the db yet,
   111  			// we'll try to resolve it (fetch it) first
   112  			storeAt := &asserts.AtRevision{
   113  				Ref:      storeRef,
   114  				Revision: asserts.RevisionNotKnown,
   115  			}
   116  			err := pool.AddUnresolved(storeAt, storeGroup)
   117  			if err != nil {
   118  				return fmt.Errorf("cannot prepare store assertion fetching: %v", err)
   119  			}
   120  		}
   121  	}
   122  
   123  	if err := tryResolvePool(); err != nil {
   124  		return err
   125  	}
   126  
   127  	if mergedRPErr != nil {
   128  		if e := mergedRPErr.errors[storeGroup]; asserts.IsNotFound(e) || e == asserts.ErrUnresolved {
   129  			// ignore
   130  			delete(mergedRPErr.errors, storeGroup)
   131  		}
   132  		if len(mergedRPErr.errors) == 0 {
   133  			return nil
   134  		}
   135  		mergedRPErr.message = "cannot refresh snap-declarations for snaps"
   136  		return mergedRPErr
   137  	}
   138  
   139  	return nil
   140  }
   141  
   142  func bulkRefreshValidationSetAsserts(s *state.State, vsets map[string]*ValidationSetTracking, userID int, deviceCtx snapstate.DeviceContext) error {
   143  	db := cachedDB(s)
   144  	pool := asserts.NewPool(db, maxGroups)
   145  
   146  	ignoreNotFound := make(map[string]bool)
   147  
   148  	for _, vs := range vsets {
   149  		var atSeq *asserts.AtSequence
   150  		if vs.PinnedAt > 0 {
   151  			// pinned to specific sequence, update to latest revision for same
   152  			// sequence.
   153  			atSeq = &asserts.AtSequence{
   154  				Type:        asserts.ValidationSetType,
   155  				SequenceKey: []string{release.Series, vs.AccountID, vs.Name},
   156  				Sequence:    vs.PinnedAt,
   157  				Pinned:      true,
   158  			}
   159  		} else {
   160  			// not pinned, update to latest sequence
   161  			atSeq = &asserts.AtSequence{
   162  				Type:        asserts.ValidationSetType,
   163  				SequenceKey: []string{release.Series, vs.AccountID, vs.Name},
   164  				Sequence:    vs.Current,
   165  			}
   166  		}
   167  		// every sequence to resolve has own group
   168  		group := atSeq.Unique()
   169  		if vs.LocalOnly {
   170  			ignoreNotFound[group] = true
   171  		}
   172  		if err := pool.AddSequenceToUpdate(atSeq, group); err != nil {
   173  			return err
   174  		}
   175  	}
   176  
   177  	err := resolvePoolNoFallback(s, pool, userID, deviceCtx)
   178  	if err == nil {
   179  		return nil
   180  	}
   181  
   182  	if rerr, ok := err.(*resolvePoolError); ok {
   183  		// ignore resolving errors for validation sets that are local only (no
   184  		// assertion in the store).
   185  		for group := range ignoreNotFound {
   186  			if e := rerr.errors[group]; asserts.IsNotFound(e) || e == asserts.ErrUnresolved {
   187  				delete(rerr.errors, group)
   188  			}
   189  		}
   190  		if len(rerr.errors) == 0 {
   191  			return nil
   192  		}
   193  	}
   194  
   195  	return fmt.Errorf("cannot refresh validation set assertions: %v", err)
   196  }
   197  
   198  // marker error to request falling back to the old implemention for assertion
   199  // refreshes
   200  type bulkAssertionFallbackError struct {
   201  	err error
   202  }
   203  
   204  func (e *bulkAssertionFallbackError) Error() string {
   205  	return fmt.Sprintf("unsuccessful bulk assertion refresh, fallback: %v", e.err)
   206  }
   207  
   208  type resolvePoolError struct {
   209  	message string
   210  	// errors maps groups to errors
   211  	errors map[string]error
   212  }
   213  
   214  func (rpe *resolvePoolError) merge(rpe1 *resolvePoolError) {
   215  	// we expect usually rpe and rpe1 errors to be disjunct, but is also
   216  	// ok for rpe1 errors to win
   217  	for k, e := range rpe1.errors {
   218  		rpe.errors[k] = e
   219  	}
   220  }
   221  
   222  func (rpe *resolvePoolError) Error() string {
   223  	message := rpe.message
   224  	if message == "" {
   225  		message = "cannot fetch and resolve assertions"
   226  	}
   227  	s := make([]string, 0, 1+len(rpe.errors))
   228  	s = append(s, fmt.Sprintf("%s:", message))
   229  	groups := make([]string, 0, len(rpe.errors))
   230  	for g := range rpe.errors {
   231  		groups = append(groups, g)
   232  	}
   233  	sort.Strings(groups)
   234  	for _, g := range groups {
   235  		s = append(s, fmt.Sprintf(" - %s: %v", g, rpe.errors[g]))
   236  	}
   237  	return strings.Join(s, "\n")
   238  }
   239  
   240  func resolvePool(s *state.State, pool *asserts.Pool, userID int, deviceCtx snapstate.DeviceContext) error {
   241  	user, err := userFromUserID(s, userID)
   242  	if err != nil {
   243  		return err
   244  	}
   245  	sto := snapstate.Store(s, deviceCtx)
   246  	db := cachedDB(s)
   247  	unsupported := handleUnsupported(db)
   248  
   249  	for {
   250  		// TODO: pass refresh options?
   251  		s.Unlock()
   252  		_, aresults, err := sto.SnapAction(context.TODO(), nil, nil, pool, user, nil)
   253  		s.Lock()
   254  		if err != nil {
   255  			// request fallback on
   256  			//  * unexpected SnapActionErrors or
   257  			//  * unexpected HTTP status of 4xx or 500
   258  			ignore := false
   259  			switch stoErr := err.(type) {
   260  			case *store.SnapActionError:
   261  				if !stoErr.NoResults || len(stoErr.Other) != 0 {
   262  					return &bulkAssertionFallbackError{stoErr}
   263  				}
   264  				// simply no results error, we are likely done
   265  				ignore = true
   266  			case *store.UnexpectedHTTPStatusError:
   267  				if stoErr.StatusCode >= 400 && stoErr.StatusCode <= 500 {
   268  					return &bulkAssertionFallbackError{stoErr}
   269  				}
   270  			}
   271  			if !ignore {
   272  				return err
   273  			}
   274  		}
   275  		if len(aresults) == 0 {
   276  			// everything resolved if no errors
   277  			break
   278  		}
   279  
   280  		for _, ares := range aresults {
   281  			b := asserts.NewBatch(unsupported)
   282  			s.Unlock()
   283  			err := sto.DownloadAssertions(ares.StreamURLs, b, user)
   284  			s.Lock()
   285  			if err != nil {
   286  				pool.AddGroupingError(err, ares.Grouping)
   287  				continue
   288  			}
   289  			_, err = pool.AddBatch(b, ares.Grouping)
   290  			if err != nil {
   291  				return err
   292  			}
   293  		}
   294  	}
   295  
   296  	pool.CommitTo(db)
   297  
   298  	errors := pool.Errors()
   299  	if len(errors) != 0 {
   300  		return &resolvePoolError{errors: errors}
   301  	}
   302  
   303  	return nil
   304  }
   305  
   306  func resolvePoolNoFallback(s *state.State, pool *asserts.Pool, userID int, deviceCtx snapstate.DeviceContext) error {
   307  	err := resolvePool(s, pool, userID, deviceCtx)
   308  	if err != nil {
   309  		// no fallback, report inner error.
   310  		if ferr, ok := err.(*bulkAssertionFallbackError); ok {
   311  			err = ferr.err
   312  		}
   313  	}
   314  	return err
   315  }