github.com/safing/portbase@v0.19.5/updater/updating.go

package updater

import (
	"context"
	"fmt"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"

	"golang.org/x/exp/slices"

	"github.com/safing/jess/filesig"
	"github.com/safing/jess/lhash"
	"github.com/safing/portbase/log"
	"github.com/safing/portbase/utils"
)

// UpdateIndexes downloads all indexes. An error is only returned when all
// indexes fail to update.
func (reg *ResourceRegistry) UpdateIndexes(ctx context.Context) error {
	var lastErr error
	var anySuccess bool

	// Start registry operation.
	reg.state.StartOperation(StateChecking)
	defer reg.state.EndOperation()

	client := &http.Client{}
	for _, idx := range reg.getIndexes() {
		if err := reg.downloadIndex(ctx, client, idx); err != nil {
			lastErr = err
			log.Warningf("%s: failed to update index %s: %s", reg.Name, idx.Path, err)
		} else {
			anySuccess = true
		}
	}

	// If all indexes failed to update, fail.
	if !anySuccess {
		err := fmt.Errorf("failed to update all indexes, last error was: %w", lastErr)
		reg.state.ReportUpdateCheck(nil, err)
		return err
	}

	// Get pending resources and update status.
	pendingResourceVersions, _ := reg.GetPendingDownloads(true, false)
	reg.state.ReportUpdateCheck(
		humanInfoFromResourceVersions(pendingResourceVersions),
		nil,
	)

	return nil
}

func (reg *ResourceRegistry) downloadIndex(ctx context.Context, client *http.Client, idx *Index) error {
	var (
		// Index.
		indexErr    error
		indexData   []byte
		downloadURL string

		// Signature.
		sigErr       error
		verifiedHash *lhash.LabeledHash
		sigFileData  []byte
		verifOpts    = reg.GetVerificationOptions(idx.Path)
	)

	// Upgrade to v2 index if verification is enabled.
	downloadIndexPath := idx.Path
	if verifOpts != nil {
		downloadIndexPath = strings.TrimSuffix(downloadIndexPath, baseIndexExtension) + v2IndexExtension
	}

	// Download new index and signature.
	for tries := 0; tries < 3; tries++ {
		// Index and signature need to be fetched together, so that they are
		// fetched from the same source. One source should always have a matching
		// index and signature. Backup sources may be behind a little.
		// If the signature verification fails, another source should be tried.

		// Get index data.
		indexData, downloadURL, indexErr = reg.fetchData(ctx, client, downloadIndexPath, tries)
		if indexErr != nil {
			log.Debugf("%s: failed to fetch index %s: %s", reg.Name, downloadURL, indexErr)
			continue
		}

		// Get signature and verify it.
		if verifOpts != nil {
			verifiedHash, sigFileData, sigErr = reg.fetchAndVerifySigFile(
				ctx, client,
				verifOpts, downloadIndexPath+filesig.Extension, nil,
				tries,
			)
			if sigErr != nil {
				log.Debugf("%s: failed to verify signature of %s: %s", reg.Name, downloadURL, sigErr)
				continue
			}

			// Check if the index matches the verified hash.
			if verifiedHash.Matches(indexData) {
				log.Infof("%s: verified signature of %s", reg.Name, downloadURL)
			} else {
				sigErr = ErrIndexChecksumMismatch
				log.Debugf("%s: checksum does not match file from %s", reg.Name, downloadURL)
				continue
			}
		}

		break
	}
	if indexErr != nil {
		return fmt.Errorf("failed to fetch index %s: %w", downloadIndexPath, indexErr)
	}
	if sigErr != nil {
		return fmt.Errorf("failed to fetch or verify index %s signature: %w", downloadIndexPath, sigErr)
	}

	// Parse the index file.
	indexFile, err := ParseIndexFile(indexData, idx.Channel, idx.LastRelease)
	if err != nil {
		return fmt.Errorf("failed to parse index %s: %w", idx.Path, err)
	}

	// Add index data to registry.
	if len(indexFile.Releases) > 0 {
		// Check if all resources are within the indexes' authority.
		authoritativePath := path.Dir(idx.Path) + "/"
		if authoritativePath == "./" {
			// Fix path for indexes at the storage root.
			authoritativePath = ""
		}
		cleanedData := make(map[string]string, len(indexFile.Releases))
		for key, version := range indexFile.Releases {
			if strings.HasPrefix(key, authoritativePath) {
				cleanedData[key] = version
			} else {
				log.Warningf("%s: index %s oversteps it's authority by defining version for %s", reg.Name, idx.Path, key)
			}
		}

		// add resources to registry
		err = reg.AddResources(cleanedData, idx, false, true, idx.PreRelease)
		if err != nil {
			log.Warningf("%s: failed to add resources: %s", reg.Name, err)
		}
	} else {
		log.Debugf("%s: index %s is empty", reg.Name, idx.Path)
	}

	// Check if dest dir exists.
	indexDir := filepath.FromSlash(path.Dir(idx.Path))
	err = reg.storageDir.EnsureRelPath(indexDir)
	if err != nil {
		log.Warningf("%s: failed to ensure directory for updated index %s: %s", reg.Name, idx.Path, err)
	}

	// Index files must be readable by portmaster-staert with user permissions in order to load the index.
	err = os.WriteFile( //nolint:gosec
		filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)),
		indexData, 0o0644,
	)
	if err != nil {
		log.Warningf("%s: failed to save updated index %s: %s", reg.Name, idx.Path, err)
	}

	// Write signature file, if we have one.
	if len(sigFileData) > 0 {
		err = os.WriteFile( //nolint:gosec
			filepath.Join(reg.storageDir.Path, filepath.FromSlash(idx.Path)+filesig.Extension),
			sigFileData, 0o0644,
		)
		if err != nil {
			log.Warningf("%s: failed to save updated index signature %s: %s", reg.Name, idx.Path+filesig.Extension, err)
		}
	}

	log.Infof("%s: updated index %s with %d entries", reg.Name, idx.Path, len(indexFile.Releases))
	return nil
}

// DownloadUpdates checks if updates are available and downloads updates of used components.
func (reg *ResourceRegistry) DownloadUpdates(ctx context.Context, includeManual bool) error {
	// Start registry operation.
	reg.state.StartOperation(StateDownloading)
	defer reg.state.EndOperation()

	// Get pending updates.
	toUpdate, missingSigs := reg.GetPendingDownloads(includeManual, true)
	downloadDetailsResources := humanInfoFromResourceVersions(toUpdate)
	reg.state.UpdateOperationDetails(&StateDownloadingDetails{
		Resources: downloadDetailsResources,
	})

	// nothing to update
	if len(toUpdate) == 0 && len(missingSigs) == 0 {
		log.Infof("%s: everything up to date", reg.Name)
		return nil
	}

	// check download dir
	if err := reg.tmpDir.Ensure(); err != nil {
		return fmt.Errorf("could not prepare tmp directory for download: %w", err)
	}

	// download updates
	log.Infof("%s: starting to download %d updates", reg.Name, len(toUpdate))
	client := &http.Client{}
	var reportError error

	for i, rv := range toUpdate {
		log.Infof(
			"%s: downloading update [%d/%d]: %s version %s",
			reg.Name,
			i+1, len(toUpdate),
			rv.resource.Identifier, rv.VersionNumber,
		)
		var err error
		for tries := 0; tries < 3; tries++ {
			err = reg.fetchFile(ctx, client, rv, tries)
			if err == nil {
				// Update resource version state.
				rv.resource.Lock()
				rv.Available = true
				if rv.resource.VerificationOptions != nil {
					rv.SigAvailable = true
				}
				rv.resource.Unlock()

				break
			}
		}
		if err != nil {
			reportError := fmt.Errorf("failed to download %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
			log.Warningf("%s: %s", reg.Name, reportError)
		}

		reg.state.UpdateOperationDetails(&StateDownloadingDetails{
			Resources:    downloadDetailsResources,
			FinishedUpTo: i + 1,
		})
	}

	if len(missingSigs) > 0 {
		log.Infof("%s: downloading %d missing signatures", reg.Name, len(missingSigs))

		for _, rv := range missingSigs {
			var err error
			for tries := 0; tries < 3; tries++ {
				err = reg.fetchMissingSig(ctx, client, rv, tries)
				if err == nil {
					// Update resource version state.
					rv.resource.Lock()
					rv.SigAvailable = true
					rv.resource.Unlock()

					break
				}
			}
			if err != nil {
				reportError := fmt.Errorf("failed to download missing sig of %s version %s: %w", rv.resource.Identifier, rv.VersionNumber, err)
				log.Warningf("%s: %s", reg.Name, reportError)
			}
		}
	}

	reg.state.ReportDownloads(
		downloadDetailsResources,
		reportError,
	)
	log.Infof("%s: finished downloading updates", reg.Name)

	return nil
}

// DownloadUpdates checks if updates are available and downloads updates of used components.

// GetPendingDownloads returns the list of pending downloads.
// If manual is set, indexes with AutoDownload=false will be checked.
// If auto is set, indexes with AutoDownload=true will be checked.
func (reg *ResourceRegistry) GetPendingDownloads(manual, auto bool) (resources, sigs []*ResourceVersion) {
	reg.RLock()
	defer reg.RUnlock()

	// create list of downloads
	var toUpdate []*ResourceVersion
	var missingSigs []*ResourceVersion

	for _, res := range reg.resources {
		func() {
			res.Lock()
			defer res.Unlock()

			// Skip resources without index or indexes that should not be reported
			// according to parameters.
			switch {
			case res.Index == nil:
				// Cannot download if resource is not part of an index.
				return
			case manual && !res.Index.AutoDownload:
				// Manual update report and index is not auto-download.
			case auto && res.Index.AutoDownload:
				// Auto update report and index is auto-download.
			default:
				// Resource should not be reported.
				return
			}

			// Skip resources we don't need.
			switch {
			case res.inUse():
				// Update if resource is in use.
			case res.available():
				// Update if resource is available locally, ie. was used in the past.
			case utils.StringInSlice(reg.MandatoryUpdates, res.Identifier):
				// Update is set as mandatory.
			default:
				// Resource does not need to be updated.
				return
			}

			// Go through all versions until we find versions that need updating.
			for _, rv := range res.Versions {
				switch {
				case !rv.CurrentRelease:
					// We are not interested in older releases.
				case !rv.Available:
					// File not available locally, download!
					toUpdate = append(toUpdate, rv)
				case !rv.SigAvailable && res.VerificationOptions != nil:
					// File signature is not available and verification is enabled, download signature!
					missingSigs = append(missingSigs, rv)
				}
			}
		}()
	}

	slices.SortFunc[[]*ResourceVersion, *ResourceVersion](toUpdate, func(a, b *ResourceVersion) int {
		return strings.Compare(a.resource.Identifier, b.resource.Identifier)
	})
	slices.SortFunc[[]*ResourceVersion, *ResourceVersion](missingSigs, func(a, b *ResourceVersion) int {
		return strings.Compare(a.resource.Identifier, b.resource.Identifier)
	})

	return toUpdate, missingSigs
}

func humanInfoFromResourceVersions(resourceVersions []*ResourceVersion) []string {
	identifiers := make([]string, len(resourceVersions))

	for i, rv := range resourceVersions {
		identifiers[i] = fmt.Sprintf("%s v%s", rv.resource.Identifier, rv.VersionNumber)
	}

	return identifiers
}