github.com/quay/claircore@v1.5.28/indexer/controller/coalesce.go (about)

     1  package controller
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"sync"
     7  
     8  	"golang.org/x/sync/errgroup"
     9  
    10  	"github.com/quay/claircore"
    11  	"github.com/quay/claircore/indexer"
    12  )
    13  
    14  // Coalesce calls each ecosystem's Coalescer and merges the returned IndexReports.
    15  func coalesce(ctx context.Context, s *Controller) (State, error) {
    16  	mu := sync.Mutex{}
    17  	reports := []*claircore.IndexReport{}
    18  	g, gctx := errgroup.WithContext(ctx)
    19  	// Dispatch a Coalescer goroutine for each ecosystem.
    20  	for _, ecosystem := range s.Ecosystems {
    21  		select {
    22  		case <-gctx.Done():
    23  			break
    24  		default:
    25  		}
    26  		artifacts := []*indexer.LayerArtifacts{}
    27  		pkgScanners, _ := ecosystem.PackageScanners(gctx)
    28  		distScanners, _ := ecosystem.DistributionScanners(gctx)
    29  		repoScanners, _ := ecosystem.RepositoryScanners(gctx)
    30  		fileScanners := []indexer.FileScanner{}
    31  		if ecosystem.FileScanners != nil {
    32  			fileScanners, _ = ecosystem.FileScanners(gctx)
    33  		}
    34  		// Pack "artifacts" variable.
    35  		for _, layer := range s.manifest.Layers {
    36  			la := &indexer.LayerArtifacts{
    37  				Hash: layer.Hash,
    38  			}
    39  			var vscnrs indexer.VersionedScanners
    40  			vscnrs.PStoVS(pkgScanners)
    41  			// Get packages from layer.
    42  			pkgs, err := s.Store.PackagesByLayer(gctx, layer.Hash, vscnrs)
    43  			if err != nil {
    44  				// On an early return gctx is canceled, and all in-flight
    45  				// Coalescers are canceled as well.
    46  				return Terminal, fmt.Errorf("failed to retrieve packages for %v: %w", layer.Hash, err)
    47  			}
    48  			la.Pkgs = append(la.Pkgs, pkgs...)
    49  			// Get repos that have been created via the package scanners.
    50  			pkgRepos, err := s.Store.RepositoriesByLayer(gctx, layer.Hash, vscnrs)
    51  			if err != nil {
    52  				return Terminal, fmt.Errorf("failed to retrieve repositories for %v: %w", layer.Hash, err)
    53  			}
    54  			la.Repos = append(la.Repos, pkgRepos...)
    55  
    56  			// Get distributions from layer.
    57  			vscnrs.DStoVS(distScanners) // Method allocates new "vscnr" underlying array, clearing old contents.
    58  			dists, err := s.Store.DistributionsByLayer(gctx, layer.Hash, vscnrs)
    59  			if err != nil {
    60  				return Terminal, fmt.Errorf("failed to retrieve distributions for %v: %w", layer.Hash, err)
    61  			}
    62  			la.Dist = append(la.Dist, dists...)
    63  			// Get repositories from layer.
    64  			vscnrs.RStoVS(repoScanners)
    65  			repos, err := s.Store.RepositoriesByLayer(gctx, layer.Hash, vscnrs)
    66  			if err != nil {
    67  				return Terminal, fmt.Errorf("failed to retrieve repositories for %v: %w", layer.Hash, err)
    68  			}
    69  			la.Repos = append(la.Repos, repos...)
    70  			// Get files from layer.
    71  			vscnrs.FStoVS(fileScanners)
    72  			files, err := s.Store.FilesByLayer(gctx, layer.Hash, vscnrs)
    73  			if err != nil {
    74  				return Terminal, fmt.Errorf("failed to retrieve files for %v: %w", layer.Hash, err)
    75  			}
    76  			la.Files = append(la.Files, files...)
    77  			// Pack artifacts array in layer order.
    78  			artifacts = append(artifacts, la)
    79  		}
    80  		coalescer, err := ecosystem.Coalescer(gctx)
    81  		if err != nil {
    82  			return Terminal, fmt.Errorf("failed to get coalescer from ecosystem: %v", err)
    83  		}
    84  		// Dispatch.
    85  		g.Go(func() error {
    86  			sr, err := coalescer.Coalesce(gctx, artifacts)
    87  			if err != nil {
    88  				return err
    89  			}
    90  
    91  			mu.Lock()
    92  			defer mu.Unlock()
    93  			reports = append(reports, sr)
    94  			return nil
    95  		})
    96  	}
    97  	if err := g.Wait(); err != nil {
    98  		return Terminal, err
    99  	}
   100  	s.report = MergeSR(s.report, reports)
   101  	for _, r := range s.Resolvers {
   102  		s.report = r.Resolve(ctx, s.report, s.manifest.Layers)
   103  	}
   104  	return IndexManifest, nil
   105  }
   106  
   107  // MergeSR merges IndexReports.
   108  //
   109  // "Source" is the IndexReport that the Indexer is working on.
   110  // "Merge" is a slice of IndexReports returned from Coalescers.
   111  //
   112  // The "SR" suffix is a historical accident.
   113  func MergeSR(source *claircore.IndexReport, merge []*claircore.IndexReport) *claircore.IndexReport {
   114  	for _, ir := range merge {
   115  		for k, v := range ir.Environments {
   116  			source.Environments[k] = append(source.Environments[k], v...)
   117  		}
   118  		for k, v := range ir.Packages {
   119  			source.Packages[k] = v
   120  		}
   121  
   122  		for k, v := range ir.Distributions {
   123  			source.Distributions[k] = v
   124  		}
   125  
   126  		for k, v := range ir.Repositories {
   127  			source.Repositories[k] = v
   128  		}
   129  
   130  		for k, v := range ir.Files {
   131  			source.Files[k] = v
   132  		}
   133  	}
   134  	return source
   135  }