github.com/quay/claircore@v1.5.28/datastore/postgres/indexrepository.go (about)

     1  package postgres
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"time"
     7  
     8  	"github.com/prometheus/client_golang/prometheus"
     9  	"github.com/prometheus/client_golang/prometheus/promauto"
    10  	"github.com/quay/claircore"
    11  	"github.com/quay/claircore/indexer"
    12  	"github.com/quay/claircore/pkg/microbatch"
    13  )
    14  
    15  var (
    16  	indexRepositoriesCounter = promauto.NewCounterVec(
    17  		prometheus.CounterOpts{
    18  			Namespace: "claircore",
    19  			Subsystem: "indexer",
    20  			Name:      "indexrepositories_total",
    21  			Help:      "Total number of database queries issued in the IndexRepositories method.",
    22  		},
    23  		[]string{"query"},
    24  	)
    25  
    26  	indexRepositoriesDuration = promauto.NewHistogramVec(
    27  		prometheus.HistogramOpts{
    28  			Namespace: "claircore",
    29  			Subsystem: "indexer",
    30  			Name:      "indexrepositories_duration_seconds",
    31  			Help:      "The duration of all queries issued in the IndexRepositories method",
    32  		},
    33  		[]string{"query"},
    34  	)
    35  )
    36  
    37  func (s *IndexerStore) IndexRepositories(ctx context.Context, repos []*claircore.Repository, l *claircore.Layer, scnr indexer.VersionedScanner) error {
    38  	const (
    39  		insert = `
    40  		INSERT INTO repo
    41  			(name, key, uri, cpe)
    42  		VALUES ($1, $2, $3, $4)
    43  		ON CONFLICT (name, key, uri) DO NOTHING;
    44  		`
    45  
    46  		insertWith = `
    47  		WITH repositories AS (
    48  			SELECT id AS repo_id
    49  			FROM repo
    50  			WHERE name = $1
    51  			  AND key = $2
    52  			  AND uri = $3
    53  		),
    54  			 scanner AS (
    55  				 SELECT id AS scanner_id
    56  				 FROM scanner
    57  				 WHERE name = $4
    58  				   AND version = $5
    59  				   AND kind = $6
    60  			 ),
    61  			 layer AS (
    62  				 SELECT id AS layer_id
    63  				 FROM layer
    64  				 WHERE layer.hash = $7
    65  			 )
    66  		INSERT
    67  		INTO repo_scanartifact (layer_id, repo_id, scanner_id)
    68  		VALUES ((SELECT layer_id FROM layer),
    69  				(SELECT repo_id FROM repositories),
    70  				(SELECT scanner_id FROM scanner))
    71  		ON CONFLICT DO NOTHING;
    72  		`
    73  	)
    74  	// obtain a transaction scoped batch
    75  	tx, err := s.pool.Begin(ctx)
    76  	if err != nil {
    77  		return fmt.Errorf("store:indexRepositories failed to create transaction: %w", err)
    78  	}
    79  	defer tx.Rollback(ctx)
    80  
    81  	insertRepoStmt, err := tx.Prepare(ctx, "insertRepoStmt", insert)
    82  	if err != nil {
    83  		return fmt.Errorf("failed to create insert repo statement: %w", err)
    84  	}
    85  	insertRepoScanArtifactWithStmt, err := tx.Prepare(ctx, "insertRepoScanArtifactWith", insertWith)
    86  	if err != nil {
    87  		return fmt.Errorf("failed to create insert repo scanartifact statement: %w", err)
    88  	}
    89  
    90  	start := time.Now()
    91  	mBatcher := microbatch.NewInsert(tx, 500, time.Minute)
    92  	for _, repo := range repos {
    93  		err := mBatcher.Queue(
    94  			ctx,
    95  			insertRepoStmt.SQL,
    96  			repo.Name,
    97  			repo.Key,
    98  			repo.URI,
    99  			repo.CPE,
   100  		)
   101  		if err != nil {
   102  			return fmt.Errorf("batch insert failed for repo %v: %w", repo, err)
   103  		}
   104  	}
   105  	err = mBatcher.Done(ctx)
   106  	if err != nil {
   107  		return fmt.Errorf("final batch insert failed for repo: %w", err)
   108  	}
   109  	indexRepositoriesCounter.WithLabelValues("insert_batch").Add(1)
   110  	indexRepositoriesDuration.WithLabelValues("insert_batch").Observe(time.Since(start).Seconds())
   111  
   112  	// make repo scan artifacts
   113  
   114  	start = time.Now()
   115  	mBatcher = microbatch.NewInsert(tx, 500, time.Minute)
   116  	for _, repo := range repos {
   117  		err := mBatcher.Queue(
   118  			ctx,
   119  			insertRepoScanArtifactWithStmt.SQL,
   120  			repo.Name,
   121  			repo.Key,
   122  			repo.URI,
   123  			scnr.Name(),
   124  			scnr.Version(),
   125  			scnr.Kind(),
   126  			l.Hash,
   127  		)
   128  		if err != nil {
   129  			return fmt.Errorf("batch insert failed for repo_scanartifact %v: %w", repo, err)
   130  		}
   131  	}
   132  	err = mBatcher.Done(ctx)
   133  	if err != nil {
   134  		return fmt.Errorf("final batch insert failed for repo_scanartifact: %w", err)
   135  	}
   136  	indexRepositoriesCounter.WithLabelValues("insertWith_batch").Add(1)
   137  	indexRepositoriesDuration.WithLabelValues("insertWith_batch").Observe(time.Since(start).Seconds())
   138  
   139  	err = tx.Commit(ctx)
   140  	if err != nil {
   141  		return fmt.Errorf("store:indexRepositories failed to commit tx: %w", err)
   142  	}
   143  	return nil
   144  }