github.com/quay/claircore@v1.5.28/datastore/postgres/indexfiles.go (about)

     1  package postgres
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"time"
     7  
     8  	"github.com/prometheus/client_golang/prometheus"
     9  	"github.com/prometheus/client_golang/prometheus/promauto"
    10  
    11  	"github.com/quay/claircore"
    12  	"github.com/quay/claircore/indexer"
    13  	"github.com/quay/claircore/pkg/microbatch"
    14  )
    15  
    16  var (
    17  	indexFilesCounter = promauto.NewCounterVec(
    18  		prometheus.CounterOpts{
    19  			Namespace: "claircore",
    20  			Subsystem: "indexer",
    21  			Name:      "indexfiles_total",
    22  			Help:      "Total number of database queries issued in the IndexFiles method.",
    23  		},
    24  		[]string{"query"},
    25  	)
    26  
    27  	indexFilesDuration = promauto.NewHistogramVec(
    28  		prometheus.HistogramOpts{
    29  			Namespace: "claircore",
    30  			Subsystem: "indexer",
    31  			Name:      "indexfiles_duration_seconds",
    32  			Help:      "The duration of all queries issued in the IndexFiles method",
    33  		},
    34  		[]string{"query"},
    35  	)
    36  )
    37  
    38  func (s *IndexerStore) IndexFiles(ctx context.Context, files []claircore.File, layer *claircore.Layer, scnr indexer.VersionedScanner) error {
    39  	const (
    40  		lookupLayerID = `
    41  		SELECT id FROM layer WHERE hash = $1
    42  		`
    43  
    44  		lookupScannerID = `
    45  		SELECT id FROM scanner WHERE scanner.name = $1 AND version = $2 AND kind = $3
    46  		`
    47  
    48  		insert = `
    49  		INSERT INTO file
    50  			(path, kind)
    51  		VALUES
    52  			($1, $2)
    53  		ON CONFLICT (path, kind) DO NOTHING;
    54  		`
    55  
    56  		insertWith = `
    57  		INSERT
    58  		INTO file_scanartifact (file_id, layer_id, scanner_id)
    59  		VALUES (
    60  			(SELECT id FROM file WHERE file.path = $1 AND file.kind = $2),
    61  			$3,
    62  			$4
    63  		)
    64  		ON CONFLICT DO NOTHING;
    65  		`
    66  	)
    67  
    68  	var layerID, scannerID int64
    69  
    70  	tx, err := s.pool.Begin(ctx)
    71  	if err != nil {
    72  		return fmt.Errorf("failed to create transaction: %v", err)
    73  	}
    74  	defer tx.Rollback(ctx)
    75  
    76  	// Get layerID
    77  	start := time.Now()
    78  	layerRow := tx.QueryRow(ctx, lookupLayerID, layer.Hash)
    79  	err = layerRow.Scan(&layerID)
    80  	if err != nil {
    81  		return fmt.Errorf("failed look up layer ID: %v", err)
    82  	}
    83  	indexFilesCounter.WithLabelValues("lookup_layer").Add(1)
    84  	indexFilesDuration.WithLabelValues("lookup_layer").Observe(time.Since(start).Seconds())
    85  
    86  	// Get scannerID
    87  	start = time.Now()
    88  	scannerRow := tx.QueryRow(ctx, lookupScannerID, scnr.Name(), scnr.Version(), scnr.Kind())
    89  	err = scannerRow.Scan(&scannerID)
    90  	if err != nil {
    91  		return fmt.Errorf("failed look up scanner ID: %v", err)
    92  	}
    93  	indexFilesCounter.WithLabelValues("lookup_scanner").Add(1)
    94  	indexFilesDuration.WithLabelValues("lookup_scanner").Observe(time.Since(start).Seconds())
    95  
    96  	insertFileStmt, err := tx.Prepare(ctx, "insertFileStmt", insert)
    97  	if err != nil {
    98  		return fmt.Errorf("failed to create statement: %w", err)
    99  	}
   100  	insertFileScanArtifactWithStmt, err := tx.Prepare(ctx, "insertFileScanArtifactWithStmt", insertWith)
   101  	if err != nil {
   102  		return fmt.Errorf("failed to create statement: %w", err)
   103  	}
   104  
   105  	start = time.Now()
   106  	mBatcher := microbatch.NewInsert(tx, 500, time.Minute)
   107  	for _, f := range files {
   108  		err := mBatcher.Queue(
   109  			ctx,
   110  			insertFileStmt.SQL,
   111  			f.Path,
   112  			f.Kind,
   113  		)
   114  		if err != nil {
   115  			return fmt.Errorf("batch insert failed for file %v: %w", f, err)
   116  		}
   117  	}
   118  	err = mBatcher.Done(ctx)
   119  	if err != nil {
   120  		return fmt.Errorf("final batch insert failed for file: %w", err)
   121  	}
   122  	indexFilesCounter.WithLabelValues("insert_batch").Add(1)
   123  	indexFilesDuration.WithLabelValues("insert_batch").Observe(time.Since(start).Seconds())
   124  
   125  	// make file scan artifacts
   126  	start = time.Now()
   127  	mBatcher = microbatch.NewInsert(tx, 500, time.Minute)
   128  	for _, f := range files {
   129  		err := mBatcher.Queue(
   130  			ctx,
   131  			insertFileScanArtifactWithStmt.SQL,
   132  			f.Path,
   133  			f.Kind,
   134  			layerID,
   135  			scannerID,
   136  		)
   137  		if err != nil {
   138  			return fmt.Errorf("batch insert failed for file_scanartifact %v: %w", f, err)
   139  		}
   140  	}
   141  	err = mBatcher.Done(ctx)
   142  	if err != nil {
   143  		return fmt.Errorf("final batch insert failed for file_scanartifact: %w", err)
   144  	}
   145  	indexFilesCounter.WithLabelValues("insertWith_batch").Add(1)
   146  	indexFilesDuration.WithLabelValues("insertWith_batch").Observe(time.Since(start).Seconds())
   147  
   148  	err = tx.Commit(ctx)
   149  	if err != nil {
   150  		return fmt.Errorf("failed to commit tx: %w", err)
   151  	}
   152  	return nil
   153  }