github.com/grafana/synthetic-monitoring-agent@v0.64.2/internal/scraper/scraper.go

package scraper

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"math"
	"math/rand"
	"slices"
	"strconv"
	"strings"
	"time"

	kitlog "github.com/go-kit/kit/log" //nolint:staticcheck // TODO(mem): replace in BBE
	"github.com/go-kit/kit/log/level"  //nolint:staticcheck // TODO(mem): replace in BBE
	"github.com/go-logfmt/logfmt"
	"github.com/google/uuid"
	"github.com/mmcloughlin/geohash"
	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	prom "github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/prompb"
	"github.com/rs/zerolog"

	logproto "github.com/grafana/loki/pkg/push"
	"github.com/grafana/synthetic-monitoring-agent/internal/feature"
	"github.com/grafana/synthetic-monitoring-agent/internal/k6runner"
	"github.com/grafana/synthetic-monitoring-agent/internal/model"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober"
	"github.com/grafana/synthetic-monitoring-agent/internal/pusher"
	"github.com/grafana/synthetic-monitoring-agent/internal/secrets"
	"github.com/grafana/synthetic-monitoring-agent/internal/telemetry"
	sm "github.com/grafana/synthetic-monitoring-agent/pkg/pb/synthetic_monitoring"
)

const (
	ProbeSuccessMetricName = "probe_success"
	CheckInfoMetricName    = "sm_check_info"
	CheckInfoSource        = "synthetic-monitoring-agent"
	maxLabelValueLength    = 2048 // this is the default value in Prometheus
	maxPublishInterval     = 2 * time.Minute
	minPublishGap          = 10 * time.Second
)

const (
	probeLabelName         = "probe"
	regionLabelName        = "region"
	smRegionLabelName      = "sm_region"
	unsetRegionValue       = "unset"
	configVersionLabelName = "config_version"
	instanceLabelName      = "instance"
	checkNameLabelName     = "check_name"
)

// mimirMaxLabelsIngest protects us and Mimir from hitting an error due to
// ingest limit; is short of the true ingest limit to retain room for growth.
const mimirMaxLabelsIngest = 40

var (
	staleNaN    uint64  = 0x7ff0000000000002
	staleMarker float64 = math.Float64frombits(staleNaN)
)

type Metrics interface {
	AddScrape()
	AddCheckError()
	AddCollectorError()
}

type LabelsLimiter interface {
	MetricLabels(ctx context.Context, tenantID model.GlobalID) (int, error)
	LogLabels(ctx context.Context, tenantID model.GlobalID) (int, error)
}

type TenantCals interface {
	CostAttributionLabels(ctx context.Context, tenantID model.GlobalID) ([]string, error)
}

type TenantLabelMode interface {
	ForTenant(ctx context.Context, tenantID model.GlobalID) (sm.LabelMode, error)
}

type Telemeter interface {
	AddExecution(e telemetry.Execution)
}

type Scraper struct {
	publisher     pusher.Publisher
	cancel        context.CancelFunc
	checkName     string
	target        string
	logger        zerolog.Logger
	check         model.Check
	probe         sm.Probe
	prober        prober.Prober
	labelsLimiter LabelsLimiter
	labellingMode TenantLabelMode
	stop          chan struct{}
	metrics       Metrics
	summaries     map[uint64]prometheus.Summary
	histograms    map[uint64]prometheus.Histogram
	telemeter     Telemeter
	cals          TenantCals
}

type Factory func(
	ctx context.Context, check model.Check, publisher pusher.Publisher, probe sm.Probe,
	features feature.Collection,
	logger zerolog.Logger,
	metrics Metrics,
	k6runner k6runner.Runner,
	labelsLimiter LabelsLimiter,
	telemeter *telemetry.Telemeter,
	secretStore secrets.SecretProvider,
	cals TenantCals,
	labellingMode TenantLabelMode,
) (*Scraper, error)

type (
	TimeSeries = []prompb.TimeSeries
	Streams    = []logproto.Stream
)

type probeData struct {
	tenantId model.GlobalID
	ts       TimeSeries
	streams  Streams
}

func (d *probeData) Metrics() TimeSeries {
	return d.ts
}

func (d *probeData) Streams() Streams {
	return d.streams
}

func (d *probeData) Tenant() model.GlobalID {
	return d.tenantId
}

func New(
	ctx context.Context, check model.Check, publisher pusher.Publisher, probe sm.Probe,
	features feature.Collection,
	logger zerolog.Logger,
	metrics Metrics,
	k6runner k6runner.Runner,
	labelsLimiter LabelsLimiter,
	telemeter *telemetry.Telemeter,
	secretStore secrets.SecretProvider,
	cals TenantCals,
	labellingMode TenantLabelMode,
) (*Scraper, error) {
	return NewWithOpts(ctx, check, ScraperOpts{
		Probe:                 probe,
		Publisher:             publisher,
		Logger:                logger,
		Metrics:               metrics,
		ProbeFactory:          prober.NewProberFactory(k6runner, probe.Id, features, secretStore),
		LabelsLimiter:         labelsLimiter,
		Telemeter:             telemeter,
		CostAttributionLabels: cals,
		LabellingMode:         labellingMode,
	})
}

var _ Factory = New

type ScraperOpts struct {
	Probe                 sm.Probe
	Publisher             pusher.Publisher
	Logger                zerolog.Logger
	Metrics               Metrics
	ProbeFactory          prober.ProberFactory
	LabelsLimiter         LabelsLimiter
	LabellingMode         TenantLabelMode
	Telemeter             Telemeter
	CostAttributionLabels TenantCals
}

func NewWithOpts(ctx context.Context, check model.Check, opts ScraperOpts) (*Scraper, error) {
	checkName := check.Type().String()

	logger := opts.Logger.With().
		Int("region_id", check.RegionId).
		Int64("tenantId", check.TenantId).
		Int64("check_id", check.Id).
		Str(probeLabelName, opts.Probe.Name).
		Str("target", check.Target).
		Str("job", check.Job).
		Str("check", checkName).
		Logger()

	sctx, cancel := context.WithCancel(ctx)

	smProber, target, err := opts.ProbeFactory.New(sctx, logger, check)
	if err != nil {
		cancel()
		return nil, err
	}

	return &Scraper{
		publisher:     opts.Publisher,
		cancel:        cancel,
		checkName:     checkName,
		target:        target,
		logger:        logger,
		check:         check,
		probe:         opts.Probe,
		prober:        smProber,
		labelsLimiter: opts.LabelsLimiter,
		labellingMode: opts.LabellingMode,
		stop:          make(chan struct{}),
		metrics:       opts.Metrics,
		summaries:     make(map[uint64]prometheus.Summary),
		histograms:    make(map[uint64]prometheus.Histogram),
		telemeter:     opts.Telemeter,
		cals:          opts.CostAttributionLabels,
	}, nil
}

var (
	errCheckFailed       = errors.New("probe failed")
	errUnsupportedMetric = errors.New("unsupported metric type")
	errTooManyLabels     = errors.New("invalid configuration, too many labels")
)

type checkStateMachine struct {
	passes    int
	failures  int
	threshold int
}

func (sm *checkStateMachine) fail(cb func()) {
	wasFailing := sm.isFailing()
	sm.passes = 0
	sm.failures++
	isFailing := sm.isFailing()

	if isFailing != wasFailing {
		cb()
	}
}

func (sm *checkStateMachine) pass(cb func()) {
	wasPassing := sm.isPassing()
	sm.passes++
	sm.failures = 0
	isPassing := sm.isPassing()

	if isPassing != wasPassing {
		cb()
	}
}

func (sm checkStateMachine) isPassing() bool {
	return sm.passes > sm.threshold
}

func (sm checkStateMachine) isFailing() bool {
	return sm.failures > sm.threshold
}

func (s *Scraper) Run(ctx context.Context) {
	s.logger.Info().Msg("starting scraper")

	// TODO(mem): keep count of the number of successive errors and
	// collect logs if threshold is reached.

	var (
		frequency = ms(s.check.Frequency)
		offset    = ms(s.check.Offset)
	)

	if offset == 0 {
		offset = randDuration(min(frequency, maxPublishInterval))
	}

	scrapeHandler := scrapeHandler{scraper: s}

	tickWithOffset(
		ctx,
		s.stop,
		scrapeHandler.scrape, scrapeHandler.republish, scrapeHandler.cleanup,
		frequency, offset,
		maxPublishInterval, minPublishGap,
	)

	s.cancel()

	s.logger.Info().Msg("scraper stopped")
}

type scrapeHandler struct {
	scraper *Scraper
	payload *probeData // needed to keep the most recently published payload for republication and clean up
	sm      checkStateMachine
}

func (h *scrapeHandler) scrape(ctx context.Context, t time.Time) {
	h.scraper.metrics.AddScrape()

	var (
		err      error
		duration time.Duration
	)

	h.payload, duration, err = h.scraper.collectData(ctx, t)

	switch {
	case errors.Is(err, errCheckFailed):
		h.scraper.metrics.AddCheckError()
		h.sm.fail(func() {
			h.scraper.logger.Info().Err(err).Msg("check entered FAIL state")
		})

	case err != nil:
		h.scraper.metrics.AddCollectorError()
		h.scraper.logger.Error().Err(err).Msg("error collecting data")

		return

	default:
		h.sm.pass(func() {
			h.scraper.logger.Info().Msg("check entered PASS state")
		})
	}

	costAttributionLabels, err := h.scraper.getCostAttributionLabels(ctx, h.payload.tenantId)
	if err != nil {
		// If cals can't be found, do not block
		h.scraper.logger.Error().
			Int64("tenantId", int64(h.payload.tenantId)).
			Msg("Could not load cals")
	}

	h.scraper.logger.Debug().
		Int64("tenantId", int64(h.payload.tenantId)).
		Int("costAttributionLabelsCount", len(costAttributionLabels)).
		Any("CostAttributionLabels", costAttributionLabels).
		Msg("tenant telemetry")

	// If we are dropping the data in case of errors, we should not count that execution.
	h.scraper.telemeter.AddExecution(telemetry.Execution{
		LocalTenantID:         h.scraper.check.TenantId,
		RegionID:              int32(h.scraper.check.RegionId),
		CheckClass:            h.scraper.check.Class(),
		CostAttributionLabels: costAttributionLabels,
		Duration:              duration,
	})

	if h.payload != nil {
		h.scraper.publisher.Publish(h.payload)
	}
}

func (h *scrapeHandler) republish(ctx context.Context, t time.Time) {
	if h.payload == nil {
		return
	}

	h.payload.streams = nil // do not republish logs

	// Update the timestamps of all collected samples to now.
	now := t.UnixMilli()

	for i := range h.payload.ts {
		ts := &h.payload.ts[i]
		for j := range ts.Samples {
			ts.Samples[j].Timestamp = now
		}
	}

	h.scraper.publisher.Publish(h.payload)
}

func (h *scrapeHandler) cleanup(ctx context.Context, t time.Time) {
	if h.payload == nil {
		return
	}

	staleSample := prompb.Sample{
		Timestamp: t.UnixNano()/1e6 + 1, // ms
		Value:     staleMarker,
	}

	for i := range h.payload.ts {
		ts := &h.payload.ts[i]
		for j := range ts.Samples {
			ts.Samples[j] = staleSample
		}
	}

	h.payload.streams = nil

	h.scraper.publisher.Publish(h.payload)

	h.payload = nil
}

func ms(n int64) time.Duration {
	return time.Duration(n) * time.Millisecond
}

func randDuration(d time.Duration) time.Duration {
	return time.Duration(rand.Int63n(int64(d)))
}

func (s *Scraper) Stop() {
	s.logger.Info().Msg("stopping scraper")
	close(s.stop)
}

func (s Scraper) CheckType() sm.CheckType {
	return s.check.Type()
}

func (s Scraper) ConfigVersion() string {
	return s.check.ConfigVersion()
}

func (s Scraper) LastModified() float64 {
	return s.check.Modified
}

func tickWithOffset(
	ctx context.Context,
	stop <-chan struct{},
	work, idle, cleanup func(context.Context, time.Time),
	period, offset, maxIdle, minGap time.Duration,
) {
	// wait for up to offset duration, paying attention to cancellation signals.
	offsetTimer := time.NewTimer(offset)

	var lastTick time.Time

	drainOffsetTimer := func() {
		if !offsetTimer.Stop() {
			<-offsetTimer.C
		}
	}

	// Check context and stop channels first, to avoid runnig the work
	// function if we have been cancelled already. This can happen with a 0
	// offset, and the Go runtime choosing to send the offset tick before
	// other two channels.
	select {
	case <-ctx.Done():
		drainOffsetTimer()
		return

	case <-stop:
		drainOffsetTimer()
		// we haven't done anything yet, no clean up
		return

	default:
	}

	select {
	case <-ctx.Done():
		drainOffsetTimer()
		return

	case <-stop:
		drainOffsetTimer()
		// we haven't done anything yet, no clean up
		return

	case t := <-offsetTimer.C:
		lastTick = t
		work(ctx, t)
	}

	// create a ticker that won't fire, and replace it with a running
	// ticker in case we need it.
	inactivityTicker := &time.Ticker{}

	if period > maxIdle {
		inactivityTicker = time.NewTicker(maxIdle)
	}

	workTicker := time.NewTicker(period)

	for {
		select {
		case <-ctx.Done():
			workTicker.Stop()
			inactivityTicker.Stop()
			// no clean up if the context is cancelled.
			return

		case <-stop:
			workTicker.Stop()
			inactivityTicker.Stop()
			// if we are here, we already pushed something
			// at least once, lastTick cannot be zero, but
			// just in case...
			if !lastTick.IsZero() {
				cleanup(ctx, lastTick)
			}

			return

		case t := <-inactivityTicker.C:
			// if the amount of time since the last run is greater
			// than minGap and the amount of time left until the
			// next run is larger than minGap, then run the idle
			// function.
			//
			// p----I----I----Ip---I----I----I-p
			//      ^    ^    ^
			//      |    |    +-- don't run here because it's too close to the next run
			//      |    +-- run here
			//      +-- don't run here because it's too close to the last run
			if t.Sub(lastTick) >= minGap && lastTick.Add(period).Sub(t) >= minGap {
				idle(ctx, t)
			}

		case t := <-workTicker.C:
			lastTick = t
			work(ctx, t)
		}
	}
}

// CollectData runs the configured prober once at time t and returns transformed
// metrics and logs without publishing. A failed probe returns populated
// metrics and logs alongside a non-nil error; fatal collection errors return
// no data.
func (s *Scraper) CollectData(ctx context.Context, t time.Time) (TimeSeries, Streams, model.GlobalID, time.Duration, error) {
	pd, d, err := s.collectDataWith(ctx, t, logicalScrapeClock(t))
	if err != nil && !errors.Is(err, errCheckFailed) {
		return nil, nil, 0, 0, err
	}

	return pd.Metrics(), pd.Streams(), pd.Tenant(), d, err
}

func (s Scraper) collectData(ctx context.Context, t time.Time) (*probeData, time.Duration, error) {
	return s.collectDataWith(ctx, t, scheduledScrapeClock(t))
}

func (s Scraper) collectDataWith(ctx context.Context, t time.Time, clock scrapeClock) (*probeData, time.Duration, error) {
	target := s.target

	labelMode, err := s.labellingMode.ForTenant(ctx, s.check.GlobalTenantID())
	if err != nil {
		// ForTenant falls back to UNPREFIXED on transient errors so that a temporary tenant
		// cache miss does not abort the scrape.
		s.logger.Warn().Err(err).Msg("could not retrieve tenant label mode, falling back to UNPREFIXED")
	}

	// TODO(mem): this is constant for the scraper, move this
	// outside this function?
	// sysMetricLabels are the common labels present on ALL metrics
	// emitted by the system.
	sysMetricLabels := []labelPair{
		{name: probeLabelName, value: s.probe.Name},
		{name: configVersionLabelName, value: s.check.ConfigVersion()},
		{name: instanceLabelName, value: s.check.Target},
		{name: "job", value: s.check.Job},
		// {name: "source", value: CheckInfoSource}, // identify metrics that belong to synthetic-monitoring-agent
	}

	var (
		// These are the labels defined by the user.
		userLabels = buildUserLabels(s.probe.Labels, s.check.Labels, labelMode)
		// These labels are applied to the sm_check_info metric.
		checkInfoLabels = s.buildCheckInfoLabels(userLabels, sysMetricLabels, labelMode)
		// This is the execution ID for the check run.
		executionID = uuid.New().String()
	)

	maxMetricLabels, err := s.labelsLimiter.MetricLabels(ctx, s.check.GlobalTenantID())
	if err != nil {
		return nil, 0, fmt.Errorf("retrieving tenant metric labels limit: %w", err)
	}

	maxLogLabels, err := s.labelsLimiter.LogLabels(ctx, s.check.GlobalTenantID())
	if err != nil {
		return nil, 0, fmt.Errorf("retrieving tenant log labels limit: %w", err)
	}

	// In DUAL_WRITE mode, buildUserLabels emits both prefixed and un-prefixed
	// forms, so the user portion can be up to 2× the user label entries.
	// Cap at mimirMaxLabelsIngest to avoid false-firing while still guarding
	// against genuinely invalid sizes.
	checkInfoLimit := min(maxMetricLabels*2, mimirMaxLabelsIngest)
	if labelMode != sm.LabelMode_LABEL_MODE_DUAL_WRITE {
		checkInfoLimit = maxMetricLabels
	}

	if len(checkInfoLabels) > checkInfoLimit {
		// This should never happen.
		return nil, 0, fmt.Errorf("%w: %d", errTooManyLabels, len(checkInfoLabels))
	}

	logLabels := []labelPair{
		{name: probeLabelName, value: s.probe.Name},
		regionLabelForMode(labelMode, s.probe.Region),
		{name: instanceLabelName, value: s.check.Target},
		{name: "job", value: s.check.Job},
		{name: checkNameLabelName, value: s.checkName},
		{name: "source", value: CheckInfoSource}, // identify log lines that belong to synthetic-monitoring-agent
	}
	logLabels = mergeLogLabels(logLabels, userLabels)

	// set up logger to capture check logs
	logs := bytes.Buffer{}
	bl := kitlog.NewLogfmtLogger(&logs)

	// set up logger to capture all the labels as part of the log entry
	loggerLabels := make([]any, 0, 2*(2+len(logLabels)))

	loggerLabels = append(loggerLabels, "ts", clock.wrapperLogTimestamp(), "target", target)
	for _, l := range logLabels {
		loggerLabels = append(loggerLabels, l.name, l.value)
	}

	sl := kitlog.With(bl, loggerLabels...)

	var timeout time.Duration

	switch s.CheckType() {
	case sm.CheckTypeMultiHttp, sm.CheckTypeScripted, sm.CheckTypeBrowser:
		// k6-backed checks have a more sophisticated handling of timeouts, where the k6 runner receives the check
		// timeout from the specification and may implement a retrying mechanism. For this reason, we set the prober
		// timeout to the check frequency instead, which is the hard limit at which no further retries are possible as
		// the next execution is due.
		timeout = time.Duration(s.check.Frequency) * time.Millisecond
	default:
		timeout = time.Duration(s.check.Timeout) * time.Millisecond
	}

	wallStart := time.Now()

	success, mfs, err := getProbeMetrics(
		ctx,
		s.prober,
		target,
		timeout,
		checkInfoLabels,
		s.summaries, s.histograms,
		sl,
		s.check.BasicMetricsOnly,
		executionID,
		clock,
	)
	if err != nil {
		return nil, 0, err
	}

	duration, found := patchDuration(mfs)
	if !found {
		duration = clock.fallbackDuration(wallStart)
	}

	// At this point we know the check has been executed, regardless of
	// whether it succeeded or not.

	// In DUAL_WRITE and UNPREFIXED modes, un-prefixed user labels are appended to
	// execution metrics (probe_success, probe_duration_seconds, probe_http_*, etc.).
	// user-defined label values are retained over system-defined values.
	executionMetricLabels := appendUniqueLabels(customMetricLabels(s.probe.Labels, s.check.Labels, labelMode), sysMetricLabels)

	ts := s.extractTimeseries(t, mfs, executionMetricLabels)

	successValue := "1"

	if !success {
		err = errCheckFailed
		successValue = "0"
	}

	// Split logLabels into stream labels and structured metadata overflow.
	// maxLogLabels - 1 reserves one slot for probe_success which is always appended.
	var (
		streamLogLabels  []labelPair
		overflowMetadata []logproto.LabelAdapter
	)

	maxUserLogLabels := maxLogLabels - 1
	if len(logLabels) > maxUserLogLabels {
		streamLogLabels = logLabels[:maxUserLogLabels]
		for _, lp := range logLabels[maxUserLogLabels:] {
			overflowMetadata = append(overflowMetadata, logproto.LabelAdapter{Name: lp.name, Value: lp.value})
		}
	} else {
		streamLogLabels = logLabels
	}

	streamLogLabels = append(streamLogLabels, labelPair{name: ProbeSuccessMetricName, value: successValue}) // identify log lines that are failures

	// full set of structured metadata consists of any overflow labels and the execution_id
	overflowMetadata = append(overflowMetadata, logproto.LabelAdapter{Name: "execution_id", Value: executionID})
	structuredMetadata := overflowMetadata

	// streams need to have all the labels applied to them because loki does not support joins
	streams := s.extractLogs(t, logs.Bytes(), streamLogLabels, structuredMetadata)

	return &probeData{ts: ts, streams: streams, tenantId: s.check.GlobalTenantID()}, duration, err
}

// getCostAttributionLabels looks for the cost attribution labels for a specific tenant and
// generates a []sm.CostAttributionLabel where the Name's are the cal, and the value
// is derived from the checks label with the corresponding name.
func (s *Scraper) getCostAttributionLabels(ctx context.Context, tenantID model.GlobalID) ([]sm.CostAttributionLabel, error) {
	cals, err := s.cals.CostAttributionLabels(ctx, tenantID)
	if err != nil {
		return nil, err
	}

	vals := make([]sm.CostAttributionLabel, 0, len(cals))
	for _, cal := range cals {
		// Make the assumption there are no labels that match the CAL and set the default value to __MISSING__
		value := telemetry.CalNilStringTerminator

		idx := slices.IndexFunc(s.check.Labels, func(l sm.Label) bool {
			return l.Name == cal
		})
		if idx >= 0 {
			value = s.check.Labels[idx].Value
		}

		vals = append(vals, sm.CostAttributionLabel{
			Name:  cal,
			Value: value,
		})
	}

	return vals, nil
}

// patchDuration will modify the probe_duration_seconds metric to match the
// probe_script_duration_seconds metric. This is necessary because the
// probe_duration_seconds metric includes the time it takes waiting to run the
// script plus the time it takes to actually run the script.
//
// TODO(mem): find a better way to handle this.
func patchDuration(mfs []*dto.MetricFamily) (time.Duration, bool) {
	var duration, scriptDuration *dto.Gauge

	getGauge := func(mf *dto.MetricFamily) *dto.Gauge {
		for _, m := range mf.GetMetric() {
			if g := m.GetGauge(); g != nil {
				return g
			}
		}

		return nil
	}

	for _, mf := range mfs {
		switch mf.GetName() {
		case "probe_duration_seconds":
			duration = getGauge(mf)

		case "probe_script_duration_seconds":
			scriptDuration = getGauge(mf)
		}

		if duration != nil && scriptDuration != nil {
			*duration.Value = *scriptDuration.Value
			break
		}
	}

	if duration != nil {
		return time.Duration(*duration.Value * float64(time.Second)), true
	}

	return 0, false
}

func getProbeMetrics(
	ctx context.Context,
	prober prober.Prober,
	target string,
	timeout time.Duration,
	checkInfoLabels map[string]string,
	summaries map[uint64]prometheus.Summary,
	histograms map[uint64]prometheus.Histogram,
	logger kitlog.Logger,
	basicMetricsOnly bool,
	executionID string,
	clock scrapeClock,
) (bool, []*dto.MetricFamily, error) {
	registry := prometheus.NewRegistry()

	success := runProber(ctx, prober, target, timeout, registry, checkInfoLabels, logger, executionID, clock)

	mfs, err := registry.Gather()
	if err != nil {
		return success, nil, fmt.Errorf(`extracting data from blackbox-exporter: %w`, err)
	}

	registry = prometheus.NewRegistry()

	if err := getDerivedMetrics(mfs, summaries, histograms, registry, basicMetricsOnly); err != nil {
		return success, nil, fmt.Errorf(`getting derived metrics: %w`, err)
	}

	dmfs, err := registry.Gather()
	if err != nil {
		return success, nil, fmt.Errorf(`extracting derived metrics: %w`, err)
	}

	mfs = append(mfs, dmfs...)

	return success, mfs, nil
}

func runProber(
	ctx context.Context,
	prober prober.Prober,
	target string,
	timeout time.Duration,
	registry *prometheus.Registry,
	checkInfoLabels map[string]string,
	logger kitlog.Logger,
	executionID string,
	clock scrapeClock,
) bool {
	wallStart := time.Now()

	// The begin log needs no explicit time: the wrapper `ts` valuer already
	// stamps it at the stamper's start time (logical event time for the
	// collector, wall-clock now for the scheduled scraper).
	_ = level.Info(logger).Log("msg", "Beginning check", "type", prober.Name(), "timeout_seconds", timeout.Seconds())

	checkCtx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()

	success, duration := prober.Probe(checkCtx, target, registry, logger, executionID)

	wallDuration := time.Since(wallStart).Seconds()
	if duration == 0 {
		// If the prober did not provide their own duration, fallback to the wall time the scraper took to run.
		duration = wallDuration
	}

	probeSuccessGauge := prometheus.NewGauge(prometheus.GaugeOpts{
		Name: ProbeSuccessMetricName,
		Help: "Displays whether or not the probe was a success",
	})
	probeDurationGauge := prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "probe_duration_seconds",
		Help: "Returns how long the probe took to complete in seconds",
	})
	smCheckInfo := prometheus.NewGauge(prometheus.GaugeOpts{
		Name:        CheckInfoMetricName,
		Help:        "Provides information about a single check configuration",
		ConstLabels: checkInfoLabels,
	})

	registry.MustRegister(probeSuccessGauge)
	registry.MustRegister(smCheckInfo)
	registry.MustRegister(probeDurationGauge)

	probeDurationGauge.Set(duration)

	if success {
		probeSuccessGauge.Set(1)

		fields := append([]any{
			"msg", "Check succeeded",
			"duration_seconds", duration,
			"walltime_seconds", wallDuration,
		}, clock.endLogFields(duration)...)

		_ = level.Info(logger).Log(fields...)
	} else {
		probeSuccessGauge.Set(0)

		fields := append([]any{
			"msg", "Check failed",
			"duration_seconds", duration,
			"walltime_seconds", wallDuration,
		}, clock.endLogFields(duration)...)

		_ = level.Error(logger).Log(fields...)
	}

	smCheckInfo.Set(1)

	return success
}

func getDerivedMetrics(mfs []*dto.MetricFamily, summaries map[uint64]prometheus.Summary, histograms map[uint64]prometheus.Histogram, registry *prometheus.Registry, basicMetricsOnly bool) error {
	for _, mf := range mfs {
		switch {
		case mf.GetType() == dto.MetricType_GAUGE && mf.GetName() == ProbeSuccessMetricName:
			derivedMetricName := "probe_all_success"

			for _, metric := range mf.GetMetric() {
				_, err := updateSummaryFromMetric(derivedMetricName, mf.GetHelp(), metric, summaries, registry)
				if err != nil {
					return err
				}
			}

		case mf.GetType() == dto.MetricType_GAUGE:
			metricName := mf.GetName()

			// we need to keep probe_all_duration_seconds
			// because we use it to build a more reliable
			// way of computing "uptime".
			if metricName != "probe_duration_seconds" && basicMetricsOnly {
				continue
			}

			suffixes := []string{"_duration_seconds", "_time_seconds"}

			for _, suffix := range suffixes {
				if strings.HasSuffix(metricName, suffix) {
					derivedMetricName := strings.TrimSuffix(metricName, suffix) + "_all" + suffix

					for _, metric := range mf.GetMetric() {
						_, err := updateHistogramFromMetric(derivedMetricName, mf.GetHelp(), metric, histograms, registry)
						if err != nil {
							return err
						}
					}

					break
				}
			}
		}
	}

	return nil
}

// extractLogs parses logfmt-encoded log bytes and returns Loki streams using sharedLabels
// as stream labels. Any labels in structuredMetadata are attached to each log entry as
// Loki structured metadata.
func (s Scraper) extractLogs(t time.Time, logs []byte, sharedLabels []labelPair, structuredMetadata []logproto.LabelAdapter) Streams {
	var line strings.Builder

	dec := logfmt.NewDecoder(bytes.NewReader(logs))

	labels := make([]labelPair, 0, len(sharedLabels))

	var entries []logproto.Entry

RECORD:
	for dec.ScanRecord() {
		var t time.Time

		line.Reset()

		enc := logfmt.NewEncoder(&line)

		labels = labels[:0]
		labels = append(labels, sharedLabels...)

		for dec.ScanKeyval() {
			value := dec.Value()

			switch key := dec.Key(); string(key) {
			case "ts":
				if !t.IsZero() {
					continue // already set by the 'time' field
				}

				var err error

				t, err = time.Parse(time.RFC3339Nano, string(value))
				if err != nil {
					// We should never hit this as the timestamp string in the log should be valid.
					// Without a timestamp we cannot do anything. And we cannot use something like
					// time.Now() because that would mess up other entries
					s.logger.Warn().Err(err).Bytes("value", value).Msg("invalid timestamp scanning logs")
					continue RECORD
				}

			// k6 sets a 'time' field on its logs using RFC3339; if present, report it instead of the default timestamp.
			// The slogHandler adapter (used for BBE probers) also emits a 'time' field using RFC3339Nano.
			// RFC3339Nano is a strict superset of RFC3339; use it to parse either 'time' value.`
			case "time":
				var err error

				t, err = time.Parse(time.RFC3339Nano, string(value))
				if err != nil {
					s.logger.Warn().Err(err).Bytes("value", value).Msg("invalid timestamp scanning logs")
					continue RECORD
				}

			default:
				if err := enc.EncodeKeyval(key, value); err != nil {
					// We should never hit this because all the entries are valid.
					s.logger.Warn().Err(err).Bytes("key", key).Bytes("value", value).Msg("invalid entry scanning logs")
					continue RECORD
				}
			}
		}

		if err := enc.EndRecord(); err != nil {
			s.logger.Warn().Err(err).Msg("encoding logs")
		}

		entries = append(entries, logproto.Entry{
			Timestamp:          t,
			Line:               line.String(),
			StructuredMetadata: structuredMetadata,
		})
	}

	if err := dec.Err(); err != nil {
		s.logger.Error().Err(err).Msg("decoding logs")
	}

	return Streams{
		logproto.Stream{
			Labels:  fmtLabels(labels),
			Entries: entries,
		},
	}
}

func (s Scraper) extractTimeseries(t time.Time, metrics []*dto.MetricFamily, executionLabels []labelPair) TimeSeries {
	return extractTimeseries(t, metrics, executionLabels, s.summaries, s.histograms, s.logger)
}

// extractTimeseries converts gathered Prometheus metrics into remote-write time series.
//
//	executionLabels: system labels + un-prefixed user labels; applied to all check execution
//		metrics (probe_success, etc.).
func extractTimeseries(t time.Time, metrics []*dto.MetricFamily, executionLabels []labelPair, summaries map[uint64]prometheus.Summary, histograms map[uint64]prometheus.Histogram, logger zerolog.Logger) TimeSeries {
	toPrompb := func(pairs []labelPair) []prompb.Label {
		out := make([]prompb.Label, 0, len(pairs))
		for _, label := range pairs {
			out = append(out, prompb.Label{Name: label.name, Value: truncateLabelValue(label.value)})
		}

		return out
	}

	executionPrompb := toPrompb(executionLabels)

	var ts []prompb.TimeSeries

	for _, mf := range metrics {
		mName := mf.GetName()
		mType := mf.GetType()

		shared := executionPrompb
		if mName == CheckInfoMetricName {
			// sm_check_info is already constructed with the user labels
			// added to it under the correct LabeLMode paradigm.
			shared = nil
		}

		for _, m := range mf.GetMetric() {
			ts = appendDtoToTimeseries(ts, t, mName, shared, mType, m)
		}
	}

	return ts
}

func (s Scraper) buildCheckInfoLabels(userLabels []labelPair, commonLabels []labelPair, mode sm.LabelMode) map[string]string {
	baseLabels := []labelPair{
		{name: checkNameLabelName, value: s.checkName},
		regionLabelForMode(mode, s.probe.Region),
		{name: "frequency", value: strconv.FormatInt(s.check.Frequency, 10)},
		{name: "geohash", value: geohash.Encode(float64(s.probe.Latitude), float64(s.probe.Longitude))},
	}
	baseLabels = append(baseLabels, commonLabels...)

	if s.check.AlertSensitivity != "" && s.check.AlertSensitivity != "none" {
		baseLabels = append(baseLabels, labelPair{name: "alert_sensitivity", value: s.check.AlertSensitivity})
	}

	checkInfoLabels := appendUniqueLabels(userLabels, baseLabels)

	labelMap := make(map[string]string, len(checkInfoLabels))
	for _, l := range checkInfoLabels {
		labelMap[l.name] = l.value
	}

	return labelMap
}

// buildUserLabels builds the user-defined label pairs from probe and check labels
// according to the tenant's LabelMode.
//
// In DUAL_WRITE and UNPREFIXED modes, un-prefixed user labels whose names are
// reserved system labels (sm.IsSystemLabel) are dropped. This asserts the API's
// guarantee that a migrated tenant carries no such label, and prevents a stray
// reserved name from overwriting an intrinsic per-series metric label such as
// `le` or `phase`. Prefixed forms (`label_*`) are never dropped, since no reserved
// name begins with "label_", and PREFIXED mode is left untouched.
//
// In DUAL_WRITE mode, all prefixed forms are emitted first, followed by all
// surviving un-prefixed forms: [label_foo, label_bar, foo, bar]. This ordering
// ensures that in the event of log stream label overflow, the prefixed forms —
// which existing policies depend on — are preserved as stream labels, while the
// un-prefixed forms spill into StructuredMetadata.
func buildUserLabels(probeLabels, checkLabels []sm.Label, mode sm.LabelMode) []labelPair {
	// Probe labels first, then any non-colliding check labels. On a name
	// collision the check value overwrites in place, so check values win while
	// probe ordering is preserved.
	userLabels := mergeUserLabels(probeLabels, checkLabels)

	switch mode {
	case sm.LabelMode_LABEL_MODE_DUAL_WRITE:
		// All prefixed forms first, then all un-prefixed forms.
		labels := make([]labelPair, 0, 2*len(userLabels))
		for _, e := range userLabels {
			labels = append(labels, labelPair{name: "label_" + e.name, value: e.value})
		}

		for _, e := range userLabels {
			// reserved labels should be dropped from the user's custom set
			if sm.IsSystemLabel(e.name) {
				continue
			}

			labels = append(labels, labelPair{name: e.name, value: e.value})
		}

		return labels
	case sm.LabelMode_LABEL_MODE_UNPREFIXED:
		filtered := make([]labelPair, 0, len(userLabels))
		for _, e := range userLabels {
			// reserved labels should be dropped from the user's custom set
			if sm.IsSystemLabel(e.name) {
				continue
			}

			filtered = append(filtered, e)
		}

		return filtered
	default: // LABEL_MODE_PREFIXED - prefix custom label names with label_
		for i := range userLabels {
			userLabels[i].name = "label_" + userLabels[i].name
		}

		return userLabels
	}
}

// customMetricLabels returns the un-prefixed user-defined labels to be added to
// check execution metrics (probe_success, probe_duration_seconds, probe_http_*, etc.)
// in DUAL_WRITE and UNPREFIXED modes.
//
// In PREFIXED mode this returns nil — execution metrics carry no custom labels.
func customMetricLabels(probeLabels, checkLabels []sm.Label, mode sm.LabelMode) []labelPair {
	if mode == sm.LabelMode_LABEL_MODE_PREFIXED {
		return nil
	}

	return buildUserLabels(probeLabels, checkLabels, sm.LabelMode_LABEL_MODE_UNPREFIXED)
}

// regionLabelForMode returns the label under which the agent emits the
// probe's region. Tenants that have started the label migration (any mode
// other than PREFIXED) receive it as sm_region, which is part of the reserved
// set, freeing "region" for use as one of their own labels. Legacy PREFIXED
// tenants keep the historical "region" name.
//
// sm_region always carries a non-empty value — "unset" when the probe has no
// region — because its presence is how the API's alert expressions recognize
// a migrated series: an empty label value is equivalent to an absent label in
// PromQL, which would classify the series as legacy and route its alerts with
// the wrong exclusion list. The historical region label keeps its value as-is
// (absent when the probe has no region), preserving legacy series identity.
func regionLabelForMode(mode sm.LabelMode, probeRegion string) labelPair {
	if mode == sm.LabelMode_LABEL_MODE_PREFIXED {
		return labelPair{name: regionLabelName, value: probeRegion}
	}

	if probeRegion == "" {
		probeRegion = unsetRegionValue
	}

	return labelPair{name: smRegionLabelName, value: probeRegion}
}

func makeTimeseries(t time.Time, value float64, labels ...prompb.Label) prompb.TimeSeries {
	var ts prompb.TimeSeries

	// duplicate labels on timeseries are rejected by Mimir.
	// this is also a common point to enforce conflict resolution
	// for LabelMode - user-defined labels should appear first in the list
	// of labels associated with each timeseries so that de-duplication favours retaining them
	// over system-defined values.
	ts.Labels = make([]prompb.Label, 0, len(labels))

	for _, l := range labels {
		dup := false

		for i := range ts.Labels {
			if ts.Labels[i].Name == l.Name {
				dup = true
				break
			}
		}

		if !dup {
			ts.Labels = append(ts.Labels, l)
		}
	}

	ts.Samples = []prompb.Sample{
		{Timestamp: t.UnixNano() / 1e6, Value: value},
	}

	return ts
}

// mergeUserLabels combines probe- and check-level labels into a single ordered
// slice. Probe labels come first (in order), followed by any check labels whose
// names do not appear among the probe labels. When a check label's name collides
// with a probe label, the check value overwrites the probe value in place,
// preserving the label's original position.
func mergeUserLabels(probeLabels, checkLabels []sm.Label) []labelPair {
	labels := make([]labelPair, 0, len(probeLabels)+len(checkLabels))
	idx := make(map[string]int, len(probeLabels))

	for _, l := range probeLabels {
		idx[l.Name] = len(labels)
		labels = append(labels, labelPair{name: l.Name, value: l.Value})
	}

	for _, l := range checkLabels {
		if where, found := idx[l.Name]; found {
			labels[where].value = l.Value
			continue
		}

		idx[l.Name] = len(labels)
		labels = append(labels, labelPair{name: l.Name, value: l.Value})
	}

	return labels
}

// mergeLogLabels merges user-defined labels into the system log labels. On a
// name collision the user value wins in place, preserving the ordering of the
// system labels; user labels whose names do not collide are appended after them.
//
// Ordering matters: the system labels keep their leading positions so they are
// retained as stream labels when the label set exceeds the stream cap and the
// tail spills into structured metadata. User values still win on collision,
// which protects a tenant whose existing label name later becomes reserved.
func mergeLogLabels(systemLabels, userLabels []labelPair) []labelPair {
	merged := make([]labelPair, len(systemLabels), len(systemLabels)+len(userLabels))
	copy(merged, systemLabels)

	idx := make(map[string]int, len(systemLabels))
	for i, l := range merged {
		idx[l.name] = i
	}

	for _, l := range userLabels {
		if where, found := idx[l.name]; found {
			merged[where].value = l.value
			continue
		}

		idx[l.name] = len(merged)
		merged = append(merged, l)
	}

	return merged
}

// appendUniqueLabels returns a slice of key-value labels which
// are unique by key. Duplicates are discarded, meaning values
// from prior slices are favoured over latter ones.
func appendUniqueLabels(slice1, slice2 []labelPair) []labelPair {
	if slice1 == nil && slice2 == nil {
		return nil
	}

	result := make([]labelPair, 0, len(slice1)+len(slice2))
	seen := make(map[string]bool)

	addUnique := func(slice []labelPair) {
		for _, val := range slice {
			if !seen[val.name] {
				seen[val.name] = true
				result = append(result, val)
			}
		}
	}

	addUnique(slice1)
	addUnique(slice2)

	return slices.Clip(result)
}

func appendDtoToTimeseries(ts []prompb.TimeSeries, t time.Time, mName string, sharedLabels []prompb.Label, mType dto.MetricType, metric *dto.Metric) []prompb.TimeSeries {
	ml := metric.GetLabel()

	labels := make([]prompb.Label, 0, 1+len(sharedLabels)+len(ml))
	labels = append(labels, prompb.Label{Name: prom.MetricNameLabel, Value: mName})

	labels = append(labels, sharedLabels...)
	for _, l := range ml {
		labels = append(labels, prompb.Label{Name: *(l.Name), Value: truncateLabelValue(*(l.Value))})
	}

	switch mType {
	case dto.MetricType_COUNTER:
		if v := metric.GetCounter(); v != nil && v.Value != nil {
			ts = append(ts, makeTimeseries(t, *v.Value, labels...))
		}

	case dto.MetricType_GAUGE:
		if v := metric.GetGauge(); v != nil && v.Value != nil {
			ts = append(ts, makeTimeseries(t, *v.Value, labels...))
		}

	case dto.MetricType_UNTYPED:
		if v := metric.GetUntyped(); v != nil && v.Value != nil {
			ts = append(ts, makeTimeseries(t, *v.Value, labels...))
		}

	case dto.MetricType_SUMMARY:
		if s := metric.GetSummary(); s != nil {
			sLabels := make([]prompb.Label, len(labels))
			copy(sLabels, labels)

			sLabels[0] = prompb.Label{Name: prom.MetricNameLabel, Value: mName + "_sum"}
			ts = append(ts, makeTimeseries(t, s.GetSampleSum(), sLabels...))

			sLabels[0] = prompb.Label{Name: prom.MetricNameLabel, Value: mName + "_count"}
			ts = append(ts, makeTimeseries(t, float64(s.GetSampleCount()), sLabels...))

			sLabels = make([]prompb.Label, len(labels)+1)
			copy(sLabels, labels)

			for _, v := range s.GetQuantile() {
				sLabels[len(sLabels)-1] = prompb.Label{
					Name:  "quantile",
					Value: strconv.FormatFloat(v.GetQuantile(), 'G', -1, 64),
				}
				ts = append(ts, makeTimeseries(t, v.GetValue(), sLabels...))
			}
		}

	case dto.MetricType_HISTOGRAM:
		if h := metric.GetHistogram(); h != nil {
			if b := h.GetBucket(); b != nil {
				hLabels := make([]prompb.Label, len(labels))
				copy(hLabels, labels)

				hLabels[0] = prompb.Label{Name: prom.MetricNameLabel, Value: mName + "_sum"}
				ts = append(ts, makeTimeseries(t, h.GetSampleSum(), hLabels...))

				hLabels[0] = prompb.Label{Name: prom.MetricNameLabel, Value: mName + "_count"}
				ts = append(ts, makeTimeseries(t, float64(h.GetSampleCount()), hLabels...))

				hLabels = make([]prompb.Label, len(labels)+1)
				copy(hLabels, labels)

				hLabels[0] = prompb.Label{Name: prom.MetricNameLabel, Value: mName + "_bucket"}
				for _, v := range b {
					hLabels[len(hLabels)-1] = prompb.Label{
						Name:  "le",
						Value: strconv.FormatFloat(v.GetUpperBound(), 'G', -1, 64),
					}
					ts = append(ts, makeTimeseries(t, float64(v.GetCumulativeCount()), hLabels...))
				}

				// Add the +Inf bucket, which corresponds to the sample count.
				hLabels[len(hLabels)-1] = prompb.Label{
					Name:  "le",
					Value: "+Inf",
				}
				ts = append(ts, makeTimeseries(t, float64(h.GetSampleCount()), hLabels...))
			}
		}
	}

	return ts
}

type labelPair struct {
	name  string
	value string
}

func fmtLabels(labels []labelPair) string {
	if len(labels) == 0 {
		return ""
	}

	var s strings.Builder

	// these calls do not produce errors, the errors are required to
	// satisfy interfaces
	_, _ = s.WriteRune('{')

	for i, pair := range labels {
		if i > 0 {
			_, _ = s.WriteRune(',')
		}

		_, _ = s.WriteString(pair.name)
		_, _ = s.WriteRune('=')
		_, _ = s.WriteRune('"')
		_, _ = s.WriteString(pair.value)
		_, _ = s.WriteRune('"')
	}

	_, _ = s.WriteRune('}')

	return s.String()
}

func updateSummaryFromMetric(mName, help string, m *dto.Metric, summaries map[uint64]prometheus.Summary, registry *prometheus.Registry) (prometheus.Summary, error) {
	var value float64

	switch {
	case m.GetCounter() != nil:
		value = m.GetCounter().GetValue()

	case m.GetGauge() != nil:
		value = m.GetGauge().GetValue()

	default:
		return nil, errUnsupportedMetric
	}

	mHash := hashMetricNameAndLabels(mName, m.GetLabel())

	summary, found := summaries[mHash]
	if !found {
		summary = prometheus.NewSummary(prometheus.SummaryOpts{
			Name:        mName,
			Help:        help + " (summary)",
			ConstLabels: getLabels(m),
		})

		summaries[mHash] = summary
	}

	if err := registry.Register(summary); err != nil {
		return nil, err
	}

	summary.Observe(value)

	return summary, nil
}

func updateHistogramFromMetric(mName, help string, m *dto.Metric, histograms map[uint64]prometheus.Histogram, registry *prometheus.Registry) (prometheus.Histogram, error) {
	var value float64

	switch {
	case m.GetCounter() != nil:
		value = m.GetCounter().GetValue()

	case m.GetGauge() != nil:
		value = m.GetGauge().GetValue()

	default:
		return nil, errUnsupportedMetric
	}

	mHash := hashMetricNameAndLabels(mName, m.GetLabel())

	histogram, found := histograms[mHash]
	if !found {
		histogram = prometheus.NewHistogram(prometheus.HistogramOpts{
			Name:        mName,
			Help:        help + " (histogram)",
			ConstLabels: getLabels(m),
			Buckets:     prometheus.DefBuckets,
		})

		histograms[mHash] = histogram
	}

	if err := registry.Register(histogram); err != nil {
		return nil, err
	}

	histogram.Observe(value)

	return histogram, nil
}

func hashMetricNameAndLabels(name string, dtoLabels []*dto.LabelPair) uint64 {
	ls := prom.LabelSet{
		prom.MetricNameLabel: prom.LabelValue(name),
	}

	for _, label := range dtoLabels {
		ls[prom.LabelName(label.GetName())] = prom.LabelValue(label.GetValue())
	}

	return uint64(ls.Fingerprint())
}

func getLabels(m *dto.Metric) map[string]string {
	if len(m.GetLabel()) == 0 {
		return nil
	}

	labels := make(map[string]string)

	for _, label := range m.GetLabel() {
		labels[label.GetName()] = label.GetValue()
	}

	return labels
}

func truncateLabelValue(str string) string {
	if len(str) > maxLabelValueLength {
		b := []byte(str[:maxLabelValueLength])
		for i := maxLabelValueLength - 3; i < maxLabelValueLength; i++ {
			b[i] = '.'
		}

		str = string(b)
	}

	return str
}