github.com/grafana/synthetic-monitoring-agent@v0.64.2/internal/scraper/scraper_test.go

package scraper

import (
	"bytes"
	"context"
	"crypto/tls"
	"errors"
	"flag"
	"fmt"
	"io"
	"maps"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"slices"
	"sort"
	"strconv"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	"github.com/go-logfmt/logfmt"
	"github.com/google/uuid"
	logproto "github.com/grafana/loki/pkg/push"
	"github.com/grafana/synthetic-monitoring-agent/internal/k6runner"
	"github.com/grafana/synthetic-monitoring-agent/internal/model"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober/browser"
	dnsProber "github.com/grafana/synthetic-monitoring-agent/internal/prober/dns"
	grpcProber "github.com/grafana/synthetic-monitoring-agent/internal/prober/grpc"
	httpProber "github.com/grafana/synthetic-monitoring-agent/internal/prober/http"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober/icmp"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober/logger"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober/multihttp"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober/scripted"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober/tcp"
	"github.com/grafana/synthetic-monitoring-agent/internal/prober/traceroute"
	"github.com/grafana/synthetic-monitoring-agent/internal/pusher"
	"github.com/grafana/synthetic-monitoring-agent/internal/telemetry"
	"github.com/grafana/synthetic-monitoring-agent/internal/testhelper"
	sm "github.com/grafana/synthetic-monitoring-agent/pkg/pb/synthetic_monitoring"
	"github.com/miekg/dns"
	"github.com/mmcloughlin/geohash"
	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/prometheus/common/expfmt"
	prom "github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/model/labels"
	"github.com/prometheus/prometheus/prompb"
	"github.com/prometheus/prometheus/promql/parser"
	"github.com/rs/zerolog"
	"github.com/stretchr/testify/require"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	grpchealth "google.golang.org/grpc/health/grpc_health_v1"
	"kernel.org/pub/linux/libs/security/libcap/cap"
)

var updateGolden = flag.Bool("update-golden", false, "update golden files")

var metricParser = parser.NewParser(parser.Options{
	EnableExperimentalFunctions:  false,
	ExperimentalDurationExpr:     false,
	EnableExtendedRangeSelectors: false,
	EnableBinopFillModifiers:     false,
})

// TestValidateMetrics verify that the exposed metrics have not changed.
//
// It does this by setting up local targets (HTTP server, DNS server,
// TCP server) that the BBE probers can run against. The resulting
// metrics are then compared (only names and labels, not values) against
// known outputs. If metrics are added or removed, this test will fail.
//
// The golden files can be updated by running:
//
// go test -v -race -run TestValidateMetrics ./internal/scraper/
func TestValidateMetrics(t *testing.T) {
	for name, setup := range validateMetricSetups() {
		t.Run(name, func(t *testing.T) {
			verifyProberMetrics(t, name, setup, false)
		})
		t.Run(name+"_basic", func(t *testing.T) {
			verifyProberMetrics(t, name+"_basic", setup, true)
		})
	}
}

// verifyProberMetrics runs the specified prober against the server (if
// any) started by the setup function. The resup function provides the
// target as well as a clean up function that will be called once the
// test ends.
//
// Optionally, this function will update the golden files if the
// -update-golden flag was passed to the test.
func verifyProberMetrics(
	t *testing.T,
	name string,
	setup func(context.Context, *testing.T) (prober.Prober, model.Check, func()),
	basicMetricsOnly bool,
) {
	timeout := 10 * time.Second

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	summaries := make(map[uint64]prometheus.Summary)
	histograms := make(map[uint64]prometheus.Histogram)
	logger := &testLogger{w: io.Discard}

	if os.Getenv("CI") == "true" {
		logger.w = os.Stdout
	}

	prober, check, stop := setup(ctx, t)
	defer stop()

	success, mfs, err := getProbeMetrics(
		ctx,
		prober,
		check.Target,
		timeout,
		nil,
		summaries,
		histograms,
		logger,
		basicMetricsOnly,
		"test-execution-id",
		scheduledScrapeClock(time.Now()),
	)

	require.NoError(t, err, "probe failed")
	require.Truef(t, success, "probe failed: %s", prober.Name())

	fn := filepath.Join("testdata", name+".txt")

	if *updateGolden {
		var buf bytes.Buffer

		enc := expfmt.NewEncoder(&buf, expfmt.NewFormat(expfmt.TypeTextPlain))

		for _, m := range mfs {
			if err := enc.Encode(m); err != nil {
				t.Fatalf("encoding golden file: %s", err.Error())
			}
		}

		fh, err := os.Create(fn)
		if err != nil {
			t.Fatalf("cannot create file %s: %s", fn, err.Error())
		}
		defer fh.Close()

		if _, err := buf.WriteTo(fh); err != nil {
			t.Fatalf("cannot write to file %s: %s", fn, err.Error())
		}

		return
	}

	actualMetrics := map[string]struct{}{}

	for _, m := range mfs {
		addMetricToIndex(m, actualMetrics)
	}

	expectedMetrics, err := readGoldenFile(fn)
	require.NoError(t, err, "reading golden file")

	require.Equal(t, expectedMetrics, actualMetrics, "maps must be equal")
}

func addMetricToIndex(mf *dto.MetricFamily, index map[string]struct{}) {
	for _, metric := range mf.GetMetric() {
		labels := make([]string, 0, len(metric.GetLabel()))
		for _, label := range metric.GetLabel() {
			labels = append(labels, fmt.Sprintf(`%q=%q`, label.GetName(), label.GetValue()))
		}

		sort.Strings(labels)
		index[mf.GetName()+"{"+strings.Join(labels, ",")+"}"] = struct{}{}
	}
}

func readGoldenFile(fn string) (map[string]struct{}, error) {
	fh, err := os.Open(fn)
	if err != nil {
		return nil, fmt.Errorf("cannot open file %s: %w", fn, err)
	}
	defer fh.Close()

	dec := expfmt.NewDecoder(fh, expfmt.NewFormat(expfmt.TypeTextPlain))

	metrics := map[string]struct{}{}

	for {
		var mf dto.MetricFamily

		switch err := dec.Decode(&mf); {
		case errors.Is(err, io.EOF):
			return metrics, nil

		case err != nil:
			return nil, fmt.Errorf("cannot decode golden file %s: %w", fn, err)
		}

		addMetricToIndex(&mf, metrics)
	}
}

func setupPingProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	check := model.Check{
		Check: sm.Check{
			Target:  "127.0.0.1",
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Ping: &sm.PingSettings{
					IpVersion: sm.IpVersion_V4,
				},
			},
		},
	}

	prober, err := icmp.NewProber(check)
	if err != nil {
		t.Fatalf("cannot create ICMP prober: %s", err)
	}

	return prober, check, func() {}
}

func setupHTTPProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	httpSrv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.WriteHeader(http.StatusOK)
	}))
	httpSrv.Start()

	check := model.Check{
		Check: sm.Check{
			Target:  httpSrv.URL,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Http: &sm.HttpSettings{
					IpVersion: sm.IpVersion_V4,
				},
			},
		},
	}

	var store testhelper.NoopSecretStore

	prober, err := httpProber.NewProber(
		ctx,
		check,
		zerolog.New(io.Discard),
		http.Header{},
		store,
	)
	if err != nil {
		t.Fatalf("cannot create HTTP prober: %s", err)
	}

	return prober, check, httpSrv.Close
}

// TestHTTPTimeoutTelemetryContract pins the production HTTP prober behavior
// that deterministic history generators must reproduce. A request that does
// not receive response headers before the deadline has no HTTP status, emits
// the request-error log, and reports failure at approximately the timeout.
func TestHTTPTimeoutTelemetryContract(t *testing.T) {
	const timeout = 75 * time.Millisecond

	httpSrv := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, req *http.Request) {
		<-req.Context().Done()
	}))
	defer httpSrv.Close()

	check := model.Check{Check: sm.Check{
		Target:  httpSrv.URL,
		Timeout: timeout.Milliseconds(),
		Settings: sm.CheckSettings{Http: &sm.HttpSettings{
			IpVersion: sm.IpVersion_V4,
		}},
	}}

	var store testhelper.NoopSecretStore

	p, err := httpProber.NewProber(context.Background(), check, zerolog.New(io.Discard), http.Header{}, store)
	require.NoError(t, err)

	var logs bytes.Buffer

	success, mfs, err := getProbeMetrics(
		context.Background(),
		p,
		check.Target,
		timeout,
		nil,
		make(map[uint64]prometheus.Summary),
		make(map[uint64]prometheus.Histogram),
		&testLogger{w: &logs},
		false,
		"timeout-contract",
		scheduledScrapeClock(time.Now()),
	)
	require.NoError(t, err)
	require.False(t, success)
	require.Zero(t, metricGaugeValue(t, mfs, "probe_success", nil))
	require.Zero(t, metricGaugeValue(t, mfs, "probe_http_status_code", nil))
	require.Zero(t, metricGaugeValue(t, mfs, "probe_http_duration_seconds", map[string]string{"phase": "processing"}))
	require.Zero(t, metricGaugeValue(t, mfs, "probe_http_duration_seconds", map[string]string{"phase": "transfer"}))
	require.InDelta(t, timeout.Seconds(), metricGaugeValue(t, mfs, "probe_duration_seconds", nil), 0.050)
	require.Contains(t, logs.String(), "Error for HTTP request")
	require.NotContains(t, logs.String(), "Received HTTP response")
}

func metricGaugeValue(t *testing.T, families []*dto.MetricFamily, name string, labels map[string]string) float64 {
	t.Helper()

	for _, family := range families {
		if family.GetName() != name {
			continue
		}

		for _, metric := range family.GetMetric() {
			matched := true

			for label, value := range labels {
				found := false

				for _, pair := range metric.GetLabel() {
					if pair.GetName() == label && pair.GetValue() == value {
						found = true
						break
					}
				}

				if !found {
					matched = false
					break
				}
			}

			if matched && metric.GetGauge() != nil {
				return metric.GetGauge().GetValue()
			}
		}
	}

	t.Fatalf("metric %q with labels %v not found", name, labels)

	return 0
}

func setupHTTPSSLProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	httpSrv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Last-Modified", "Wed, 16 Apr 2026 19:00:00 GMT")
		w.Header().Set("ETag", `"fixture-http-ssl"`)
		w.Header().Set("Content-Type", "text/plain; charset=utf-8")
		w.WriteHeader(http.StatusOK)
		_, _ = io.WriteString(w, "fixture ok")
	}))
	httpSrv.StartTLS()

	check := model.Check{
		Check: sm.Check{
			Target:  httpSrv.URL,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Http: &sm.HttpSettings{
					IpVersion: sm.IpVersion_V4,
					TlsConfig: &sm.TLSConfig{
						InsecureSkipVerify: true,
					},
				},
			},
		},
	}

	var store testhelper.NoopSecretStore

	prober, err := httpProber.NewProber(
		ctx,
		check,
		zerolog.New(io.Discard),
		http.Header{},
		store,
	)
	if err != nil {
		t.Fatalf("cannot create HTTP prober: %s", err)
	}

	return prober, check, httpSrv.Close
}

func setupDNSProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	srv, clean := setupDNSServer(t)
	check := model.Check{
		Check: sm.Check{
			Target:  "example.org",
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Dns: &sm.DnsSettings{
					Server:     srv,
					IpVersion:  sm.IpVersion_V4,
					Protocol:   sm.DnsProtocol_UDP,
					RecordType: sm.DnsRecordType_SOA,
				},
			},
		},
	}

	prober, err := dnsProber.NewProber(check)
	if err != nil {
		clean()
		t.Fatalf("cannot create DNS prober: %s", err)
	}

	return prober, check, clean
}

func setupTCPProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	srv, clean := setupTCPServer(t)
	check := model.Check{
		Check: sm.Check{
			Target:  srv,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Tcp: &sm.TcpSettings{
					IpVersion: sm.IpVersion_V4,
				},
			},
		},
	}

	prober, err := tcp.NewProber(
		ctx,
		check,
		zerolog.New(io.Discard))
	if err != nil {
		clean()
		t.Fatalf("cannot create TCP prober: %s", err)
	}

	return prober, check, clean
}

func setupTCPSSLProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	srv, clean := setupTCPServerWithSSL(t)
	check := model.Check{
		Check: sm.Check{
			Target:  srv,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Tcp: &sm.TcpSettings{
					IpVersion: sm.IpVersion_V4,
					Tls:       true,
					TlsConfig: &sm.TLSConfig{
						InsecureSkipVerify: true,
					},
				},
			},
		},
	}

	prober, err := tcp.NewProber(
		ctx,
		check,
		zerolog.New(io.Discard))
	if err != nil {
		clean()
		t.Fatalf("cannot create TCP prober: %s", err)
	}

	return prober, check, clean
}

func setupTracerouteProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	checkCap := func(set *cap.Set, v cap.Value) {
		if permitted, err := set.GetFlag(cap.Permitted, v); err != nil {
			t.Fatalf("cannot get %s flag: %s", v, err)
		} else if !permitted {
			t.Skipf("traceroute cannot run, process doesn't have %s capability", v)
		}
	}
	c := cap.GetProc()
	checkCap(c, cap.NET_ADMIN)
	checkCap(c, cap.NET_RAW)

	check := model.Check{
		Check: sm.Check{
			Target: "127.0.0.1",
			Settings: sm.CheckSettings{
				Traceroute: &sm.TracerouteSettings{},
			},
		},
	}

	p, err := traceroute.NewProber(check, zerolog.New(io.Discard))
	if err != nil {
		t.Fatalf("cannot create traceroute prober %s", err)
	}

	return p, check, func() {}
}

func setupScriptedProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	httpSrv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte("scripted fixture ok"))
	}))

	check := model.Check{
		Check: sm.Check{
			Target:  httpSrv.URL,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Scripted: &sm.ScriptedSettings{
					Script: []byte(fmt.Sprintf(`
import http from 'k6/http';
import { check } from 'k6';

export const options = {
  insecureSkipTLSVerify: true,
};

export default function() {
  const response = http.get(%q);
  check(response, {
    ['status is 200']: (r) => r.status === 200,
  });
}
`, httpSrv.URL)),
				},
			},
		},
	}

	var (
		runner k6runner.Runner
		err    error
	)

	if k6Path := os.Getenv("K6_PATH"); k6Path != "" {
		runner, err = k6runner.New(k6runner.RunnerOpts{Uri: k6Path})
	} else {
		runner = &testRunner{
			metrics: testhelper.MustReadFile(t, "testdata/k6.dat"),
			logs:    nil,
		}
	}

	if err != nil {
		t.Fatalf("creating runner: %v", err)
	}

	var store testhelper.NoopSecretStore

	prober, err := scripted.NewProber(
		ctx,
		check,
		zerolog.New(zerolog.NewTestWriter(t)),
		runner,
		store,
	)
	if err != nil {
		t.Fatalf("cannot create scripted prober: %s", err)
	}

	return prober, check, httpSrv.Close
}

func setupMultiHTTPProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	httpSrv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(http.StatusOK)
		_, _ = w.Write([]byte(`{"status":"ok","message":"multihttp fixture ok"}`))
	}))

	check := model.Check{
		Check: sm.Check{
			Target:  httpSrv.URL,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Multihttp: &sm.MultiHttpSettings{
					Entries: []*sm.MultiHttpEntry{
						{
							Request: &sm.MultiHttpEntryRequest{
								Method: sm.HttpMethod_GET,
								Url:    httpSrv.URL,
							},
							Assertions: []*sm.MultiHttpEntryAssertion{
								{
									Type:      sm.MultiHttpEntryAssertionType_TEXT,
									Subject:   sm.MultiHttpEntryAssertionSubjectVariant_HTTP_STATUS_CODE,
									Condition: sm.MultiHttpEntryAssertionConditionVariant_EQUALS,
									Value:     "200",
								},
								{
									Type:      sm.MultiHttpEntryAssertionType_TEXT,
									Subject:   sm.MultiHttpEntryAssertionSubjectVariant_RESPONSE_BODY,
									Condition: sm.MultiHttpEntryAssertionConditionVariant_CONTAINS,
									Value:     "multihttp fixture ok",
								},
							},
						},
					},
				},
			},
		},
	}

	var (
		runner k6runner.Runner
		err    error
	)

	if k6Path := os.Getenv("K6_PATH"); k6Path != "" {
		t.Setenv("K6_INSECURE_SKIP_TLS_VERIFY", "true")

		runner, err = k6runner.New(k6runner.RunnerOpts{Uri: k6Path})
	} else {
		runner = &testRunner{
			metrics: testhelper.MustReadFile(t, "testdata/multihttp.dat"),
			logs:    nil,
		}
	}

	if err != nil {
		t.Fatalf("building k6 runner: %v", err)
	}

	prober, err := multihttp.NewProber(
		ctx,
		check,
		zerolog.New(zerolog.NewTestWriter(t)),
		runner,
		http.Header{},
		testhelper.NoopSecretStore{},
	)
	if err != nil {
		t.Fatalf("cannot create MultiHTTP prober: %s", err)
	}

	return prober, check, httpSrv.Close
}

func setupBrowserProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	mux := http.NewServeMux()
	mux.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		_, _ = io.WriteString(w, `<!doctype html>
<html>
  <head>
    <title>browser fixture</title>
    <link rel="stylesheet" href="/style.css">
    <script defer src="/app.js"></script>
  </head>
  <body>
    <h1 id="title">Synthetic Monitoring</h1>
    <p id="status">browser fixture ok</p>
    <button id="interact" type="button">Interact</button>
    <img src="/pixel.png" alt="pixel">
  </body>
</html>`)
	})
	mux.HandleFunc("/style.css", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "text/css")
		_, _ = io.WriteString(w, "body { font-family: sans-serif; }")
	})
	mux.HandleFunc("/app.js", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "application/javascript")
		_, _ = io.WriteString(w, `window.addEventListener('load', () => {
  const status = document.getElementById('status');
  const button = document.getElementById('interact');
  if (status) status.textContent = 'browser fixture ready';
  if (button && status) {
    button.addEventListener('click', () => {
      const start = performance.now();
      while (performance.now() - start < 120) {
        // Busy wait long enough to ensure the browser runtime records an input delay.
      }
      requestAnimationFrame(() => {
        status.textContent = 'interaction complete';
      });
    });
  }
});`)
	})
	mux.HandleFunc("/pixel.png", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Content-Type", "image/png")
		_, _ = w.Write([]byte{
			0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a,
			0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52,
			0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01,
			0x08, 0x06, 0x00, 0x00, 0x00, 0x1f, 0x15, 0xc4,
			0x89, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x44, 0x41,
			0x54, 0x78, 0x9c, 0x63, 0xf8, 0xcf, 0xc0, 0x00,
			0x00, 0x03, 0x01, 0x01, 0x00, 0xc9, 0xfe, 0x92,
			0xef, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e,
			0x44, 0xae, 0x42, 0x60, 0x82,
		})
	})
	httpSrv := httptest.NewServer(mux)

	check := model.Check{
		Check: sm.Check{
			Target:  httpSrv.URL,
			Timeout: 10000,
			Settings: sm.CheckSettings{
				Browser: &sm.BrowserSettings{
					Script: []byte(fmt.Sprintf(`
import { browser } from 'k6/browser';
import { check, sleep } from 'k6';

export const options = {
  scenarios: {
    ui: {
      executor: 'shared-iterations',
      options: {
        browser: {
          type: 'chromium',
        },
      },
    },
  },
};

export default async function () {
  const context = await browser.newContext();
  const page = await context.newPage();

  try {
    await page.goto(%q, { waitUntil: 'networkidle' });
    const title = await page.locator('#title').textContent();
    check(title, {
      ['heading is present']: (value) => value === 'Synthetic Monitoring',
    });
    const button = page.locator('#interact');
    await button.click();
    await page.waitForFunction(() => {
      const status = document.getElementById('status');
      return status && status.textContent === 'interaction complete';
    });
    sleep(2);
  } finally {
    await page.close();
  }
}
`, httpSrv.URL)),
				},
			},
		},
	}

	var (
		runner k6runner.Runner
		err    error
	)

	if k6Path := os.Getenv("K6_PATH"); k6Path != "" {
		runner, err = k6runner.New(k6runner.RunnerOpts{Uri: k6Path})
	} else {
		runner = &testRunner{
			metrics: testhelper.MustReadFile(t, "testdata/browser.dat"),
			logs:    nil,
		}
	}

	if err != nil {
		t.Fatalf("building k6 runner: %v", err)
	}

	prober, err := browser.NewProber(
		ctx,
		check,
		zerolog.New(zerolog.NewTestWriter(t)),
		runner,
		testhelper.NoopSecretStore{},
	)
	if err != nil {
		t.Fatalf("cannot create scripted prober: %s", err)
	}

	return prober, check, httpSrv.Close
}

func setupGRPCProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	srv, clean := setupGRPCServer(t)
	check := model.Check{
		Check: sm.Check{
			Target:  srv,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Grpc: &sm.GrpcSettings{
					IpVersion: sm.IpVersion_V4,
				},
			},
		},
	}

	prober, err := grpcProber.NewProber(
		ctx,
		check,
		zerolog.New(io.Discard))
	if err != nil {
		clean()
		t.Fatalf("cannot create gRPC prober: %s", err)
	}

	return prober, check, clean
}

func setupGRPCSSLProbe(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func()) {
	srv, clean := setupGRPCServerWithSSL(t)
	check := model.Check{
		Check: sm.Check{
			Target:  srv,
			Timeout: 2000,
			Settings: sm.CheckSettings{
				Grpc: &sm.GrpcSettings{
					IpVersion: sm.IpVersion_V4,
					Tls:       true,
					TlsConfig: &sm.TLSConfig{
						InsecureSkipVerify: true,
					},
				},
			},
		},
	}

	prober, err := grpcProber.NewProber(
		ctx,
		check,
		zerolog.New(io.Discard))
	if err != nil {
		clean()
		t.Fatalf("cannot create gRPC prober: %s", err)
	}

	return prober, check, clean
}

func setupDNSServer(t *testing.T) (string, func()) {
	dnsSrv, dnsAddr := startDNSServer(":0", "udp", recursiveDNSHandler)

	errCh := make(chan error)

	go func() {
		dnsSrv.NotifyStartedFunc = func() {
			close(errCh)
		}

		err := dnsSrv.ActivateAndServe()
		if err != nil {
			errCh <- err

			close(errCh)
		}
	}()

	if err := <-errCh; err != nil {
		t.Fatalf("error activating DNS server: %s", err.Error())
	}

	return dnsAddr.String(), func() {
		err := dnsSrv.Shutdown()
		if err != nil {
			// this should never happen, but log it if it does
			t.Fatalf("error shutting down DNS server: %s", err.Error())
		}
	}
}

// startDNSServer starts a DNS server with a given handler function on a random port.
// Returns the Server object itself as well as the net.Addr corresponding to the server port.
func startDNSServer(addr, protocol string, handler func(dns.ResponseWriter, *dns.Msg)) (*dns.Server, net.Addr) {
	h := dns.NewServeMux()
	h.HandleFunc(".", handler)

	server := &dns.Server{Net: protocol, Handler: h}
	switch protocol {
	case "udp", "udp4", "udp6":
		a, err := net.ResolveUDPAddr(server.Net, server.Addr)
		if err != nil {
			panic(err)
		}

		l, err := net.ListenUDP(server.Net, a)
		if err != nil {
			panic(err)
		}

		server.Addr = l.LocalAddr().String()
		server.PacketConn = l
	case "tcp", "tcp4", "tcp6":
		a, err := net.ResolveTCPAddr(server.Net, server.Addr)
		if err != nil {
			panic(err)
		}

		l, err := net.ListenTCP(server.Net, a)
		if err != nil {
			panic(err)
		}

		server.Addr = l.Addr().String()
		server.Listener = l
	default:
		panic("unknown protocol")
	}

	if protocol == "tcp" {
		return server, server.Listener.Addr()
	}

	return server, server.PacketConn.LocalAddr()
}

func recursiveDNSHandler(w dns.ResponseWriter, r *dns.Msg) {
	m := new(dns.Msg)
	m.SetReply(r)

	if len(r.Question) == 0 {
		if err := w.WriteMsg(m); err != nil {
			panic(err)
		}

		return
	}

	var answers []string

	switch r.Question[0].Qtype {
	case dns.TypeSOA:
		answers = []string{
			"example.org. 3600 IN SOA ns1.example.org. hostmaster.example.org. 2026041601 7200 3600 1209600 3600",
		}
	default:
		answers = []string{
			"example.org. 3600 IN A 127.0.0.1",
			"example.org. 3600 IN A 127.0.0.2",
		}
	}

	for _, rr := range answers {
		a, err := dns.NewRR(rr)
		if err != nil {
			panic(err)
		}

		m.Answer = append(m.Answer, a)
	}

	if err := w.WriteMsg(m); err != nil {
		panic(err)
	}
}

func setupTCPServer(t *testing.T) (string, func()) {
	ln, err := net.Listen("tcp4", ":0")
	if err != nil {
		t.Fatalf("Error listening on socket: %s", err)
	}

	done := make(chan (struct{}))

	go func() {
		conn, err := ln.Accept()
		if err != nil {
			panic(fmt.Sprintf("Error accepting on socket: %s", err))
		}

		defer func() {
			conn.Close()
			close(done)
		}()

		buf := make([]byte, 0, 1024)
		_, _ = conn.Read(buf)
	}()

	return ln.Addr().String(), func() {
		<-done
		ln.Close()
	}
}

func setupTCPServerWithSSL(t *testing.T) (string, func()) {
	ln, err := net.Listen("tcp4", ":0")
	if err != nil {
		t.Fatalf("Error listening on socket: %s", err)
	}

	cert, err := tls.X509KeyPair(localhostCert, localhostKey)
	if err != nil {
		t.Fatalf("creating X509 key pair: %s", err.Error())
	}

	tlsCfg := tls.Config{
		Certificates: []tls.Certificate{cert},
		NextProtos:   []string{"foo"},
	}

	ln = tls.NewListener(ln, &tlsCfg)

	done := make(chan (struct{}))

	go func() {
		conn, err := ln.Accept()
		if err != nil {
			panic(fmt.Sprintf("Error accepting on socket: %s", err))
		}

		defer func() {
			conn.Close()
			close(done)
		}()

		buf := make([]byte, 0, 1024)
		_, _ = conn.Read(buf)
	}()

	return ln.Addr().String(), func() {
		<-done
		ln.Close()
	}
}

type gRPCSrv struct {
	grpchealth.HealthServer
}

func (s *gRPCSrv) Check(
	context.Context, *grpchealth.HealthCheckRequest,
) (*grpchealth.HealthCheckResponse, error) {
	return &grpchealth.HealthCheckResponse{
		Status: grpchealth.HealthCheckResponse_SERVING,
	}, nil
}

func setupGRPCServer(t *testing.T) (string, func()) {
	lis, err := net.Listen("tcp4", ":0")
	if err != nil {
		t.Fatalf("Error listening on socket: %v", err)
	}

	srv := grpc.NewServer()
	grpchealth.RegisterHealthServer(srv, &gRPCSrv{})

	go func() {
		_ = srv.Serve(lis)
	}()

	return lis.Addr().String(), func() {
		srv.Stop()
		lis.Close()
	}
}

func setupGRPCServerWithSSL(t *testing.T) (string, func()) {
	lis, err := net.Listen("tcp4", ":0")
	if err != nil {
		t.Fatalf("Error listening on socket: %v", err)
	}

	cert, err := tls.X509KeyPair(localhostCert, localhostKey)
	if err != nil {
		t.Fatalf("Error creating X509 key pair: %v", err)
	}

	tlsCfg := tls.Config{
		Certificates: []tls.Certificate{cert},
		NextProtos:   []string{"foo"},
	}

	srv := grpc.NewServer(
		grpc.Creds(credentials.NewTLS(&tlsCfg)),
	)
	grpchealth.RegisterHealthServer(srv, &gRPCSrv{})

	go func() {
		_ = srv.Serve(lis)
	}()

	return lis.Addr().String(), func() {
		srv.Stop()
		lis.Close()
	}
}

// TestValidateLabels validates that no probe generates more metric or log
// labels than the exported constants that specify these maximums.
// This test is required to be aware if any modification extends these
// maximums. These maximums are useful to calculate how many check labels can
// be set without exceeding the Mimir and Loki limits.
func TestValidateLabels(t *testing.T) {
	testCases := map[string]struct {
		setup func(ctx context.Context, t *testing.T) (prober.Prober, model.Check, func())
	}{
		"ping": {
			setup: setupPingProbe,
		},
		"http": {
			setup: setupHTTPProbe,
		},
		"http_ssl": {
			setup: setupHTTPSSLProbe,
		},
		"dns": {
			setup: setupDNSProbe,
		},
		"tcp": {
			setup: setupTCPProbe,
		},
		"tcp_ssl": {
			setup: setupTCPSSLProbe,
		},
		"traceroute": {
			setup: setupTracerouteProbe,
		},
		"scripted": {
			setup: setupScriptedProbe,
		},
		"multihttp": {
			setup: setupMultiHTTPProbe,
		},
		"grpc": {
			setup: setupGRPCProbe,
		},
		"grpc_ssl": {
			setup: setupGRPCSSLProbe,
		},
	}

	type maxMetricLabels struct {
		AllMetrics      int
		CheckInfoMetric int
	}

	isCheckInfoMetric := func(labels []prompb.Label) bool {
		for _, l := range labels {
			if l.Name == "__name__" && l.Value == CheckInfoMetricName {
				return true
			}
		}

		return false
	}

	// maxProbeMetricLabels returns the maximum number of labels for any
	// timeseries from probeData, and also the maximum number of labels for the
	// sm_check_info metric (that should be fairly static across all checks,
	// only depends on alert sensitivity, but review all generated tss anyway).
	maxProbeMetricLabels := func(t *testing.T, tss TimeSeries) maxMetricLabels {
		max := 0
		maxCheckInfoMetric := 0

		for _, ts := range tss {
			labels := ts.GetLabels()

			nLabels := len(labels) - 1 // Discount special __name__ label
			if nLabels > max {
				max = nLabels
			}
			// Check for CheckInfoMetric
			if isCheckInfoMetric(labels) {
				if nLabels > maxCheckInfoMetric {
					maxCheckInfoMetric = nLabels
				}
			}
		}

		return maxMetricLabels{
			AllMetrics:      max,
			CheckInfoMetric: maxCheckInfoMetric,
		}
	}

	// maxProbeLogLabels returns the maximum number of labels for any
	// log stream from probeData
	maxProbeLogLabels := func(t *testing.T, ss Streams) int {
		n := 0

		for _, s := range ss {
			labels, err := metricParser.ParseMetric(s.Labels)
			require.NoError(t, err)

			n = max(n, labels.Len())
		}

		return n
	}

	timeout := 10 * time.Second

	ctx, cancel := context.WithTimeout(context.Background(), timeout)
	defer cancel()

	var (
		totalMaxMetricLabels    int
		totalMaxCheckInfoLabels int
		totalMaxLogLabels       int
	)

	for name, tc := range testCases {
		t.Run(name, func(t *testing.T) {
			prober, check, stop := tc.setup(ctx, t)
			defer stop()

			check.AlertSensitivity = "low" // Force sm_check_info metric to include alert sensitivity label

			s := Scraper{
				checkName: "check name",
				target:    check.Target,
				logger:    testhelper.Logger(t),
				prober:    prober,
				labelsLimiter: testLabelsLimiter{
					maxMetricLabels: 100,
					maxLogLabels:    100,
				},
				labellingMode: testLabellingMode{},
				summaries:     make(map[uint64]prometheus.Summary),
				histograms:    make(map[uint64]prometheus.Histogram),
				check:         check,
				probe: sm.Probe{
					Id:        100,
					TenantId:  200,
					Name:      "probe name",
					Latitude:  -1,
					Longitude: -2,
					Region:    "region",
				},
			}

			data, duration, err := s.collectData(context.Background(), time.Unix(int64(3141000)/1000, 0))
			require.NoError(t, err)
			require.NotNil(t, data)
			require.NotZero(t, duration)

			metricLabels := maxProbeMetricLabels(t, data.Metrics())
			logLabels := maxProbeLogLabels(t, data.Streams())

			if metricLabels.AllMetrics > totalMaxMetricLabels {
				totalMaxMetricLabels = metricLabels.AllMetrics
			}

			if metricLabels.CheckInfoMetric > totalMaxCheckInfoLabels {
				totalMaxCheckInfoLabels = metricLabels.CheckInfoMetric
			}

			if logLabels > totalMaxLogLabels {
				totalMaxLogLabels = logLabels
			}
		})
	}

	require.Equal(t, sm.MaxAgentMetricLabels(), totalMaxMetricLabels)
	require.Equal(t, sm.MaxAgentCheckInfoLabels(), totalMaxCheckInfoLabels)
	require.Equal(t, sm.MaxAgentLogLabels(), totalMaxLogLabels)
}

func TestMakeTimeseries(t *testing.T) {
	testTime := time.Now()
	testValue := 42.0
	testcases := map[string]struct {
		t      time.Time
		value  float64
		labels []prompb.Label
	}{
		"no labels": {
			t:      testTime,
			value:  testValue,
			labels: []prompb.Label{},
		},
		"one label": {
			t:     testTime,
			value: testValue,
			labels: []prompb.Label{
				{Name: "name", Value: "value"},
			},
		},
		"many labels": {
			t:     testTime,
			value: testValue,
			labels: []prompb.Label{
				{Name: "name 1", Value: "value 1"},
				{Name: "name 2", Value: "value 2"},
				{Name: "name 3", Value: "value 3"},
				{Name: "name 4", Value: "value 4"},
			},
		},
	}

	for name, tc := range testcases {
		t.Run(name, func(t *testing.T) {
			actual := makeTimeseries(tc.t, tc.value, tc.labels...)
			require.Equal(t, len(tc.labels), len(actual.Labels))
			require.Equal(t, tc.labels, actual.Labels)
			require.Equal(t, 1, len(actual.Samples))
			require.Equal(t, tc.t.UnixNano()/1e6, actual.Samples[0].Timestamp)
			require.Equal(t, tc.value, actual.Samples[0].Value)
		})
	}
}

// TestMakeTimeseriesDeduplicates verifies that makeTimeseries drops duplicate
// label names, keeping the first occurrence (Mimir rejects duplicate keys)
func TestMakeTimeseriesDeduplicates(t *testing.T) {
	ts := makeTimeseries(time.Unix(1, 0), 1,
		prompb.Label{Name: prom.MetricNameLabel, Value: "m"},
		prompb.Label{Name: "probe", Value: "user-probe"},
		prompb.Label{Name: "job", Value: "system"},
		prompb.Label{Name: "probe", Value: "system"}, // duplicate name — must be dropped
		prompb.Label{Name: "env", Value: "prod"},
	)

	require.Equal(t, []prompb.Label{
		{Name: prom.MetricNameLabel, Value: "m"},
		{Name: "probe", Value: "user-probe"}, // first occurrence wins
		{Name: "job", Value: "system"},
		{Name: "env", Value: "prod"},
	}, ts.Labels)
}

// TestScraperCollectDataTooManyMetricLabels verifies that the check-info label
// guard rejects a scrape whose non-system label count exceeds the tenant's
// metric-label budget, before any data is emitted.
func TestScraperCollectDataTooManyMetricLabels(t *testing.T) {
	s := Scraper{
		checkName:     "check name",
		target:        "test target",
		logger:        testhelper.Logger(t),
		prober:        testProber{},
		labelsLimiter: testLabelsLimiter{maxMetricLabels: 1, maxLogLabels: 15},
		labellingMode: testLabellingMode{},
		summaries:     make(map[uint64]prometheus.Summary),
		histograms:    make(map[uint64]prometheus.Histogram),
		check: model.Check{
			Check: sm.Check{
				Id:               1,
				TenantId:         2,
				Frequency:        2000,
				Timeout:          2000,
				Enabled:          true,
				Target:           "target name",
				Job:              "job name",
				BasicMetricsOnly: true,
				Created:          42,
				Modified:         42,
				Labels: []sm.Label{
					{Name: "a", Value: "1"},
					{Name: "b", Value: "2"},
					{Name: "c", Value: "3"},
				},
				Settings: sm.CheckSettings{Http: &sm.HttpSettings{}},
			},
		},
		probe: sm.Probe{Id: 100, TenantId: 200, Name: "probe name", Region: "REGION"},
	}

	_, _, err := s.collectData(context.Background(), time.Unix(3141, 0))
	require.ErrorIs(t, err, errTooManyLabels)
}

// TestScraperCollectDataFatalError verifies that the exported CollectData
// surfaces a fatal collection error (one that is not errCheckFailed) with no
// telemetry, matching its documented contract.
func TestScraperCollectDataFatalError(t *testing.T) {
	s := Scraper{
		checkName:     "check name",
		target:        "test target",
		logger:        testhelper.Logger(t),
		prober:        testProber{},
		labelsLimiter: testLabelsLimiter{maxMetricLabels: 1, maxLogLabels: 15},
		labellingMode: testLabellingMode{},
		summaries:     make(map[uint64]prometheus.Summary),
		histograms:    make(map[uint64]prometheus.Histogram),
		check: model.Check{
			Check: sm.Check{
				Id:               1,
				TenantId:         2,
				Frequency:        2000,
				Timeout:          2000,
				Enabled:          true,
				Target:           "target name",
				Job:              "job name",
				BasicMetricsOnly: true,
				Created:          42,
				Modified:         42,
				Labels: []sm.Label{
					{Name: "a", Value: "1"},
					{Name: "b", Value: "2"},
					{Name: "c", Value: "3"},
				},
				Settings: sm.CheckSettings{Http: &sm.HttpSettings{}},
			},
		},
		probe: sm.Probe{Id: 100, TenantId: 200, Name: "probe name", Region: "REGION"},
	}

	ts, streams, tenant, d, err := s.CollectData(context.Background(), time.Unix(3141, 0))
	require.ErrorIs(t, err, errTooManyLabels)
	require.NotErrorIs(t, err, errCheckFailed)
	require.Nil(t, ts)
	require.Nil(t, streams)
	require.Zero(t, tenant)
	require.Zero(t, d)
}

func TestAppendDtoToTimeseries(t *testing.T) {
	makeUint64Ptr := func(n uint64) *uint64 {
		return &n
	}

	makeFloat64Ptr := func(n float64) *float64 {
		return &n
	}

	testTime := time.Now()
	testTimestamp := testTime.UnixNano() / 1e6
	testValue := 42.0
	testcases := map[string]struct {
		t            time.Time
		mName        string
		sharedLabels []prompb.Label
		mType        dto.MetricType
		metric       *dto.Metric
		expected     []prompb.TimeSeries
	}{
		"counter": {
			t:     testTime,
			mName: "test",
			sharedLabels: []prompb.Label{
				{Name: "label 1", Value: "value 1"},
			},
			mType: dto.MetricType_COUNTER,
			metric: &dto.Metric{
				Counter: &dto.Counter{
					Value: &testValue,
				},
			},
			expected: []prompb.TimeSeries{
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test"},
						{Name: "label 1", Value: "value 1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: testValue},
					},
				},
			},
		},
		"gauge": {
			t:     testTime,
			mName: "test",
			sharedLabels: []prompb.Label{
				{Name: "label 1", Value: "value 1"},
			},
			mType: dto.MetricType_GAUGE,
			metric: &dto.Metric{
				Gauge: &dto.Gauge{
					Value: &testValue,
				},
			},
			expected: []prompb.TimeSeries{
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test"},
						{Name: "label 1", Value: "value 1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: testValue},
					},
				},
			},
		},
		"untyped": {
			t:     testTime,
			mName: "test",
			sharedLabels: []prompb.Label{
				{Name: "label 1", Value: "value 1"},
			},
			mType: dto.MetricType_UNTYPED,
			metric: &dto.Metric{
				Untyped: &dto.Untyped{
					Value: &testValue,
				},
			},
			expected: []prompb.TimeSeries{
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test"},
						{Name: "label 1", Value: "value 1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: testValue},
					},
				},
			},
		},
		"summary": {
			t:     testTime,
			mName: "test",
			sharedLabels: []prompb.Label{
				{Name: "label 1", Value: "value 1"},
			},
			mType: dto.MetricType_SUMMARY,
			metric: &dto.Metric{
				Summary: &dto.Summary{
					SampleCount: makeUint64Ptr(7),
					SampleSum:   makeFloat64Ptr(0.25),
				},
			},
			expected: []prompb.TimeSeries{
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_sum"},
						{Name: "label 1", Value: "value 1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 0.25},
					},
				},
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_count"},
						{Name: "label 1", Value: "value 1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 7},
					},
				},
			},
		},
		"histogram": {
			t:     testTime,
			mName: "test",
			sharedLabels: []prompb.Label{
				{Name: "label 1", Value: "value 1"},
			},
			mType: dto.MetricType_HISTOGRAM,
			metric: &dto.Metric{
				Histogram: &dto.Histogram{
					SampleCount: makeUint64Ptr(17),
					SampleSum:   makeFloat64Ptr(120),
					Bucket: []*dto.Bucket{
						{
							CumulativeCount: makeUint64Ptr(1),
							UpperBound:      makeFloat64Ptr(0.1),
						},
						{
							CumulativeCount: makeUint64Ptr(5),
							UpperBound:      makeFloat64Ptr(1.0),
						},
						{
							CumulativeCount: makeUint64Ptr(11),
							UpperBound:      makeFloat64Ptr(10.0),
						},
					},
				},
			},
			expected: []prompb.TimeSeries{
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_sum"},
						{Name: "label 1", Value: "value 1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 120},
					},
				},
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_count"},
						{Name: "label 1", Value: "value 1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 17},
					},
				},
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_bucket"},
						{Name: "label 1", Value: "value 1"},
						{Name: "le", Value: "0.1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 1},
					},
				},
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_bucket"},
						{Name: "label 1", Value: "value 1"},
						{Name: "le", Value: "1"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 5},
					},
				},
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_bucket"},
						{Name: "label 1", Value: "value 1"},
						{Name: "le", Value: "10"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 11},
					},
				},
				{
					Labels: []prompb.Label{
						{Name: "__name__", Value: "test_bucket"},
						{Name: "label 1", Value: "value 1"},
						{Name: "le", Value: "+Inf"},
					},
					Samples: []prompb.Sample{
						{Timestamp: testTimestamp, Value: 17},
					},
				},
			},
		},
	}

	for name, tc := range testcases {
		t.Run(name, func(t *testing.T) {
			actual := appendDtoToTimeseries(
				nil,
				tc.t,
				tc.mName,
				tc.sharedLabels,
				tc.mType,
				tc.metric,
			)

			require.Equal(t, tc.expected, actual)
		})
	}
}

type testProber struct{}

func (p testProber) Name() string {
	return "test prober"
}

func (p testProber) Probe(ctx context.Context, target string, registry *prometheus.Registry, logger logger.Logger, _ string) (bool, float64) {
	counter := prometheus.NewCounter(prometheus.CounterOpts{
		Name: "test_counter",
	})
	registry.MustRegister(counter)
	counter.Inc()

	return true, 1
}

type testLabelsLimiter struct {
	maxMetricLabels int
	maxLogLabels    int
}

func (l testLabelsLimiter) MetricLabels(ctx context.Context, tenantID model.GlobalID) (int, error) {
	return l.maxMetricLabels, nil
}

func (l testLabelsLimiter) LogLabels(ctx context.Context, tenantID model.GlobalID) (int, error) {
	return l.maxLogLabels, nil
}

type testLabellingMode struct {
	mode sm.LabelMode
	err  error
}

func (l testLabellingMode) ForTenant(ctx context.Context, tenantID model.GlobalID) (sm.LabelMode, error) {
	return l.mode, l.err
}

type testCalTenants struct {
	costAttributionLabels []string
}

func (t testCalTenants) CostAttributionLabels(_ context.Context, tenantID model.GlobalID) ([]string, error) {
	return t.costAttributionLabels, nil
}

//nolint:gocyclo
func TestScraperCollectData(t *testing.T) {
	const (
		checkName     = "check name"
		checkTarget   = "target name"
		frequency     = 2000
		job           = "job name"
		modifiedTs    = 42
		probeLatitude = -1
		probeLongitde = -2
		probeName     = "probe name"
		region        = "REGION"
		sampleTsMs    = int64(3141000)
	)

	var (
		baseExpectedMetricLabels = map[string]string{
			configVersionLabelName: strconv.Itoa(modifiedTs * 1_000_000_000),
			"instance":             checkTarget,
			"job":                  job,
			"probe":                probeName,
			// "source":         CheckInfoSource,
		}
		baseExpectedInfoLabels = map[string]string{
			"check_name": checkName,
			"frequency":  strconv.Itoa(frequency),
			"geohash":    geohash.Encode(probeLatitude, probeLongitde),
			"region":     region,
		}
		baseExpectedLogLabels = map[string]string{
			"check_name":           checkName,
			"instance":             checkTarget,
			"job":                  job,
			"probe":                probeName,
			ProbeSuccessMetricName: "1",
			"region":               region,
			"source":               CheckInfoSource,
		}

		// Tenants that have started the label migration (DUAL_WRITE/UNPREFIXED)
		// receive the agent's region label as sm_region instead of region.
		baseExpectedInfoLabelsSMRegion = map[string]string{
			"check_name": checkName,
			"frequency":  strconv.Itoa(frequency),
			"geohash":    geohash.Encode(probeLatitude, probeLongitde),
			"sm_region":  region,
		}
		baseExpectedLogLabelsSMRegion = map[string]string{
			"check_name":           checkName,
			"instance":             checkTarget,
			"job":                  job,
			"probe":                probeName,
			ProbeSuccessMetricName: "1",
			"sm_region":            region,
			"source":               CheckInfoSource,
		}
	)

	generateLabels := func(offset, count int, valuePrefix string) []sm.Label {
		var labels []sm.Label

		for i := range count {
			n := strconv.Itoa(offset + i)
			labels = append(labels, sm.Label{
				Name:  "l" + n,
				Value: valuePrefix + n,
			})
		}

		return labels
	}

	generateLabelSet := func(offset, count int, valuePrefix string) map[string]string {
		labels := make(map[string]string)

		for i := range count {
			n := strconv.Itoa(offset + i)
			labels["label_l"+n] = valuePrefix + n
		}

		return labels
	}

	// generateUnprefixedLabelSet produces {"l0":"c0","l1":"c1",...} — used to describe
	// expected user labels on execution metrics in DUAL_WRITE and UNPREFIXED modes.
	generateUnprefixedLabelSet := func(offset, count int, valuePrefix string) map[string]string {
		labels := make(map[string]string)

		for i := range count {
			n := strconv.Itoa(offset + i)
			labels["l"+n] = valuePrefix + n
		}

		return labels
	}

	type testcase struct {
		labelMode            sm.LabelMode // zero value = PREFIXED
		labelModeErr         error        // simulates a tenant label-mode lookup failure
		maxMetricLabels      int
		maxLogLabels         int
		checkLabels          []sm.Label
		probeLabels          []sm.Label
		expectedMetricLabels map[string]string
		expectedInfoLabels   map[string]string
		expectedLogLabels    map[string]string
		expectedLogEntries   map[string]string
	}

	const (
		defMaxMetricLabels = 22
		defMaxLogLabels    = 15
	)

	testcases := map[string]testcase{
		"trivial": {
			maxMetricLabels:      defMaxMetricLabels,
			maxLogLabels:         defMaxLogLabels,
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabels),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabels),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabels),
		},
		"probe labels": {
			maxMetricLabels:      defMaxMetricLabels,
			maxLogLabels:         defMaxLogLabels,
			probeLabels:          generateLabels(1, 3, "p"),
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabels, generateLabelSet(1, 3, "p")),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabels, generateLabelSet(1, 3, "p")),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabels, generateLabelSet(1, 3, "p")),
		},
		"check labels": {
			maxMetricLabels:      defMaxMetricLabels,
			maxLogLabels:         defMaxLogLabels,
			checkLabels:          generateLabels(1, 3, "c"),
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabels, generateLabelSet(1, 3, "c")),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabels, generateLabelSet(1, 3, "c")),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabels, generateLabelSet(1, 3, "c")),
		},
		"check and probe labels": {
			maxMetricLabels:      defMaxMetricLabels,
			maxLogLabels:         defMaxLogLabels,
			checkLabels:          generateLabels(1, 2, "c"),
			probeLabels:          generateLabels(3, 1, "p"),
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabels, generateLabelSet(3, 1, "p"), generateLabelSet(1, 2, "c")),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabels, generateLabelSet(3, 1, "p"), generateLabelSet(1, 2, "c")),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabels, generateLabelSet(3, 1, "p"), generateLabelSet(1, 2, "c")),
		},
		"check and probe labels overlapping": {
			maxMetricLabels:      defMaxMetricLabels,
			maxLogLabels:         defMaxLogLabels,
			checkLabels:          generateLabels(1, 2, "c"),
			probeLabels:          generateLabels(2, 2, "p"),
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabels, generateLabelSet(3, 1, "p"), generateLabelSet(1, 2, "c")),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabels, generateLabelSet(3, 1, "p"), generateLabelSet(1, 2, "c")),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabels, generateLabelSet(3, 1, "p"), generateLabelSet(1, 2, "c")),
		},
		"max labels truncate": {
			maxMetricLabels:      defMaxMetricLabels,
			maxLogLabels:         defMaxLogLabels,
			checkLabels:          generateLabels(0, 10, "c"),
			probeLabels:          generateLabels(10, 3, "p"),
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabels, generateLabelSet(0, 10, "c"), generateLabelSet(10, 3, "p")),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabels, generateLabelSet(0, 5, "c"), generateLabelSet(10, 3, "p")),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabels, generateLabelSet(0, 10, "c"), generateLabelSet(10, 3, "p")),
		},
		"max labels override": {
			maxMetricLabels:      23,
			maxLogLabels:         23,
			checkLabels:          generateLabels(0, 10, "c"),
			probeLabels:          generateLabels(10, 4, "p"),
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabels, generateLabelSet(0, 10, "c"), generateLabelSet(10, 4, "p")),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabels, generateLabelSet(0, 10, "c"), generateLabelSet(10, 4, "p")),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabels, generateLabelSet(0, 10, "c"), generateLabelSet(10, 4, "p")),
		},

		// ── DUAL_WRITE and UNPREFIXED mode tests ──────────────────────────────────
		//
		// Key differences:
		//   expectedMetricLabels includes un-prefixed user labels (execution metrics
		//     gain user labels via customMetricLabels in DUAL_WRITE/UNPREFIXED).
		//   expectedInfoLabels includes both forms in DUAL_WRITE (prefixed first in the
		//     slice produced by buildUserLabels, then un-prefixed) and un-prefixed only
		//     in UNPREFIXED.
		//   expectedLogLabels follows DUAL_WRITE ordering: all prefixed forms before all
		//     un-prefixed forms in the stream labels; overflow goes to StructuredMetadata.

		"dual write basic": {
			// 3 check labels + 1 probe label; no stream-label overflow.
			// Verifies: execution metrics carry un-prefixed user labels;
			//           sm_check_info carries both forms;
			//           log stream carries both forms in correct ordering.
			labelMode:       sm.LabelMode_LABEL_MODE_DUAL_WRITE,
			maxMetricLabels: defMaxMetricLabels,
			maxLogLabels:    defMaxLogLabels,
			checkLabels:     generateLabels(0, 3, "c"),
			probeLabels:     generateLabels(10, 1, "p"),
			// Execution metrics: base + un-prefixed user labels.
			expectedMetricLabels: mergeMaps(
				baseExpectedMetricLabels,
				generateUnprefixedLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
			// sm_check_info: system sharedLabels (probe, config_version, instance, job)
			// + check-info system labels + both forms of user labels from ConstLabels.
			expectedInfoLabels: mergeMaps(
				baseExpectedMetricLabels,
				baseExpectedInfoLabelsSMRegion,
				generateLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(10, 1, "p"),
				generateLabelSet(0, 3, "c"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
			// Log stream: DUAL_WRITE ordering — all prefixed first, then un-prefixed.
			// 4 user labels × 2 = 8 entries + 6 system labels = 14; probe_success
			// is appended last, totalling 15 = defMaxLogLabels. No overflow.
			expectedLogLabels: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				generateLabelSet(10, 1, "p"),
				generateLabelSet(0, 3, "c"),
				generateUnprefixedLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
			expectedLogEntries: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				generateLabelSet(10, 1, "p"),
				generateLabelSet(0, 3, "c"),
				generateUnprefixedLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
		},

		"unprefixed basic": {
			// 3 check labels + 1 probe label in UNPREFIXED mode.
			// Verifies: execution metrics carry un-prefixed user labels;
			//           sm_check_info and log stream carry only un-prefixed forms.
			labelMode:       sm.LabelMode_LABEL_MODE_UNPREFIXED,
			maxMetricLabels: defMaxMetricLabels,
			maxLogLabels:    defMaxLogLabels,
			checkLabels:     generateLabels(0, 3, "c"),
			probeLabels:     generateLabels(10, 1, "p"),
			expectedMetricLabels: mergeMaps(
				baseExpectedMetricLabels,
				generateUnprefixedLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
			expectedInfoLabels: mergeMaps(
				baseExpectedMetricLabels,
				baseExpectedInfoLabelsSMRegion,
				generateUnprefixedLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
			expectedLogLabels: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				generateUnprefixedLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
			expectedLogEntries: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				generateUnprefixedLabelSet(10, 1, "p"),
				generateUnprefixedLabelSet(0, 3, "c"),
			),
		},

		"dual write log overflow": {
			// 10 check labels + 3 probe labels in DUAL_WRITE mode causes log stream
			// overflow. DUAL_WRITE ordering means all prefixed forms come first in the
			// stream labels, so they are preserved when the label count exceeds the cap;
			// the un-prefixed forms spill into StructuredMetadata instead.
			//
			// Label budget: 6 system + 26 user (13×2) = 32 total; cap at
			// defMaxLogLabels-1=14 stream slots before probe_success.
			// Stream labels: system labels (6) + first 8 prefixed user labels.
			// Overflow into StructuredMetadata: remaining prefixed (5) + all 13
			// un-prefixed user labels.
			labelMode:       sm.LabelMode_LABEL_MODE_DUAL_WRITE,
			maxMetricLabels: defMaxMetricLabels,
			maxLogLabels:    defMaxLogLabels,
			checkLabels:     generateLabels(0, 10, "c"),
			probeLabels:     generateLabels(10, 3, "p"),
			// Execution metrics: base + all un-prefixed user labels.
			expectedMetricLabels: mergeMaps(
				baseExpectedMetricLabels,
				generateUnprefixedLabelSet(10, 3, "p"),
				generateUnprefixedLabelSet(0, 10, "c"),
			),
			// sm_check_info: system sharedLabels + both forms of user labels from ConstLabels.
			expectedInfoLabels: mergeMaps(
				baseExpectedMetricLabels,
				baseExpectedInfoLabelsSMRegion,
				generateLabelSet(10, 3, "p"),
				generateUnprefixedLabelSet(10, 3, "p"),
				generateLabelSet(0, 10, "c"),
				generateUnprefixedLabelSet(0, 10, "c"),
			),
			// Stream labels: system(6) + 8 prefixed user labels (prefixed-first
			// ordering means label_l10,label_l11,label_l12 + label_l0..label_l4
			// fill the 8 available slots before the cap).
			expectedLogLabels: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				generateLabelSet(10, 3, "p"), // label_l10, label_l11, label_l12
				generateLabelSet(0, 5, "c"),  // label_l0 .. label_l4
			),
			// Log entry body includes all user labels in both forms (log entry is not
			// truncated, only stream labels are capped). The entries in
			// expectedLogEntries \ expectedLogLabels become StructuredMetadata overflow
			// and are validated by assertStructuredMetadata.
			expectedLogEntries: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				generateLabelSet(10, 3, "p"),
				generateLabelSet(0, 10, "c"),
				generateUnprefixedLabelSet(10, 3, "p"),
				generateUnprefixedLabelSet(0, 10, "c"),
			),
		},

		// In UNPREFIXED mode a user label named "job" collides with the reserved
		// system "job" label. buildUserLabels drops it (sm.IsSystemLabel), so the
		// system value is retained on every surface — execution metrics,
		// sm_check_info, and the log stream — rather than being overwritten. The
		// non-reserved "env" label is emitted normally.
		"unprefixed reserved label dropped": {
			labelMode:       sm.LabelMode_LABEL_MODE_UNPREFIXED,
			maxMetricLabels: defMaxMetricLabels,
			maxLogLabels:    defMaxLogLabels,
			checkLabels: []sm.Label{
				{Name: "job", Value: "user-job"},
				{Name: "env", Value: "prod"},
			},
			// "job" keeps its system value (from base); only "env" is added.
			expectedMetricLabels: mergeMaps(
				baseExpectedMetricLabels,
				map[string]string{"env": "prod"},
			),
			expectedInfoLabels: mergeMaps(
				baseExpectedMetricLabels,
				baseExpectedInfoLabelsSMRegion,
				map[string]string{"env": "prod"},
			),
			expectedLogLabels: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				map[string]string{"env": "prod"},
			),
			expectedLogEntries: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				map[string]string{"env": "prod"},
			),
		},

		// Renaming the agent's own region label to sm_region frees "region" for
		// tenant use: a user label named region must reach every surface intact,
		// alongside the agent's sm_region.
		"unprefixed user region label freed": {
			labelMode:       sm.LabelMode_LABEL_MODE_UNPREFIXED,
			maxMetricLabels: defMaxMetricLabels,
			maxLogLabels:    defMaxLogLabels,
			checkLabels: []sm.Label{
				{Name: "region", Value: "user-region"},
			},
			expectedMetricLabels: mergeMaps(
				baseExpectedMetricLabels,
				map[string]string{"region": "user-region"},
			),
			expectedInfoLabels: mergeMaps(
				baseExpectedMetricLabels,
				baseExpectedInfoLabelsSMRegion,
				map[string]string{"region": "user-region"},
			),
			expectedLogLabels: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				map[string]string{"region": "user-region"},
			),
			expectedLogEntries: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				map[string]string{"region": "user-region"},
			),
		},

		// Same as above but in DUAL_WRITE: the riskiest mode, where the user's
		// region (as both region and label_region) coexists with the agent's
		// sm_region — three distinct keys, no duplicates, on every surface.
		"dual write user region label freed": {
			labelMode:       sm.LabelMode_LABEL_MODE_DUAL_WRITE,
			maxMetricLabels: defMaxMetricLabels,
			maxLogLabels:    defMaxLogLabels,
			checkLabels: []sm.Label{
				{Name: "region", Value: "user-region"},
			},
			// Execution metrics carry only the un-prefixed user form.
			expectedMetricLabels: mergeMaps(
				baseExpectedMetricLabels,
				map[string]string{"region": "user-region"},
			),
			expectedInfoLabels: mergeMaps(
				baseExpectedMetricLabels,
				baseExpectedInfoLabelsSMRegion,
				map[string]string{"label_region": "user-region", "region": "user-region"},
			),
			expectedLogLabels: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				map[string]string{"label_region": "user-region", "region": "user-region"},
			),
			expectedLogEntries: mergeMaps(
				baseExpectedLogLabelsSMRegion,
				map[string]string{"label_region": "user-region", "region": "user-region"},
			),
		},

		// This tests simulates the default label mode falling back to UNPREFIXED, but note
		// the scraper would actually abort if there was no tenant in cache due to
		// the label limits being unavailable.
		"label mode fetch error degrades to unprefixed": {
			labelMode:            sm.LabelMode_LABEL_MODE_UNPREFIXED,
			labelModeErr:         errors.New("simulated tenant label-mode fetch failure"),
			maxMetricLabels:      defMaxMetricLabels,
			maxLogLabels:         defMaxLogLabels,
			checkLabels:          generateLabels(1, 3, "c"),
			expectedMetricLabels: mergeMaps(baseExpectedMetricLabels, generateUnprefixedLabelSet(1, 3, "c")),
			expectedInfoLabels:   mergeMaps(baseExpectedMetricLabels, baseExpectedInfoLabelsSMRegion, generateUnprefixedLabelSet(1, 3, "c")),
			expectedLogLabels:    mergeMaps(baseExpectedLogLabelsSMRegion, generateUnprefixedLabelSet(1, 3, "c")),
			expectedLogEntries:   mergeMaps(baseExpectedLogLabelsSMRegion, generateUnprefixedLabelSet(1, 3, "c")),
		},
	}

	getMetricName := func(t *testing.T, ts prompb.TimeSeries) string {
		for _, l := range ts.GetLabels() {
			if l.GetName() != prom.MetricNameLabel {
				continue
			}

			return l.GetValue()
		}

		require.Fail(t, "metric name not found")

		return ""
	}

	validateMetrics := func(t *testing.T, ts prompb.TimeSeries, tc testcase) {
		require.NotNil(t, ts)

		metricName := getMetricName(t, ts)

		// Ensure no label name appears more than once. Duplicate label keys are
		// rejected by Mimir and indicate a bug in label assembly (e.g. user labels
		// reaching a metric via both ConstLabels and sharedLabels).
		seenLabelNames := make(map[string]bool, len(ts.GetLabels()))
		for _, l := range ts.GetLabels() {
			require.Falsef(t, seenLabelNames[l.GetName()],
				"duplicate label %q in metric %s", l.GetName(), metricName)
			seenLabelNames[l.GetName()] = true
		}

		actualLabels := make(map[string]string)
		actualLabelsCount := 0
		actualInfoLabels := make(map[string]string)
		actualInfoLabelsCount := 0

		// Verify that all the expected metric labels are present

		for _, l := range ts.GetLabels() {
			switch {
			case l.GetName() == prom.MetricNameLabel:
				// ignore

			case l.GetName() == labels.BucketLabel:
				// ignore

			case metricName == CheckInfoMetricName:
				expectedValue, isExpected := tc.expectedInfoLabels[l.GetName()]
				require.Truef(t, isExpected, "metric=%s label=%s value=%s", metricName, l.GetName(), l.GetValue())
				require.Equal(t, expectedValue, l.GetValue())
				actualInfoLabels[l.GetName()] = l.GetValue()
				actualInfoLabelsCount++

			default:
				expectedValue, isExpected := tc.expectedMetricLabels[l.GetName()]
				require.Truef(t, isExpected, "unexpected label: metric=%s label=%s value=%s", metricName, l.GetName(), l.GetValue())
				require.Equal(t, expectedValue, l.GetValue())
				actualLabels[l.GetName()] = l.GetValue()
				actualLabelsCount++
			}
		}

		if metricName == CheckInfoMetricName {
			require.Equal(t, tc.expectedInfoLabels, actualInfoLabels)
			require.Equal(t, len(tc.expectedInfoLabels), actualInfoLabelsCount)
		} else {
			require.Equal(t, tc.expectedMetricLabels, actualLabels)
			require.Equal(t, len(tc.expectedMetricLabels), actualLabelsCount)
		}

		for _, sample := range ts.GetSamples() {
			// This encodes the assumption that there's a single timestamp included in the
			// resulting metrics.
			require.Equal(t, sampleTsMs, sample.Timestamp)
		}
	}

	// assertStructuredMetadata verifies that each log entry's StructuredMetadata
	// contains a valid execution_id UUID plus any overflow labels (those present in
	// expectedLogEntries but absent from expectedLogLabels — they fit in the log
	// entry body but were truncated from the stream label set).
	assertStructuredMetadata := func(t *testing.T, entry logproto.Entry, tc testcase) {
		t.Helper()

		executionIDFound := false
		overflowFound := 0

		for _, meta := range entry.StructuredMetadata {
			if meta.Name == "execution_id" {
				executionIDFound = true
				_, err := uuid.Parse(meta.Value)
				require.NoError(t, err, "execution_id should be a valid UUID")
			} else {
				expectedVal, inEntries := tc.expectedLogEntries[meta.Name]
				_, inLabels := tc.expectedLogLabels[meta.Name]
				require.Truef(t, inEntries && !inLabels,
					"unexpected structured metadata entry: %s=%s", meta.Name, meta.Value)
				require.Equal(t, expectedVal, meta.Value, "overflow label value mismatch: %s", meta.Name)

				overflowFound++
			}
		}

		require.True(t, executionIDFound, "execution_id not found in structured metadata")

		expectedOverflow := 0

		for name := range tc.expectedLogEntries {
			if _, inLabels := tc.expectedLogLabels[name]; !inLabels {
				expectedOverflow++
			}
		}

		require.Equal(t, expectedOverflow, overflowFound, "overflow label count in structured metadata")
	}

	validateStreams := func(t *testing.T, s Scraper, stream logproto.Stream, tc testcase) {
		sLabels, err := metricParser.ParseMetric(stream.Labels)
		require.NoError(t, err)

		// Verify that all the expected log labels are present as labels in the stream labels.
		found := 0

		sLabels.Range(func(label labels.Label) {
			expected, ok := tc.expectedLogLabels[label.Name]
			require.Truef(t, ok, "key=%s value=%s labels=%s", label.Name, label.Value, stream.Labels)
			require.Equalf(t, expected, label.Value, "key=%s", label.Name)

			found++
		})

		require.Equal(t, len(tc.expectedLogLabels), found, stream.Labels)

		// Verify that all the expected log labels are present as part of the actual log entry.
		for _, entry := range stream.Entries {
			dec := logfmt.NewDecoder(strings.NewReader(entry.Line))
			for dec.ScanRecord() {
				labelsFound := 1 // probe_success is NOT included in the log entry

				for dec.ScanKeyval() {
					key := string(dec.Key())
					val := string(dec.Value())

					switch key {
					case "level", "msg", "timeout_seconds", "duration_seconds", "walltime_seconds":
					case "target":
						require.Equal(t, s.target, val)
					case "type":
						require.Equal(t, s.prober.Name(), val)
					default:
						expected, found := tc.expectedLogEntries[key]
						require.Truef(t, found, "key=%s value=%s", key, val)
						require.Equalf(t, expected, val, "key=%s", key)

						labelsFound++
					}
				}

				require.Equal(t, len(tc.expectedLogEntries), labelsFound)
			}

			require.NoError(t, dec.Err())

			assertStructuredMetadata(t, entry, tc)
		}
	}

	for name, tc := range testcases {
		t.Run(name, func(t *testing.T) {
			s := Scraper{
				checkName: checkName,
				target:    "test target",
				logger:    testhelper.Logger(t),
				prober:    testProber{},
				labelsLimiter: testLabelsLimiter{
					maxMetricLabels: tc.maxMetricLabels,
					maxLogLabels:    tc.maxLogLabels,
				},
				labellingMode: testLabellingMode{mode: tc.labelMode, err: tc.labelModeErr},
				summaries:     make(map[uint64]prometheus.Summary),
				histograms:    make(map[uint64]prometheus.Histogram),
				check: model.Check{
					Check: sm.Check{
						Id:               1,
						TenantId:         2,
						Frequency:        frequency,
						Timeout:          frequency,
						Enabled:          true,
						Target:           checkTarget,
						Job:              job,
						BasicMetricsOnly: true,
						Created:          modifiedTs,
						Modified:         modifiedTs,
						Labels:           tc.checkLabels,
						// [Check.Type] panics if all settings are nil. To work around that, we add an empty, non nil
						// HTTP settings section.
						Settings: sm.CheckSettings{
							Http: &sm.HttpSettings{},
						},
					},
				},
				probe: sm.Probe{
					Id:        100,
					TenantId:  200,
					Name:      probeName,
					Latitude:  probeLatitude,
					Longitude: probeLongitde,
					Region:    region,
					Labels:    tc.probeLabels,
				},
			}

			data, duration, err := s.collectData(context.Background(), time.Unix(sampleTsMs/1000, 0))
			require.NoError(t, err)
			require.NotNil(t, data)
			require.NotZero(t, duration)

			for _, ts := range data.Metrics() {
				validateMetrics(t, ts, tc)
			}

			for _, stream := range data.Streams() {
				validateStreams(t, s, stream, tc)
			}
		})
	}
}

func findPromPbLabel(t *testing.T, labels []prompb.Label, name string) string {
	idx := slices.IndexFunc(labels, func(v prompb.Label) bool { return v.GetName() == name })
	require.GreaterOrEqualf(t, idx, int(0), "label %q must be present", name)

	return labels[idx].GetValue()
}

// histogramProber emits a single histogram (test_duration_seconds) with several
// buckets, so the gathered metric families contain multiple *_bucket series that
// differ only by their intrinsic `le` label. This is the shape that makes a
// reserved-name collision destructive: unlike sm_check_info's single-valued
// identity labels, `le` is per-series.
type histogramProber struct{}

func (histogramProber) Name() string { return "histogram prober" }

func (histogramProber) Probe(_ context.Context, _ string, registry *prometheus.Registry, _ logger.Logger, _ string) (bool, float64) {
	h := prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "test_duration_seconds",
		Buckets: []float64{0.1, 0.5, 1},
	})
	registry.MustRegister(h)

	// Observations spread across buckets so every bucket series is distinct.
	h.Observe(0.05)
	h.Observe(0.3)
	h.Observe(0.8)

	return true, 1
}

// TestScraperCollectDataReservedLabelCollision pins the agent's behaviour when
// a tenant carries a user label whose name collides with a *per-series
// intrinsic* reserved label (`le`) while in UNPREFIXED mode.
//
// Per checks_extra.go, the API is supposed to make this unreachable: reserved
// names are rejected at check create/update and at the LabelMode transition.
// This test is the agent-side backstop for the case where such a label reaches
// the scraper anyway. It asserts the safe contract: the histogram's real
// bucket boundaries survive as distinct series ({0.1, 0.5, 1, +Inf}) and are
// NOT collapsed onto the user's value.
//
// TODO(mem): Remove this note. This test FAILS against the current
// implementation. customMetricLabels seeds the user `le` into
// executionMetricLabels, appendDtoToTimeseries appends the real bucket `le`
// last, and makeTimeseries de-dupes keeping the first occurrence, so every
// bucket collapses to le=user-value. It will pass once per-series
// reserved-name filtering is added to customMetricLabels. The same collapse
// also corrupts the agent's own probe_all_duration_seconds histogram, so this
// is not limited to custom prober metrics.
func TestScraperCollectDataReservedLabelCollision(t *testing.T) {
	s := Scraper{
		checkName:     "check name",
		target:        "test target",
		logger:        testhelper.Logger(t),
		prober:        histogramProber{},
		labelsLimiter: testLabelsLimiter{maxMetricLabels: 22, maxLogLabels: 15},
		labellingMode: testLabellingMode{mode: sm.LabelMode_LABEL_MODE_UNPREFIXED},
		summaries:     make(map[uint64]prometheus.Summary),
		histograms:    make(map[uint64]prometheus.Histogram),
		check: model.Check{
			Check: sm.Check{
				Id:               1,
				TenantId:         2,
				Frequency:        2000,
				Timeout:          2000,
				Enabled:          true,
				Target:           "target name",
				Job:              "job name",
				BasicMetricsOnly: false,
				Created:          42,
				Modified:         42,
				// User label collides with the reserved, per-series `le`.
				Labels:   []sm.Label{{Name: "le", Value: "user-value"}},
				Settings: sm.CheckSettings{Http: &sm.HttpSettings{}},
			},
		},
		probe: sm.Probe{
			Id:       100,
			TenantId: 200,
			Name:     "probe name",
			Region:   "REGION",
		},
	}

	data, _, err := s.collectData(context.Background(), time.Unix(3141, 0))
	require.NoError(t, err)
	require.NotNil(t, data)

	// Collect the `le` value from every test_duration_seconds_bucket
	// series. Scoped to this histogram so the count is deterministic; note
	// that the agent's own probe_all_duration_seconds_bucket series
	// collapse identically.
	bucketLEs := make(map[string]int)
	bucketSeries := 0

	for _, ts := range data.Metrics() {
		name := findPromPbLabel(t, ts.GetLabels(), prom.MetricNameLabel)
		if name != "test_duration_seconds_bucket" {
			continue
		}

		le := findPromPbLabel(t, ts.GetLabels(), labels.BucketLabel)

		bucketSeries++
		bucketLEs[le]++
	}

	// A 3-boundary histogram yields 4 bucket series: 0.1, 0.5, 1, +Inf.
	require.Equal(t, 4, bucketSeries, "expected one series per bucket boundary")

	// The intended contract: real bucket boundaries survive, distinct and
	// un-clobbered by the user's `le=user-value`.
	require.Equal(t,
		map[string]int{"0.1": 1, "0.5": 1, "1": 1, "+Inf": 1},
		bucketLEs,
		"histogram bucket `le` labels were overwritten/collapsed by the user label")
}

// phaseProber emits probe_http_duration_seconds as a gauge with a `phase`
// label, mirroring the real HTTP prober's per-phase timing breakdown. `phase`
// is a reserved, per-series label (checks_extra.go) but, unlike `le` above, it
// is a name a tenant might plausibly define themselves, which makes this the
// more realistic collision vehicle.
type phaseProber struct{}

func (phaseProber) Name() string { return "phase prober" }

func (phaseProber) Probe(_ context.Context, _ string, registry *prometheus.Registry, _ logger.Logger, _ string) (bool, float64) {
	g := prometheus.NewGaugeVec(prometheus.GaugeOpts{
		Name: "probe_http_duration_seconds",
	}, []string{"phase"})
	registry.MustRegister(g)

	// One series per HTTP phase, each with a distinct value.
	for i, phase := range []string{"resolve", "connect", "tls", "processing", "transfer"} {
		g.WithLabelValues(phase).Set(float64(i+1) * 0.01)
	}

	return true, 1
}

// TestScraperCollectDataReservedPhaseCollision is the realistic sibling of
// TestScraperCollectDataReservedLabelCollision: it collides a user label with
// the reserved per-series label `phase` on the multi-series
// probe_http_duration_seconds gauge instead of `le` on a histogram.
//
// It feeds a check label phase=user-value in UNPREFIXED mode and asserts the
// five per-phase series retain their distinct phase values.
//
// TODO(mem): remove this note. Like the `le` test above it FAILS against the
// current implementation: customMetricLabels seeds the user `phase` into
// executionMetricLabels ahead of each metric's own `phase` label
// (appendDtoToTimeseries appends the metric's labels after sharedLabels), and
// makeTimeseries' keep-first dedup drops the real phase, so all five series
// collapse to phase=user-value. It will pass once per-series reserved-name
// filtering is added to customMetricLabels.
func TestScraperCollectDataReservedPhaseCollision(t *testing.T) {
	s := Scraper{
		checkName:     "check name",
		target:        "test target",
		logger:        testhelper.Logger(t),
		prober:        phaseProber{},
		labelsLimiter: testLabelsLimiter{maxMetricLabels: 22, maxLogLabels: 15},
		labellingMode: testLabellingMode{mode: sm.LabelMode_LABEL_MODE_UNPREFIXED},
		summaries:     make(map[uint64]prometheus.Summary),
		histograms:    make(map[uint64]prometheus.Histogram),
		check: model.Check{
			Check: sm.Check{
				Id:               1,
				TenantId:         2,
				Frequency:        2000,
				Timeout:          2000,
				Enabled:          true,
				Target:           "target name",
				Job:              "job name",
				BasicMetricsOnly: false,
				Created:          42,
				Modified:         42,
				// User label collides with the reserved, per-series `phase`.
				Labels:   []sm.Label{{Name: "phase", Value: "user-value"}},
				Settings: sm.CheckSettings{Http: &sm.HttpSettings{}},
			},
		},
		probe: sm.Probe{
			Id:       100,
			TenantId: 200,
			Name:     "probe name",
			Region:   "REGION",
		},
	}

	data, _, err := s.collectData(context.Background(), time.Unix(3141, 0))
	require.NoError(t, err)
	require.NotNil(t, data)

	// Collect the `phase` value from every probe_http_duration_seconds series.
	phases := make(map[string]int)
	phaseSeries := 0

	for _, ts := range data.Metrics() {
		name := findPromPbLabel(t, ts.GetLabels(), prom.MetricNameLabel)
		if name != "probe_http_duration_seconds" {
			continue
		}

		phase := findPromPbLabel(t, ts.GetLabels(), "phase")

		phaseSeries++
		phases[phase]++
	}

	// Five phases emitted, so five distinct series.
	require.Equal(t, 5, phaseSeries, "expected one series per HTTP phase")

	// The intended contract: each phase survives as its own series, un-clobbered
	// by the user's phase=user-value.
	require.Equal(t, map[string]int{
		"resolve": 1, "connect": 1, "tls": 1, "processing": 1, "transfer": 1,
	}, phases, "per-phase series were overwritten/collapsed by the user label")
}

// these are generated using
// go run $(go env GOROOT)/src/crypto/tls/generate_cert.go --rsa-bits 1024 --host 127.0.0.1,::1,example.com --ca --start-date "Jan 1 00:00:00 1970" --duration=1000000h

var localhostCert = []byte(`-----BEGIN CERTIFICATE-----
MIICFDCCAX2gAwIBAgIRAIoIa8inD4pBali2dBpc6+wwDQYJKoZIhvcNAQELBQAw
EjEQMA4GA1UEChMHQWNtZSBDbzAgFw03MDAxMDEwMDAwMDBaGA8yMDg0MDEyOTE2
MDAwMFowEjEQMA4GA1UEChMHQWNtZSBDbzCBnzANBgkqhkiG9w0BAQEFAAOBjQAw
gYkCgYEA5ii8ieKKayAtIX7BbKWB+zRDnJsFg02+cHwRy7nmX1t9MdObwNAMFIET
zvKz4Yctgi3WoyMZfp6pYugHHye829DbUsm+GI6Ca3zSzADfz+zL/nIWiJje0NB1
DXPASi03nTNk06O5RGlWeLqGKfyEY3xjcUrE7rKlNtLu8oK4Jm0CAwEAAaNoMGYw
DgYDVR0PAQH/BAQDAgKkMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA8GA1UdEwEB/wQF
MAMBAf8wLgYDVR0RBCcwJYILZXhhbXBsZS5jb22HBH8AAAGHEAAAAAAAAAAAAAAA
AAAAAAEwDQYJKoZIhvcNAQELBQADgYEAyW2Yc/vnlO9PYcXw9ZRYt51hTCKCooGv
RwR1OwL9Kdr/eY+wGpO+YsXxdSmRcmg467TA5d1YpssyVceKRla22NgDoZ2psFTe
LqKfcZUN+jvtQMx4LnsRZcz2i2U35Biq4h0i3SOgROIOEjQJJ6I8wMw9jD5kS86Q
IYZvskPli5s=
-----END CERTIFICATE-----`)

var localhostKey = []byte(testingKey(`-----BEGIN RSA TESTING KEY-----
MIICdgIBADANBgkqhkiG9w0BAQEFAASCAmAwggJcAgEAAoGBAOYovIniimsgLSF+
wWylgfs0Q5ybBYNNvnB8Ecu55l9bfTHTm8DQDBSBE87ys+GHLYIt1qMjGX6eqWLo
Bx8nvNvQ21LJvhiOgmt80swA38/sy/5yFoiY3tDQdQ1zwEotN50zZNOjuURpVni6
hin8hGN8Y3FKxO6ypTbS7vKCuCZtAgMBAAECgYBzp1y2XOP5WL3U6wD/O1vJg0XG
WA+5H0Pm+jFnEg81M6ABfbfyd5jaZNIzV7oURf0UQTxt1aFmAwxS6w1JForLZn3g
PA3UVkDEZTl7C7h6kIY4PVzcki32V2YZ73e1zSCfvxIvbJ7SS697ua1sefIP5Gci
HNSRzanUyOKCZ1Or8QJBAPB1A/66Baydh+2nXddaad9d5Ifjvklk60tozLYpx7Il
R6NgU49Pa67MndGGykmbs094TYY8GswU2WoJVPUEYY8CQQD1CVB8++yK2jP0LkH/
s+3dag2S2IId5FwSbjiM8YtOFx2wN2HleX7Yr6ujPnKT+zy75eV0chPIP++QNKmy
eIJDAkAJm/OD63Usl8MF2UljwMY4We03DP/euPy6L772jKbhVKIPQls0f+0CuESa
SfOti15YD6uxcJd1jmO93A+cFwe7AkEAgLFnqHzXewWnC7PPzfA+GW+9uUYk8HYj
NTrWUI/7zgOuAALWU6M/z6ZTyuTdYIMvHrBblpDjeuS5eU9vYOCR6QJAWT7VShIT
sc8HRJOxUQWOI5hVUfLxVSosFDW7G5/75WlyeYwZOqOD54uNzavr8gpoTlvllUuP
GcRWUmpDEz0t6w==
-----END RSA TESTING KEY-----`))

func testingKey(s string) string { return strings.ReplaceAll(s, "TESTING KEY", "PRIVATE KEY") }

// testLogger impleents the Logger interface to pass it to the prober.
type testLogger struct {
	w io.Writer
}

func (l *testLogger) Log(kv ...any) error {
	var buf strings.Builder

	for i, v := range kv {
		if i >= 2 && i%2 == 0 {
			buf.WriteString(", ")
		}

		switch v := v.(type) {
		case string:
			buf.WriteString(v)

			if i%2 == 0 {
				buf.WriteRune(':')
			}

		case error:
			buf.WriteString(v.Error())

		case interface{ String() string }:
			buf.WriteString(v.String())

		default:
			fmt.Fprintf(&buf, "%#v", v)
		}
	}

	fmt.Fprintf(l.w, "%s\n", buf.String())

	return nil
}

func mergeMaps(mmaps ...map[string]string) map[string]string {
	out := make(map[string]string)
	for _, m := range mmaps {
		maps.Copy(out, m)
	}

	return out
}

func TestTruncateLabelValue(t *testing.T) {
	testcases := map[string]struct {
		length         int
		expectedLength int
	}{
		"zero": {
			length:         0,
			expectedLength: 0,
		},
		"one": {
			length:         1,
			expectedLength: 1,
		},
		"max/2": {
			length:         maxLabelValueLength / 2,
			expectedLength: maxLabelValueLength / 2,
		},
		"max-1": {
			length:         maxLabelValueLength - 1,
			expectedLength: maxLabelValueLength - 1,
		},
		"max": {
			length:         maxLabelValueLength,
			expectedLength: maxLabelValueLength,
		},
		"max+1": {
			length:         maxLabelValueLength + 1,
			expectedLength: maxLabelValueLength,
		},
		"2*max": {
			length:         2 * maxLabelValueLength,
			expectedLength: maxLabelValueLength,
		},
	}

	for name, tc := range testcases {
		t.Run(name, func(t *testing.T) {
			input := strings.Repeat("a", tc.length)
			expected := strings.Repeat("a", tc.expectedLength)
			actual := truncateLabelValue(input)
			require.Equal(t, len(expected), len(actual))

			if tc.expectedLength < tc.length {
				require.Equal(t, expected[:len(expected)-3], actual[:len(actual)-3])
				require.Equal(t, "...", actual[len(actual)-3:])
			}
		})
	}
}

type testRunner struct {
	metrics []byte
	logs    []byte
}

var _ k6runner.Runner = &testRunner{}

func (r *testRunner) Run(ctx context.Context, script k6runner.Script, secretStore k6runner.SecretStore, _ string) (*k6runner.RunResponse, error) {
	return &k6runner.RunResponse{
		Metrics: r.metrics,
		Logs:    r.logs,
	}, nil
}

func (r *testRunner) WithLogger(logger *zerolog.Logger) k6runner.Runner {
	return r
}

func (*testRunner) Versions(_ context.Context) <-chan []string {
	return nil // Blocks forever if read.
}

type testCounter struct {
	count atomic.Int32
}

func (c *testCounter) Inc() {
	c.count.Add(1)
}

type testCounterVec struct {
	counters map[string]Incrementer
	t        *testing.T
}

func (c *testCounterVec) WithLabelValues(v ...string) Incrementer {
	require.Len(c.t, v, 1)

	if _, found := c.counters[v[0]]; !found {
		c.counters[v[0]] = &testCounter{}
	}

	return c.counters[v[0]]
}

type testProberB struct {
	wantedFailures int32
	execCount      int32
	failureCount   int32
}

func (p testProberB) Name() string {
	return "test prober"
}

func (p *testProberB) Probe(ctx context.Context, target string, registry *prometheus.Registry, logger logger.Logger, _ string) (bool, float64) {
	p.execCount++

	if p.failureCount < p.wantedFailures {
		p.failureCount++
		return false, 0
	}

	return true, 1
}

type testProbeFactory struct {
	builder func() prober.Prober
}

func (f testProbeFactory) New(ctx context.Context, logger zerolog.Logger, check model.Check) (prober.Prober, string, error) {
	return f.builder(), check.Target, nil
}

type testPublisher struct{}

func (testPublisher) Publish(pusher.Payload) {
}

type testTelemeter struct {
	execCount int32
	ee        []telemetry.Execution
}

func (t *testTelemeter) AddExecution(e telemetry.Execution) {
	t.execCount++
}

// TestScraperRun will set up a scraper in such a way that it runs 5 times, and fails 2 out of those 5.
//
// This checks that the probe gets run, and that the metrics are correctly collected.
func TestScraperRun(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
	t.Cleanup(cancel)

	var check model.Check

	err := check.FromSM(sm.Check{
		Id:        1,
		TenantId:  1000,
		Frequency: 100,
		Timeout:   1000,
		Enabled:   true,
		Target:    "127.0.0.1",
		Job:       "test",
		Settings: sm.CheckSettings{
			Ping: &sm.PingSettings{},
		},
	})
	require.NoError(t, err)

	var counter testCounter

	errCounter := testCounterVec{counters: make(map[string]Incrementer), t: t}

	testProber := &testProberB{wantedFailures: 2}
	testTelemeter := &testTelemeter{}

	metrics := NewMetrics(&counter, &errCounter)

	s, err := NewWithOpts(ctx, check, ScraperOpts{
		Metrics:      metrics,
		ProbeFactory: testProbeFactory{builder: func() prober.Prober { return testProber }},
		Logger:       zerolog.New(zerolog.NewTestWriter(t)),
		Publisher:    &testPublisher{},
		LabelsLimiter: testLabelsLimiter{
			maxMetricLabels: 20,
			maxLogLabels:    15,
		},
		LabellingMode: testLabellingMode{},
		Telemeter:     testTelemeter,
		CostAttributionLabels: testCalTenants{
			costAttributionLabels: []string{"testing", "you"},
		},
	})

	require.NoError(t, err)
	require.NotNil(t, s)

	s.Run(ctx)

	require.Equal(t, testProber.execCount, counter.count.Load())
	require.Len(t, errCounter.counters, 1)
	checkErrCounter, found := errCounter.counters["check"]
	require.True(t, found)
	require.Equal(t, testProber.failureCount, checkErrCounter.(*testCounter).count.Load())

	// Verify telemetry
	require.Equal(t, counter.count.Load(), testTelemeter.execCount)

	for _, e := range testTelemeter.ee {
		require.Equal(t, 1000, e.LocalTenantID)
		require.Equal(t, check.Class(), e.CheckClass)
		require.Greater(t, 0, e.Duration)
	}
}

func TestTickWithOffset(t *testing.T) {
	const (
		WORK    = 1
		IDLE    = 2
		CLEANUP = 3
	)

	testcases := map[string]struct {
		timeout  time.Duration
		period   time.Duration
		offset   time.Duration
		maxIdle  time.Duration
		minGap   time.Duration
		expected []int
	}{
		"An idle worker running between regular runs": {
			timeout: 1050 * time.Millisecond,
			period:  500 * time.Millisecond,
			offset:  1,
			maxIdle: 100 * time.Millisecond,
			minGap:  50 * time.Millisecond,
			expected: []int{
				WORK, // 0
				IDLE, // 100
				IDLE, // 200
				IDLE, // 300
				IDLE, // 400
				WORK, // 500
				IDLE, // 600
				IDLE, // 700
				IDLE, // 800
				IDLE, // 900
				WORK, // 1000
				CLEANUP,
			},
		},
		"An idle worker trying to run within gap duration of regular runs.": {
			timeout: 1050 * time.Millisecond,
			period:  500 * time.Millisecond,
			offset:  1,
			maxIdle: 100 * time.Millisecond,
			minGap:  150 * time.Millisecond,
			expected: []int{
				WORK, // 0
				IDLE, // 200, there's no idle at 100 because it's too close to the previous run
				IDLE, // 300
				WORK, // 500, there's no idle at 400 because it's too close to the next run
				IDLE, // 700, there's no idle at 600 because it's too close to the previous run
				IDLE, // 800
				WORK, // 1000, there's no idle at 900 because it's too close to the next run
				CLEANUP,
			},
		},
		"A zero offset and a scraper that has already been cancelled.": {
			timeout:  0, // this asks for immediate cancellation
			period:   500 * time.Millisecond,
			offset:   0,
			maxIdle:  100 * time.Millisecond,
			minGap:   150 * time.Millisecond,
			expected: nil,
		},
		"A zero offset and a short timeout.": {
			timeout: 25 * time.Millisecond,
			period:  50 * time.Millisecond,
			offset:  0,
			maxIdle: 100 * time.Millisecond,
			minGap:  150 * time.Millisecond,
			expected: []int{
				WORK, // 0
				CLEANUP,
			},
		},
		"A zero offset and a larger timeout.": {
			timeout: 200 * time.Millisecond,
			period:  75 * time.Millisecond,
			offset:  0,
			maxIdle: 100 * time.Millisecond,
			minGap:  150 * time.Millisecond,
			expected: []int{
				WORK, // 0
				WORK, // 75
				WORK, // 150
				CLEANUP,
			},
		},
	}

	for name, tc := range testcases {
		t.Run(name, func(t *testing.T) {
			ctx, cancel := testhelper.Context(context.Background(), t)
			t.Cleanup(cancel)

			// Append a particular marker to the results slice
			// depending on which function was called by
			// TestTickWithOffset. This allows us to test that the
			// correct function is being called in the correct
			// order.
			var results []int

			work := func(context.Context, time.Time) { results = append(results, WORK) }
			idle := func(context.Context, time.Time) { results = append(results, IDLE) }
			cleanup := func(context.Context, time.Time) { results = append(results, CLEANUP) }

			stop := make(chan struct{})

			if tc.timeout > 0 {
				go func() {
					time.Sleep(tc.timeout)
					close(stop)
				}()
			} else {
				// why not in the goroutine? Because we need to
				// make sure the channel is closed *before*
				// calling tickWithOffset
				close(stop)
			}

			tickWithOffset(ctx, stop, work, idle, cleanup, tc.period, tc.offset, tc.maxIdle, tc.minGap)

			require.Equal(t, tc.expected, results)
		})
	}
}

func TestGetCostAttributionLabels(t *testing.T) {
	t.Parallel()

	testCases := []struct {
		name           string
		tenantCALs     []string
		checkLabels    []sm.Label
		expectedResult []sm.CostAttributionLabel
	}{
		{
			name:           "no CALs for tenant returns empty slice",
			tenantCALs:     []string{},
			checkLabels:    []sm.Label{{Name: "env", Value: "prod"}},
			expectedResult: []sm.CostAttributionLabel{},
		},
		{
			name:        "no matching labels returns all CALs with nil terminator",
			tenantCALs:  []string{"env", "team"},
			checkLabels: []sm.Label{{Name: "unrelated", Value: "value"}},
			expectedResult: []sm.CostAttributionLabel{
				{Name: "env", Value: telemetry.CalNilStringTerminator},
				{Name: "team", Value: telemetry.CalNilStringTerminator},
			},
		},
		{
			name:        "some matching labels returns mix of values and nil terminators",
			tenantCALs:  []string{"env", "team"},
			checkLabels: []sm.Label{{Name: "env", Value: "prod"}},
			expectedResult: []sm.CostAttributionLabel{
				{Name: "env", Value: "prod"},
				{Name: "team", Value: telemetry.CalNilStringTerminator},
			},
		},
		{
			name:       "all matching labels returns all values",
			tenantCALs: []string{"env", "team"},
			checkLabels: []sm.Label{
				{Name: "env", Value: "prod"},
				{Name: "team", Value: "platform"},
			},
			expectedResult: []sm.CostAttributionLabel{
				{Name: "env", Value: "prod"},
				{Name: "team", Value: "platform"},
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			calsProvider := &mockTenantCals{cals: tc.tenantCALs}

			s := &Scraper{
				cals:  calsProvider,
				check: model.Check{Check: sm.Check{Labels: tc.checkLabels}},
			}

			result, err := s.getCostAttributionLabels(context.Background(), 1)
			require.NoError(t, err)
			require.ElementsMatch(t, tc.expectedResult, result)
		})
	}
}

type mockTenantCals struct {
	cals []string
}

func (m *mockTenantCals) CostAttributionLabels(ctx context.Context, tenantID model.GlobalID) ([]string, error) {
	return m.cals, nil
}

func TestExtractLogsK6TimeOverridesDefaultTimestamp(t *testing.T) {
	// Test that k6's "time" field overrides the default "ts" timestamp from the logger
	testCases := []struct {
		name         string
		logs         string
		sharedLabels []labelPair
		expected     func(t *testing.T, streams Streams)
	}{
		{
			name: "k6 time field overrides default ts",
			logs: `ts=2023-06-01T20:00:00Z time="2023-06-01T13:40:26-06:00" level=info msg="k6 message"`,
			sharedLabels: []labelPair{
				{name: "probe", value: "test-probe"},
			},
			expected: func(t *testing.T, streams Streams) {
				require.Len(t, streams, 1)
				require.Len(t, streams[0].Entries, 1)
				entry := streams[0].Entries[0]

				expectedTime, _ := time.Parse(time.RFC3339, "2023-06-01T13:40:26-06:00")
				require.Equal(t, expectedTime, entry.Timestamp, "should use k6 time field, not default ts")
				require.Equal(t, entry.Line, "level=info msg=\"k6 message\"\n")
			},
		},
		{
			name: "k6 time field before ts (order independence)",
			// Defensive case: time= appears before ts= in the log line. The guard on the ts= case
			// (if !t.IsZero() { continue }) ensures ts= does not overwrite a time= already seen.
			logs: `time="2023-06-01T13:40:26-06:00" ts=2023-06-01T20:00:00Z level=info msg="k6 message"`,
			sharedLabels: []labelPair{
				{name: "probe", value: "test-probe"},
			},
			expected: func(t *testing.T, streams Streams) {
				require.Len(t, streams, 1)
				require.Len(t, streams[0].Entries, 1)
				entry := streams[0].Entries[0]

				expectedTime, _ := time.Parse(time.RFC3339, "2023-06-01T13:40:26-06:00")
				require.Equal(t, expectedTime, entry.Timestamp, "should use k6 time field even when it precedes ts")
				require.Equal(t, entry.Line, "level=info msg=\"k6 message\"\n")
			},
		},
		{
			name: "BBE adapter time field with nanosecond precision is used as timestamp",
			logs: `ts=2023-06-01T20:00:00.500000000Z level=INFO msg="Making HTTP request" time=2023-06-01T20:00:00.600000000Z url=https://example.com host=example.com`,
			sharedLabels: []labelPair{
				{name: "probe", value: "test-probe"},
			},
			expected: func(t *testing.T, streams Streams) {
				require.Len(t, streams, 1)
				require.Len(t, streams[0].Entries, 1)
				entry := streams[0].Entries[0]

				// time= (600ms) should win over ts= (500ms), and must preserve nanosecond precision.
				expectedTime, _ := time.Parse(time.RFC3339Nano, "2023-06-01T20:00:00.600000000Z")
				require.Equal(t, expectedTime, entry.Timestamp, "should use nanosecond-precision time= field")
				// url= and host= should be in the body; time= should not.
				require.Contains(t, entry.Line, "url=https://example.com")
				require.Contains(t, entry.Line, "host=example.com")
				require.NotContains(t, entry.Line, "time=")
			},
		},
		{
			name: "ts field only (no k6 time)",
			logs: `ts=2023-06-01T20:00:00Z level=info msg="normal message"`,
			sharedLabels: []labelPair{
				{name: "probe", value: "test-probe"},
			},
			expected: func(t *testing.T, streams Streams) {
				require.Len(t, streams, 1)
				require.Len(t, streams[0].Entries, 1)
				entry := streams[0].Entries[0]

				expectedTime, _ := time.Parse(time.RFC3339Nano, "2023-06-01T20:00:00Z")
				require.Equal(t, expectedTime, entry.Timestamp)
				require.Equal(t, entry.Line, "level=info msg=\"normal message\"\n")
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.name, func(t *testing.T) {
			t.Parallel()

			s := Scraper{
				logger: testhelper.Logger(t),
			}

			streams := s.extractLogs(time.Now(), []byte(tc.logs), tc.sharedLabels, nil)
			tc.expected(t, streams)
		})
	}
}

func TestHTTPCheckLogTimestampsAreNonDecreasing(t *testing.T) {
	ctx := context.Background()

	p, check, cleanup := setupHTTPProbe(ctx, t)
	defer cleanup()

	s := Scraper{
		checkName: check.Settings.Http.String(),
		target:    check.Target,
		logger:    testhelper.Logger(t),
		prober:    p,
		labelsLimiter: testLabelsLimiter{
			maxMetricLabels: 20,
			maxLogLabels:    15,
		},
		labellingMode: testLabellingMode{},
		summaries:     make(map[uint64]prometheus.Summary),
		histograms:    make(map[uint64]prometheus.Histogram),
		check:         check,
		probe: sm.Probe{
			Name:   "test-probe",
			Region: "test-region",
		},
	}

	// A scheduled scrape may carry an older ticker value for metric stamping,
	// but its wrapper logs must retain their original wall-clock behavior. The
	// unscheduled collector has separate coverage for logical timestamps.
	wallStart := time.Now()
	data, _, err := s.collectData(ctx, time.Date(2020, 3, 1, 12, 0, 0, 0, time.UTC))
	require.NoError(t, err)
	require.NotNil(t, data)

	streams := data.Streams()
	require.NotEmpty(t, streams, "expected at least one log stream")

	for _, stream := range streams {
		entries := stream.Entries
		require.NotEmpty(t, entries, "expected log entries in stream")

		// All 7 expected messages must be present.
		msgs := make([]string, 0, len(entries))
		for _, e := range entries {
			if strings.Contains(e.Line, `msg="Beginning check"`) || strings.Contains(e.Line, `msg="Check succeeded"`) {
				require.False(t, e.Timestamp.Before(wallStart), "scheduled wrapper log used the logical metric timestamp: %s", e.Timestamp)
			}

			dec := logfmt.NewDecoder(strings.NewReader(e.Line))
			for dec.ScanRecord() {
				for dec.ScanKeyval() {
					if string(dec.Key()) == "msg" {
						msgs = append(msgs, string(dec.Value()))
					}
				}
			}
		}

		expectedMsgs := []string{
			"Beginning check",
			"Resolving target address",
			"Resolved target address",
			"Making HTTP request",
			"Received HTTP response",
			"Response timings for roundtrip",
			"Check succeeded",
		}
		for _, expected := range expectedMsgs {
			require.Contains(t, msgs, expected, "missing log entry: %q", expected)
		}

		// Entry timestamps must be non-decreasing (Loki stream ordering requirement).
		for i := 1; i < len(entries); i++ {
			require.False(t,
				entries[i].Timestamp.Before(entries[i-1].Timestamp),
				"entry[%d] timestamp %v is before entry[%d] timestamp %v (out-of-order)\nentry[%d] msg: %s\nentry[%d] msg: %s",
				i, entries[i].Timestamp,
				i-1, entries[i-1].Timestamp,
				i-1, entries[i-1].Line,
				i, entries[i].Line,
			)
		}
	}
}