github.com/grafana/pyroscope@v1.18.0/pkg/model/pprofsplit/pprof_split_by_test.go

package pprofsplit

import (
	"fmt"
	"strings"
	"testing"

	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/model/relabel"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1"
	typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1"
)

type testSample struct {
	labels string // "foo=bar,baz=qux"
	value  int64
}

type expectedSeries struct {
	labels  string       // "foo=bar,baz=qux"
	samples []testSample // samples with their labels
}

func Test_VisitSampleSeriesBy(t *testing.T) {
	// Test cases are mostly generated by AI.
	// Some very specific cases were added manually.
	testCases := []struct {
		description  string
		seriesLabels string            // Series-level labels. Label order matters.
		samples      []testSample      // Input samples. Label order does not matter.
		splitBy      []string          // Labels to split by.
		rules        []*relabel.Config // Relabel rules to apply.
		expected     []expectedSeries
	}{
		{
			description:  "split profile by group by labels",
			seriesLabels: "__name__=profile,foo=bar",
			samples: []testSample{
				{labels: "service_name=web,endpoint=/users", value: 100},
				{labels: "service_name=api,endpoint=/users", value: 300},
			},
			splitBy: []string{"service_name"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,endpoint=/users,foo=bar,service_name=web",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,endpoint=/users,foo=bar,service_name=api",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
		{
			description:  "group by labels are not overridden",
			seriesLabels: "__name__=profile,foo=bar,service_name=app",
			samples: []testSample{
				{labels: "service_name=web,endpoint=/users", value: 100},
				{labels: "service_name=api,endpoint=/orders", value: 300},
			},
			splitBy: []string{"service_name"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,foo=bar,service_name=app",
					samples: []testSample{
						{labels: "endpoint=/users", value: 100},
						{labels: "endpoint=/orders", value: 300},
					},
				},
			},
		},
		{
			description:  "split by multiple labels",
			seriesLabels: "__name__=profile,app=web",
			samples: []testSample{
				{labels: "service_name=auth,region=us-east,endpoint=/login", value: 150},
				{labels: "service_name=auth,region=us-west,endpoint=/login", value: 200},
				{labels: "service_name=api,region=us-east,endpoint=/users", value: 250},
				{labels: "service_name=api,region=us-west,endpoint=/orders", value: 300},
			},
			splitBy: []string{"service_name", "region"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=web,endpoint=/login,region=us-east,service_name=auth",
					samples: []testSample{
						{labels: "", value: 150},
					},
				},
				{
					labels: "__name__=profile,app=web,endpoint=/login,region=us-west,service_name=auth",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,app=web,endpoint=/users,region=us-east,service_name=api",
					samples: []testSample{
						{labels: "", value: 250},
					},
				},
				{
					labels: "__name__=profile,app=web,endpoint=/orders,region=us-west,service_name=api",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
		{
			description:  "split by non-existent label",
			seriesLabels: "__name__=profile,app=test",
			samples: []testSample{
				{labels: "service_name=web,endpoint=/users", value: 100},
				{labels: "service_name=api,endpoint=/orders", value: 200},
			},
			splitBy: []string{"missing_label"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=test",
					samples: []testSample{
						{labels: "endpoint=/users,service_name=web", value: 100},
						{labels: "endpoint=/orders,service_name=api", value: 200},
					},
				},
			},
		},
		{
			description:  "samples with no labels",
			seriesLabels: "__name__=profile,env=prod",
			samples: []testSample{
				{labels: "", value: 500},
				{labels: "", value: 600},
			},
			splitBy:  []string{"service_name"},
			expected: []expectedSeries{},
		},
		{
			description:  "split by label with empty value",
			seriesLabels: "__name__=profile,app=web",
			samples: []testSample{
				{labels: "service_name=,endpoint=/health", value: 75},
				{labels: "service_name=api,endpoint=/health", value: 125},
			},
			splitBy: []string{"service_name"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=web,endpoint=/health",
					samples: []testSample{
						{labels: "", value: 75},
					},
				},
				{
					labels: "__name__=profile,app=web,endpoint=/health,service_name=api",
					samples: []testSample{
						{labels: "", value: 125},
					},
				},
			},
		},
		{
			description:  "no split by labels",
			seriesLabels: "__name__=profile,env=test",
			samples: []testSample{
				{labels: "service_name=web,endpoint=/users", value: 400},
				{labels: "service_name=api,endpoint=/orders", value: 500},
			},
			splitBy: []string{},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,env=test",
					samples: []testSample{
						{labels: "endpoint=/users,service_name=web", value: 400},
						{labels: "endpoint=/orders,service_name=api", value: 500},
					},
				},
			},
		},
		{
			description:  "multiple samples with same split-by label value",
			seriesLabels: "__name__=profile,version=1.0",
			samples: []testSample{
				{labels: "service_name=web,method=GET", value: 100},
				{labels: "service_name=web,method=POST", value: 150},
				{labels: "service_name=api,method=GET", value: 200},
			},
			splitBy: []string{"service_name"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,service_name=web,version=1.0",
					samples: []testSample{
						{labels: "method=GET", value: 100},
						{labels: "method=POST", value: 150},
					},
				},
				{
					labels: "__name__=profile,method=GET,service_name=api,version=1.0",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
			},
		},
		{
			description:  "partial overlap between series and sample labels",
			seriesLabels: "__name__=profile,service_name=main,region=us-east",
			samples: []testSample{
				{labels: "service_name=web,env=prod", value: 300},
				{labels: "region=eu-west,env=staging", value: 400},
			},
			splitBy: []string{"env"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,env=prod,region=us-east,service_name=main",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
				{
					labels: "__name__=profile,env=staging,region=us-east,service_name=main",
					samples: []testSample{
						{labels: "", value: 400},
					},
				},
			},
		},
		{
			description:  "complex scenario with overlapping labels",
			seriesLabels: "__name__=profile,app=frontend,env=prod",
			samples: []testSample{
				{labels: "service_name=auth,env=staging,version=v1", value: 100},
				{labels: "service_name=auth,region=us-west,version=v2", value: 200},
				{labels: "service_name=api,env=dev,region=eu-central", value: 300},
			},
			splitBy: []string{"service_name", "version"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=frontend,env=prod,service_name=auth,version=v1",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,app=frontend,env=prod,region=us-west,service_name=auth,version=v2",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,app=frontend,env=prod,region=eu-central,service_name=api",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
		{
			description:  "single sample with multiple labels",
			seriesLabels: "__name__=profile,region=us-east",
			samples: []testSample{
				{labels: "service_name=web,method=GET,status=200", value: 42},
			},
			splitBy: []string{"service_name"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,method=GET,region=us-east,service_name=web,status=200",
					samples: []testSample{
						{labels: "", value: 42},
					},
				},
			},
		},
		{
			description:  "mixed samples - some with labels, some without",
			seriesLabels: "__name__=profile,app=myapp",
			samples: []testSample{
				{labels: "service_name=auth,endpoint=/login", value: 100},
				{labels: "", value: 200},
				{labels: "service_name=api", value: 300},
			},
			splitBy: []string{"service_name"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=myapp",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,app=myapp,endpoint=/login,service_name=auth",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,app=myapp,service_name=api",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
		{
			description:  "split by multiple labels with partial matches",
			seriesLabels: "__name__=profile,environment=prod",
			samples: []testSample{
				{labels: "service_name=web,region=us-east,tier=frontend", value: 100},
				{labels: "service_name=web,tier=frontend", value: 150},
				{labels: "service_name=api,region=us-west", value: 200},
				{labels: "region=eu-central", value: 250},
			},
			splitBy: []string{"service_name", "region"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,environment=prod,region=us-east,service_name=web,tier=frontend",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,environment=prod,service_name=web,tier=frontend",
					samples: []testSample{
						{labels: "", value: 150},
					},
				},
				{
					labels: "__name__=profile,environment=prod,region=us-west,service_name=api",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,environment=prod,region=eu-central",
					samples: []testSample{
						{labels: "", value: 250},
					},
				},
			},
		},
		{
			description:  "unicode and special characters in labels",
			seriesLabels: "__name__=profile,app=测试应用",
			samples: []testSample{
				{labels: "service_name=微服务-api,endpoint=/用户/登录", value: 100},
				{labels: "service_name=web-frontend,endpoint=/status", value: 200},
			},
			splitBy: []string{"service_name"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=测试应用,endpoint=/用户/登录,service_name=微服务-api",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,app=测试应用,endpoint=/status,service_name=web-frontend",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
			},
		},
		{
			description:  "many labels with different combinations",
			seriesLabels: "__name__=profile,cluster=prod-cluster",
			samples: []testSample{
				{labels: "service=auth,method=POST,status=200,region=us-east,az=us-east-1a", value: 50},
				{labels: "service=auth,method=POST,status=200,region=us-east,az=us-east-1b", value: 75},
				{labels: "service=auth,method=GET,status=200,region=us-west,az=us-west-2a", value: 25},
				{labels: "service=api,method=POST,status=500,region=eu-central,az=eu-central-1a", value: 100},
			},
			splitBy: []string{"service", "method", "status"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,cluster=prod-cluster,method=POST,region=us-east,service=auth,status=200",
					samples: []testSample{
						{labels: "az=us-east-1a", value: 50},
						{labels: "az=us-east-1b", value: 75},
					},
				},
				{
					labels: "__name__=profile,az=us-west-2a,cluster=prod-cluster,method=GET,region=us-west,service=auth,status=200",
					samples: []testSample{
						{labels: "", value: 25},
					},
				},
				{
					labels: "__name__=profile,az=eu-central-1a,cluster=prod-cluster,method=POST,region=eu-central,service=api,status=500",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
			},
		},
		{
			description:  "split by labels that exist in series labels",
			seriesLabels: "__name__=profile,service_name=main-service,region=global",
			samples: []testSample{
				{labels: "service_name=auth,endpoint=/login", value: 100},
				{labels: "region=us-east,endpoint=/health", value: 200},
				{labels: "method=GET", value: 300},
			},
			splitBy: []string{"service_name", "region"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,region=global,service_name=main-service",
					samples: []testSample{
						{labels: "endpoint=/login", value: 100},
						{labels: "endpoint=/health", value: 200},
						{labels: "method=GET", value: 300},
					},
				},
			},
		},
		{
			description:  "empty string values in split-by labels",
			seriesLabels: "__name__=profile,app=test-app",
			samples: []testSample{
				{labels: "env=,version=v1.0,service=web", value: 100},
				{labels: "env=prod,version=,service=api", value: 200},
				{labels: "env=staging,version=v2.0,service=", value: 300},
			},
			splitBy: []string{"env", "version", "service"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=test-app,service=web,version=v1.0",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,app=test-app,env=prod,service=api",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,app=test-app,env=staging,version=v2.0",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
		{
			description:  "duplicate split-by label values across different samples",
			seriesLabels: "__name__=profile,datacenter=dc1",
			samples: []testSample{
				{labels: "service=web,tier=frontend,instance=web-1", value: 100},
				{labels: "service=web,tier=frontend,instance=web-2", value: 150},
				{labels: "service=web,tier=backend,instance=web-3", value: 200},
				{labels: "service=api,tier=frontend,instance=api-1", value: 250},
			},
			splitBy: []string{"service", "tier"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,datacenter=dc1,service=web,tier=frontend",
					samples: []testSample{
						{labels: "instance=web-1", value: 100},
						{labels: "instance=web-2", value: 150},
					},
				},
				{
					labels: "__name__=profile,datacenter=dc1,instance=web-3,service=web,tier=backend",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,datacenter=dc1,instance=api-1,service=api,tier=frontend",
					samples: []testSample{
						{labels: "", value: 250},
					},
				},
			},
		},
		{
			description:  "relabel rules drop entire profile",
			seriesLabels: "__name__=profile,app=test",
			samples: []testSample{
				{labels: "service=auth,env=prod", value: 100},
				{labels: "service=api,env=staging", value: 200},
			},
			splitBy: []string{"service"},
			rules: []*relabel.Config{
				{
					Action:       relabel.Drop,
					Regex:        relabel.MustNewRegexp("test"),
					SourceLabels: []model.LabelName{"app"},
				},
			},
			expected: []expectedSeries{},
		},
		{
			description:  "relabel rules drop some sample groups",
			seriesLabels: "__name__=profile,component=backend",
			samples: []testSample{
				{labels: "service=auth,env=prod", value: 100},
				{labels: "service=api,env=staging", value: 200},
				{labels: "service=web,env=prod", value: 300},
			},
			splitBy: []string{"service"},
			rules: []*relabel.Config{
				{
					Action:       relabel.Drop,
					Regex:        relabel.MustNewRegexp("staging"),
					SourceLabels: []model.LabelName{"env"},
				},
			},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,component=backend,env=prod,service=auth",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,component=backend,env=prod,service=web",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
		{
			description:  "samples with same stack trace get merged",
			seriesLabels: "__name__=profile,app=merger",
			samples: []testSample{
				{labels: "service=auth,method=GET", value: 100},
				{labels: "service=auth,method=GET", value: 50}, // Same labels but different location IDs won't merge
				{labels: "service=api,method=POST", value: 200},
			},
			splitBy: []string{"service"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=merger,method=GET,service=auth",
					samples: []testSample{
						{labels: "", value: 100},
						{labels: "", value: 50}, // Separate samples since different location IDs
					},
				},
				{
					labels: "__name__=profile,app=merger,method=POST,service=api",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
			},
		},
		{
			description:  "empty profile after all groups dropped by relabel rules",
			seriesLabels: "__name__=profile,app=filter",
			samples: []testSample{
				{labels: "drop=true,service=auth", value: 100},
				{labels: "drop=true,service=api", value: 200},
			},
			splitBy: []string{"service"},
			rules: []*relabel.Config{
				{
					Action:       relabel.Drop,
					Regex:        relabel.MustNewRegexp("true"),
					SourceLabels: []model.LabelName{"drop"},
				},
			},
			expected: []expectedSeries{},
		},
		{
			description:  "complex sample merging with multiple values",
			seriesLabels: "__name__=profile,cluster=main",
			samples: []testSample{
				{labels: "service=web,endpoint=/api", value: 100},
				{labels: "service=web,endpoint=/api", value: 150}, // Same labels but different LocationIds
				{labels: "service=web,endpoint=/health", value: 50},
				{labels: "service=web,endpoint=/api", value: 75}, // Same labels but different LocationIds
			},
			splitBy: []string{"service"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,cluster=main,service=web",
					samples: []testSample{
						{labels: "endpoint=/api", value: 100},
						{labels: "endpoint=/api", value: 150},
						{labels: "endpoint=/api", value: 75},
						{labels: "endpoint=/health", value: 50},
					},
				},
			},
		},
		{
			description:  "string table expansion with new label names and values",
			seriesLabels: "__name__=profile,existing_label=existing_value",
			samples: []testSample{
				{labels: "completely_new_label=completely_new_value", value: 100},
				{labels: "another_new_label=another_new_value", value: 200},
			},
			splitBy: []string{"completely_new_label"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,completely_new_label=completely_new_value,existing_label=existing_value",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,another_new_label=another_new_value,existing_label=existing_value",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
			},
		},
		{
			description:  "samples with identical location IDs for merging",
			seriesLabels: "__name__=profile,service=test",
			samples: []testSample{
				{labels: "endpoint=/api,method=GET", value: 200},
				{labels: "endpoint=/api,method=GET", value: 300}, // Same labels but different location IDs
				{labels: "endpoint=/health,method=GET", value: 100},
			},
			splitBy: []string{"endpoint"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,endpoint=/api,method=GET,service=test",
					samples: []testSample{
						{labels: "", value: 200},
						{labels: "", value: 300}, // Separate samples due to different location IDs
					},
				},
				{
					labels: "__name__=profile,endpoint=/health,method=GET,service=test",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
			},
		},
		{
			description:  "fingerprint collision with different label sets",
			seriesLabels: "__name__=profile,cluster=test",
			samples: []testSample{
				// These might create hash collisions but have different actual labels
				{labels: "service=a,region=b", value: 100},
				{labels: "service=c,region=d", value: 200},
				{labels: "service=a,region=b", value: 50}, // Same labels but different location IDs
			},
			splitBy: []string{"service"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,cluster=test,region=b,service=a",
					samples: []testSample{
						{labels: "", value: 100},
						{labels: "", value: 50}, // Separate samples due to different location IDs
					},
				},
				{
					labels: "__name__=profile,cluster=test,region=d,service=c",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
			},
		},
		{
			description:  "no samples in profile",
			seriesLabels: "__name__=profile,app=empty",
			samples:      []testSample{},
			splitBy:      []string{"service"},
			expected:     []expectedSeries{},
		},
		{
			description:  "profile with only samples having empty values",
			seriesLabels: "__name__=profile,app=zero",
			samples: []testSample{
				{labels: "service=auth", value: 0},
				{labels: "service=api", value: 0},
			},
			splitBy: []string{"service"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=zero,service=auth",
					samples: []testSample{
						{labels: "", value: 0},
					},
				},
				{
					labels: "__name__=profile,app=zero,service=api",
					samples: []testSample{
						{labels: "", value: 0},
					},
				},
			},
		},
		{
			description:  "force string table expansion during sample processing",
			seriesLabels: "__name__=profile,app=test",
			samples: []testSample{
				{labels: "service=auth", value: 100},
				{labels: "service=api", value: 200},
			},
			splitBy: []string{"service"},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=test,service=auth",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,app=test,service=api",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
			},
		},
		{
			description:  "relabel rules drop pprof labels",
			seriesLabels: "__name__=profile,app=test",
			samples: []testSample{
				{labels: "service=auth,internal_debug=true,endpoint=/login", value: 100},
				{labels: "service=api,internal_debug=false,endpoint=/users", value: 200},
				{labels: "service=web,temp_flag=remove_me,endpoint=/health", value: 300},
			},
			splitBy: []string{"service"},
			rules: []*relabel.Config{
				{
					Action: relabel.LabelDrop,
					Regex:  relabel.MustNewRegexp("internal_.*|temp_.*"),
				},
			},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=test,endpoint=/login,service=auth",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,app=test,endpoint=/users,service=api",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,app=test,endpoint=/health,service=web",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
		{
			description:  "relabel rules keep only labels without whitespace",
			seriesLabels: "__name__=profile,app=filtertest,service=auth",
			samples: []testSample{
				{labels: "bad label=value1,endpoint=/login,temp flag=debug", value: 100},
				{labels: "another bad=value2,endpoint=/users,good_label=keep", value: 200},
				{labels: "weird name=value3,endpoint=/health", value: 300},
			},
			splitBy: []string{"service"},
			rules: []*relabel.Config{
				{
					Action: relabel.LabelKeep,
					Regex:  relabel.MustNewRegexp("^[^\\s]+$"),
				},
			},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=filtertest,service=auth",
					samples: []testSample{
						{labels: "endpoint=/login", value: 100},
						{labels: "endpoint=/users,good_label=keep", value: 200},
						{labels: "endpoint=/health", value: 300},
					},
				},
			},
		},
		{
			description:  "relabel rules replace dots with underscores in label values",
			seriesLabels: "__name__=profile,app=normalizer",
			samples: []testSample{
				{labels: "service=com.example.auth,endpoint=/api/v1.0,version=1.2.3", value: 100},
				{labels: "service=org.service.api,endpoint=/health.check,version=2.0.1", value: 200},
				{labels: "service=net.frontend.web,endpoint=/home.page,version=1.0.0", value: 300},
			},
			splitBy: []string{"service"},
			rules: []*relabel.Config{
				{
					SourceLabels: []model.LabelName{"service"},
					Regex:        relabel.MustNewRegexp("([^.]*)\\.(.*)\\.(.*)"),
					Replacement:  "${1}_${2}_${3}",
					TargetLabel:  "service",
					Action:       relabel.Replace,
				},
				{
					SourceLabels: []model.LabelName{"endpoint"},
					Regex:        relabel.MustNewRegexp("([^.]*)\\.(.*)"),
					Replacement:  "${1}_${2}",
					TargetLabel:  "endpoint",
					Action:       relabel.Replace,
				},
				{
					SourceLabels: []model.LabelName{"version"},
					Regex:        relabel.MustNewRegexp("([^.]*)\\.(.*)\\.(.*)"),
					Replacement:  "${1}_${2}_${3}",
					TargetLabel:  "version",
					Action:       relabel.Replace,
				},
			},
			expected: []expectedSeries{
				{
					labels: "__name__=profile,app=normalizer,endpoint=/api/v1_0,service=com_example_auth,version=1_2_3",
					samples: []testSample{
						{labels: "", value: 100},
					},
				},
				{
					labels: "__name__=profile,app=normalizer,endpoint=/health_check,service=org_service_api,version=2_0_1",
					samples: []testSample{
						{labels: "", value: 200},
					},
				},
				{
					labels: "__name__=profile,app=normalizer,endpoint=/home_page,service=net_frontend_web,version=1_0_0",
					samples: []testSample{
						{labels: "", value: 300},
					},
				},
			},
		},
	}

	for _, tc := range testCases {
		t.Run(tc.description, func(t *testing.T) {
			profile := &profilev1.Profile{
				StringTable: []string{""},
				Sample:      make([]*profilev1.Sample, len(tc.samples)),
			}
			lookup := stringLookup(profile)
			reverseLookup := stringReverseLookup(profile)
			for i, s := range tc.samples {
				profile.Sample[i] = &profilev1.Sample{
					LocationId: []uint64{uint64(i + 1)}, // unique location ID per sample
					Value:      []int64{s.value},
					Label:      parseSampleLabels(t, lookup, s.labels),
				}
			}

			visitor := new(mockVisitor)
			seriesLabels := parseLabels(t, tc.seriesLabels)
			require.NoError(t, VisitSampleSeriesBy(profile, seriesLabels, tc.rules, visitor, tc.splitBy...))
			require.Len(t, visitor.series, len(tc.expected))

			for i, actual := range visitor.series {
				expected := tc.expected[i]
				expectedLabels := parseLabels(t, expected.labels)
				assert.Equal(t, expectedLabels, actual.labels, fmt.Sprintf("want: %s,\ngot:  %s",
					formatLabels(expectedLabels),
					formatLabels(actual.labels)))

				require.Len(t, actual.samples, len(expected.samples))
				for j, actualSample := range actual.samples {
					expectedSample := expected.samples[j]
					assert.Equal(t, expectedSample.value, actualSample.Value[0])
					expectedSampleLabels := parseSampleLabels(t, lookup, expectedSample.labels)
					assert.Equal(t, expectedSampleLabels, actualSample.Label, fmt.Sprintf("want: %s, got %s",
						formatSampleLabels(reverseLookup, expectedSampleLabels),
						formatSampleLabels(reverseLookup, actualSample.Label)))
				}
			}
		})
	}
}

func stringLookup(p *profilev1.Profile) func(string) int64 {
	stringIndex := map[string]int64{"": 0}
	return func(s string) int64 {
		if idx, ok := stringIndex[s]; ok {
			return idx
		}
		i := int64(len(p.StringTable))
		p.StringTable = append(p.StringTable, s)
		stringIndex[s] = i
		return i
	}
}

func stringReverseLookup(p *profilev1.Profile) func(int64) string {
	return func(i int64) string {
		return p.StringTable[i]
	}
}

func parseLabels(t *testing.T, s string) []*typesv1.LabelPair {
	if s == "" {
		// To simplify assertions we return an empty slice instead of nil.
		return []*typesv1.LabelPair{}
	}
	var labels []*typesv1.LabelPair
	for _, pair := range strings.Split(s, ",") {
		parts := strings.SplitN(pair, "=", 2)
		if len(parts) != 2 {
			t.Fatal("invalid series labels:", s)
		}
		labels = append(labels, &typesv1.LabelPair{
			Name:  parts[0],
			Value: parts[1],
		})
	}
	return labels
}

func formatLabels(labels []*typesv1.LabelPair) string {
	var b strings.Builder
	for i, label := range labels {
		if i > 0 {
			b.WriteByte(',')
		}
		b.WriteString(label.Name + "=" + label.Value)
	}
	return b.String()
}

func parseSampleLabels(t *testing.T, str func(string) int64, s string) []*profilev1.Label {
	if s == "" {
		// To simplify assertions we return an empty slice instead of nil.
		return []*profilev1.Label{}
	}
	var labels []*profilev1.Label
	for _, pair := range strings.Split(s, ",") {
		parts := strings.SplitN(pair, "=", 2)
		if len(parts) != 2 {
			t.Fatal("invalid sample labels:", s)
		}
		labels = append(labels, &profilev1.Label{
			Key: str(parts[0]),
			Str: str(parts[1]),
		})
	}
	return labels
}

func formatSampleLabels(lookup func(int64) string, labels []*profilev1.Label) string {
	var b strings.Builder
	for i, label := range labels {
		if i > 0 {
			b.WriteByte(',')
		}
		b.WriteString(lookup(label.Key) + "=" + lookup(label.Str))
	}
	return b.String()
}