github.com/grafana/pyroscope@v1.18.0/pkg/metastore/compaction/compaction.go (about)

     1  package compaction
     2  
     3  import (
     4  	"github.com/hashicorp/raft"
     5  	"go.etcd.io/bbolt"
     6  
     7  	metastorev1 "github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1"
     8  	"github.com/grafana/pyroscope/api/gen/proto/go/metastore/v1/raft_log"
     9  	"github.com/grafana/pyroscope/pkg/block/metadata"
    10  )
    11  
    12  type Compactor interface {
    13  	// Compact enqueues a new block for compaction
    14  	Compact(*bbolt.Tx, BlockEntry) error
    15  }
    16  
    17  type Planner interface {
    18  	// NewPlan is used to plan new jobs. The proposed changes will then be
    19  	// submitted for Raft consensus, with the leader's jobs being accepted
    20  	// as the final decision.
    21  	// Implementation: Plan must not change the state of the Planner.
    22  	NewPlan(*raft.Log) Plan
    23  	// UpdatePlan communicates the status of the compaction job to the planner.
    24  	// Implementation: This method must be idempotent.
    25  	UpdatePlan(*bbolt.Tx, *raft_log.CompactionPlanUpdate) error
    26  }
    27  
    28  type Plan interface {
    29  	// CreateJob creates a plan for a new job.
    30  	CreateJob() (*raft_log.CompactionJobPlan, error)
    31  }
    32  
    33  type Scheduler interface {
    34  	// NewSchedule is used to plan a schedule update. The proposed schedule
    35  	// will then be submitted for Raft consensus, with the leader's schedule
    36  	// being accepted as the final decision.
    37  	// Implementation: Schedule must not change the state of the Scheduler.
    38  	NewSchedule(*bbolt.Tx, *raft.Log) Schedule
    39  	// UpdateSchedule adds new jobs and updates the state of existing ones.
    40  	// Implementation: This method must be idempotent.
    41  	UpdateSchedule(*bbolt.Tx, *raft_log.CompactionPlanUpdate) error
    42  }
    43  
    44  // Schedule prepares changes to the compaction plan based on status updates
    45  // from compaction workers. The standard sequence assumes that job updates
    46  // (including lease renewals and completion reports) occur first, followed by
    47  // the assignment of new jobs to workers. Only after these updates are new
    48  // compaction jobs planned.
    49  type Schedule interface {
    50  	// UpdateJob is called on behalf of the worker to update the job status.
    51  	// A nil response should be interpreted as "no new lease": stop the work.
    52  	// The scheduler must validate that the worker is allowed to update the
    53  	// job by comparing the fencing token of the job.
    54  	// Refer to the documentation for details.
    55  	UpdateJob(*raft_log.CompactionJobStatusUpdate) *raft_log.CompactionJobState
    56  	// AssignJob is called on behalf of the worker to request a new job.
    57  	AssignJob() (*raft_log.AssignedCompactionJob, error)
    58  	// EvictJob is called on behalf of the planner to evict jobs that cannot
    59  	// be assigned to workers, and free up resources for new jobs.
    60  	EvictJob() *raft_log.CompactionJobState
    61  	// AddJob is called on behalf of the planner to add a new job to the schedule.
    62  	// The scheduler may decline the job by returning a nil state.
    63  	AddJob(*raft_log.CompactionJobPlan) *raft_log.CompactionJobState
    64  }
    65  
    66  // BlockEntry represents a block metadata entry compaction operates on.
    67  type BlockEntry struct {
    68  	Index      uint64
    69  	AppendedAt int64
    70  	ID         string
    71  	Tenant     string
    72  	Shard      uint32
    73  	Level      uint32
    74  }
    75  
    76  func NewBlockEntry(cmd *raft.Log, md *metastorev1.BlockMeta) BlockEntry {
    77  	return BlockEntry{
    78  		Index:      cmd.Index,
    79  		AppendedAt: cmd.AppendedAt.UnixNano(),
    80  		ID:         md.Id,
    81  		Tenant:     metadata.Tenant(md),
    82  		Shard:      md.Shard,
    83  		Level:      md.CompactionLevel,
    84  	}
    85  }