github.com/grafana/pyroscope@v1.18.0/pkg/metastore/raftnode/node_admin.go (about)

     1  package raftnode
     2  
     3  import (
     4  	"fmt"
     5  
     6  	"github.com/go-kit/log/level"
     7  	"github.com/hashicorp/raft"
     8  	"google.golang.org/grpc/codes"
     9  	"google.golang.org/grpc/status"
    10  
    11  	"github.com/grafana/pyroscope/pkg/metastore/raftnode/raftnodepb"
    12  )
    13  
    14  func (n *Node) RemoveNode(request *raftnodepb.RemoveNodeRequest) (*raftnodepb.RemoveNodeResponse, error) {
    15  	level.Info(n.logger).Log("msg", "removing node", "id", request.ServerId)
    16  
    17  	// Send a round of heartbeats to confirm we are the leader. If we are not, the request will be retried on the leader node.
    18  	if err := n.raft.VerifyLeader().Error(); err != nil {
    19  		level.Error(n.logger).Log("msg", "failed to remove node, we are not the leader", "id", request.ServerId)
    20  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
    21  	}
    22  
    23  	// Verify that we are on the same term as the one in the request. Otherwise, the request could be operating on stale information.
    24  	err := n.verifyCurrentTerm(request.CurrentTerm)
    25  	if err != nil {
    26  		return nil, err
    27  	}
    28  
    29  	// make sure we are not removing ourselves, we need to be demoted first
    30  	if n.config.ServerID == request.ServerId {
    31  		level.Error(n.logger).Log("msg", "failed to remove node, we cannot remove ourselves", "id", request.ServerId)
    32  		return nil, fmt.Errorf("leadership must be transferred first")
    33  	}
    34  
    35  	if err := n.raft.RemoveServer(raft.ServerID(request.ServerId), 0, 0).Error(); err != nil {
    36  		level.Error(n.logger).Log("msg", "failed to remove node, error from raft", "node", request.ServerId, "err", err)
    37  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
    38  	}
    39  
    40  	level.Info(n.logger).Log("msg", "node removed", "id", request.ServerId)
    41  	return &raftnodepb.RemoveNodeResponse{}, nil
    42  }
    43  
    44  func (n *Node) AddNode(request *raftnodepb.AddNodeRequest) (*raftnodepb.AddNodeResponse, error) {
    45  	level.Info(n.logger).Log("msg", "adding node", "id", request.ServerId)
    46  
    47  	// Send a round of heartbeats to confirm we are the leader. If we are not, the request will be retried on the leader node.
    48  	if err := n.raft.VerifyLeader().Error(); err != nil {
    49  		level.Error(n.logger).Log("msg", "failed to add node, we are not the leader", "id", request.ServerId)
    50  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
    51  	}
    52  
    53  	// Verify that we are on the same term as the one in the request. Otherwise, the request could be operating on stale information.
    54  	err := n.verifyCurrentTerm(request.CurrentTerm)
    55  	if err != nil {
    56  		return nil, err
    57  	}
    58  
    59  	if err := n.raft.AddVoter(raft.ServerID(request.ServerId), raft.ServerAddress(request.ServerId), 0, 0).Error(); err != nil {
    60  		level.Error(n.logger).Log("msg", "failed to add node, error from raft", "node", request.ServerId, "err", err)
    61  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
    62  	}
    63  
    64  	level.Info(n.logger).Log("msg", "node added", "id", request.ServerId)
    65  	return &raftnodepb.AddNodeResponse{}, nil
    66  }
    67  
    68  func (n *Node) DemoteLeader(request *raftnodepb.DemoteLeaderRequest) (*raftnodepb.DemoteLeaderResponse, error) {
    69  	level.Info(n.logger).Log("msg", "demoting node", "id", request.ServerId)
    70  
    71  	// Send a round of heartbeats to confirm we are the leader. If we are not, the request will be retried on the leader node.
    72  	if err := n.raft.VerifyLeader().Error(); err != nil {
    73  		level.Error(n.logger).Log("msg", "failed to demote node, we are not the leader", "id", request.ServerId)
    74  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
    75  	}
    76  
    77  	// Verify that we are on the same term as the one in the request. Otherwise, the request could be operating on stale information.
    78  	err := n.verifyCurrentTerm(request.CurrentTerm)
    79  	if err != nil {
    80  		return nil, err
    81  	}
    82  
    83  	// Make sure we are demoting the node from the request (we can only demote ourselves)
    84  	if n.config.ServerID != request.ServerId {
    85  		level.Error(n.logger).Log("msg", "failed to demote node, the target node is not the leader", "id", request.ServerId)
    86  		return nil, fmt.Errorf("the target node is not the leader")
    87  	}
    88  
    89  	if err := n.raft.LeadershipTransfer().Error(); err != nil {
    90  		level.Error(n.logger).Log("msg", "failed to demote node, error from raft", "node", request.ServerId, "err", err)
    91  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
    92  	}
    93  
    94  	level.Info(n.logger).Log("msg", "node demoted", "id", request.ServerId)
    95  	return &raftnodepb.DemoteLeaderResponse{}, nil
    96  }
    97  
    98  func (n *Node) PromoteToLeader(request *raftnodepb.PromoteToLeaderRequest) (*raftnodepb.PromoteToLeaderResponse, error) {
    99  	level.Info(n.logger).Log("msg", "promoting node", "id", request.ServerId)
   100  
   101  	// Send a round of heartbeats to confirm we are the leader. If we are not, the request will be retried on the leader node.
   102  	if err := n.raft.VerifyLeader().Error(); err != nil {
   103  		level.Error(n.logger).Log("msg", "failed to promote node, we are not the leader", "id", request.ServerId)
   104  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
   105  	}
   106  
   107  	// Verify that we are on the same term as the one in the request. Otherwise, the request could be operating on stale information.
   108  	err := n.verifyCurrentTerm(request.CurrentTerm)
   109  	if err != nil {
   110  		return nil, err
   111  	}
   112  
   113  	// make sure we are not promoting ourselves
   114  	if n.config.ServerID == request.ServerId {
   115  		level.Error(n.logger).Log("msg", "failed to promote node, we cannot promote ourselves", "node", request.ServerId)
   116  		return nil, status.Error(codes.InvalidArgument, "a node cannot promote itself")
   117  	}
   118  
   119  	if err := n.raft.LeadershipTransferToServer(raft.ServerID(request.ServerId), raft.ServerAddress(request.ServerId)).Error(); err != nil {
   120  		level.Error(n.logger).Log("msg", "failed to promote node, error from raft", "node", request.ServerId, "err", err)
   121  		return nil, WithRaftLeaderStatusDetails(err, n.raft)
   122  	}
   123  
   124  	level.Info(n.logger).Log("msg", "node promoted", "id", request.ServerId)
   125  	return &raftnodepb.PromoteToLeaderResponse{}, nil
   126  }
   127  
   128  func (n *Node) verifyCurrentTerm(requestTerm uint64) error {
   129  	currentTerm := n.raft.CurrentTerm()
   130  	if requestTerm < currentTerm {
   131  		level.Error(n.logger).Log("msg", "node change request invalid, request term lower than current term", "request_term", requestTerm, "raft_term", currentTerm)
   132  		return status.Error(codes.InvalidArgument, fmt.Sprintf("request term (%d) lower than raft term (%d)", requestTerm, currentTerm))
   133  	}
   134  	return nil
   135  }