github.com/pyroscope-io/pyroscope@v0.37.3-0.20230725203016-5f6947968bd0/pkg/agent/ebpfspy/sd/k8s.go (about)

     1  package sd
     2  
     3  import (
     4  	"bufio"
     5  	"context"
     6  	"fmt"
     7  	"github.com/pyroscope-io/pyroscope/pkg/agent/log"
     8  	"github.com/pyroscope-io/pyroscope/pkg/agent/spy"
     9  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    10  	"k8s.io/apimachinery/pkg/fields"
    11  	"os"
    12  	"regexp"
    13  	"strings"
    14  
    15  	"k8s.io/client-go/kubernetes"
    16  	"k8s.io/client-go/rest"
    17  )
    18  
    19  type K8SServiceDiscovery struct {
    20  	logger             log.Logger
    21  	cs                 *kubernetes.Clientset
    22  	nodeName           string
    23  	containerID2Labels map[string]*spy.Labels
    24  	pid2Labels         map[uint32]*spy.Labels
    25  }
    26  
    27  var knownContainerIDPrefixes = []string{"docker://", "containerd://"}
    28  var knownRuntimes = []string{"docker://", "containerd://"}
    29  
    30  func NewK8ServiceDiscovery(ctx context.Context, logger log.Logger, nodeName string) (ServiceDiscovery, error) {
    31  	config, err := rest.InClusterConfig()
    32  	if err != nil {
    33  		return nil, err
    34  	}
    35  
    36  	clientset, err := kubernetes.NewForConfig(config)
    37  	if err != nil {
    38  		return nil, err
    39  	}
    40  
    41  	node, err := clientset.CoreV1().Nodes().Get(ctx, nodeName, metav1.GetOptions{})
    42  	if err != nil {
    43  		return nil, err
    44  	}
    45  	criVersion := node.Status.NodeInfo.ContainerRuntimeVersion
    46  
    47  	if !isKnownContainerRuntime(criVersion) {
    48  		return nil, fmt.Errorf("unknown cri %s", criVersion)
    49  	}
    50  
    51  	return &K8SServiceDiscovery{
    52  		logger:             logger,
    53  		cs:                 clientset,
    54  		nodeName:           nodeName,
    55  		containerID2Labels: map[string]*spy.Labels{},
    56  		pid2Labels:         map[uint32]*spy.Labels{},
    57  	}, nil
    58  }
    59  
    60  func (sd *K8SServiceDiscovery) Refresh(ctx context.Context) error {
    61  	sd.containerID2Labels = map[string]*spy.Labels{}
    62  	sd.pid2Labels = map[uint32]*spy.Labels{}
    63  	pods, err := sd.cs.CoreV1().Pods("").List(ctx, metav1.ListOptions{
    64  		FieldSelector: fields.OneTermEqualSelector("spec.nodeName", sd.nodeName).String(),
    65  	})
    66  	if err != nil {
    67  		return err
    68  	}
    69  	sd.logger.Debugf("K8SServiceDiscovery#Refresh pods %v", pods)
    70  
    71  	for _, pod := range pods.Items {
    72  		for _, status := range pod.Status.ContainerStatuses {
    73  			if status.ContainerID == "" {
    74  				sd.logger.Debugf("Unknown containerID for pod %v, status %v", pod, status)
    75  				continue
    76  			}
    77  			cid, err := getContainerIDFromK8S(status.ContainerID)
    78  			if err != nil {
    79  				return err
    80  			}
    81  			ls := spy.NewLabels()
    82  			ls.Set("node", sd.nodeName)
    83  			ls.Set("pod", pod.Name)
    84  			ls.Set("namespace", pod.Namespace)
    85  			ls.Set("container_id", cid)
    86  			ls.Set("container_name", status.Name)
    87  			if v, ok := pod.Labels["app.kubernetes.io/name"]; ok {
    88  				ls.Set("app_kubernetes_io_name", v)
    89  			}
    90  			if v, ok := pod.Labels["app.kubernetes.io/version"]; ok {
    91  				ls.Set("app_kubernetes_io_version", v)
    92  			}
    93  			if v, ok := pod.Labels["app.kubernetes.io/instance"]; ok {
    94  				ls.Set("app_kubernetes_io_instance", v)
    95  			}
    96  			sd.containerID2Labels[cid] = ls
    97  		}
    98  	}
    99  	return nil
   100  }
   101  
   102  func (sd *K8SServiceDiscovery) GetLabels(pid uint32) *spy.Labels {
   103  	ls, ok := sd.pid2Labels[pid]
   104  	if ok {
   105  		return ls
   106  	}
   107  	cid := getContainerIDFromPID(pid)
   108  
   109  	if cid == "" {
   110  		sd.pid2Labels[pid] = nil
   111  		return nil
   112  	}
   113  	ls, ok = sd.containerID2Labels[cid]
   114  	sd.pid2Labels[pid] = ls
   115  	return ls
   116  }
   117  
   118  func isKnownContainerRuntime(criVersion string) bool {
   119  	for _, runtime := range knownRuntimes {
   120  		if strings.HasPrefix(criVersion, runtime) {
   121  			return true
   122  		}
   123  	}
   124  	return false
   125  }
   126  
   127  func getContainerIDFromK8S(k8sContainerID string) (string, error) {
   128  	for _, p := range knownContainerIDPrefixes {
   129  		if strings.HasPrefix(k8sContainerID, p) {
   130  			return strings.TrimPrefix(k8sContainerID, p), nil
   131  		}
   132  	}
   133  	return "", fmt.Errorf("unknown container id %s", k8sContainerID)
   134  }
   135  
   136  func getContainerIDFromPID(pid uint32) string {
   137  	f, err := os.Open(fmt.Sprintf("/proc/%d/cgroup", pid))
   138  	if err != nil {
   139  		return ""
   140  	}
   141  	defer f.Close()
   142  
   143  	scanner := bufio.NewScanner(f)
   144  	for scanner.Scan() {
   145  		line := scanner.Text()
   146  		cid := getContainerIDFromCGroup(line)
   147  		if cid != "" {
   148  			return cid
   149  		}
   150  	}
   151  	return ""
   152  }
   153  
   154  func getContainerIDFromCGroup(line string) string {
   155  	parts := dockerPattern.FindStringSubmatch(line)
   156  	if parts != nil {
   157  		return parts[1]
   158  	}
   159  	parts = kubePattern.FindStringSubmatch(line)
   160  	if parts != nil {
   161  		return parts[1]
   162  	}
   163  	parts = cgroupScopePattern.FindStringSubmatch(line)
   164  	if parts != nil {
   165  		return parts[1]
   166  	}
   167  	return ""
   168  }
   169  
   170  var (
   171  	kubePattern        = regexp.MustCompile(`\d+:.+:/kubepods/[^/]+/pod[^/]+/([0-9a-f]{64})`)
   172  	dockerPattern      = regexp.MustCompile(`\d+:.+:/docker/pod[^/]+/([0-9a-f]{64})`)
   173  	cgroupScopePattern = regexp.MustCompile(`^\d+:.*/(?:docker-|cri-containerd-)([0-9a-f]{64})\.scope$`)
   174  )