github.com/cilium/ebpf@v0.15.1-0.20240517100537-8079b37aa138/prog.go (about)

     1  package ebpf
     2  
     3  import (
     4  	"bytes"
     5  	"encoding/binary"
     6  	"errors"
     7  	"fmt"
     8  	"math"
     9  	"path/filepath"
    10  	"runtime"
    11  	"strings"
    12  	"time"
    13  	"unsafe"
    14  
    15  	"github.com/cilium/ebpf/asm"
    16  	"github.com/cilium/ebpf/btf"
    17  	"github.com/cilium/ebpf/internal"
    18  	"github.com/cilium/ebpf/internal/kallsyms"
    19  	"github.com/cilium/ebpf/internal/sys"
    20  	"github.com/cilium/ebpf/internal/sysenc"
    21  	"github.com/cilium/ebpf/internal/unix"
    22  )
    23  
    24  // ErrNotSupported is returned whenever the kernel doesn't support a feature.
    25  var ErrNotSupported = internal.ErrNotSupported
    26  
    27  // errBadRelocation is returned when the verifier rejects a program due to a
    28  // bad CO-RE relocation.
    29  //
    30  // This error is detected based on heuristics and therefore may not be reliable.
    31  var errBadRelocation = errors.New("bad CO-RE relocation")
    32  
    33  // errUnknownKfunc is returned when the verifier rejects a program due to an
    34  // unknown kfunc.
    35  //
    36  // This error is detected based on heuristics and therefore may not be reliable.
    37  var errUnknownKfunc = errors.New("unknown kfunc")
    38  
    39  // ProgramID represents the unique ID of an eBPF program.
    40  type ProgramID uint32
    41  
    42  const (
    43  	// Number of bytes to pad the output buffer for BPF_PROG_TEST_RUN.
    44  	// This is currently the maximum of spare space allocated for SKB
    45  	// and XDP programs, and equal to XDP_PACKET_HEADROOM + NET_IP_ALIGN.
    46  	outputPad = 256 + 2
    47  )
    48  
    49  // DefaultVerifierLogSize is the default number of bytes allocated for the
    50  // verifier log.
    51  const DefaultVerifierLogSize = 64 * 1024
    52  
    53  // maxVerifierLogSize is the maximum size of verifier log buffer the kernel
    54  // will accept before returning EINVAL.
    55  const maxVerifierLogSize = math.MaxUint32 >> 2
    56  
    57  // ProgramOptions control loading a program into the kernel.
    58  type ProgramOptions struct {
    59  	// Bitmap controlling the detail emitted by the kernel's eBPF verifier log.
    60  	// LogLevel-type values can be ORed together to request specific kinds of
    61  	// verifier output. See the documentation on [ebpf.LogLevel] for details.
    62  	//
    63  	//  opts.LogLevel = (ebpf.LogLevelBranch | ebpf.LogLevelStats)
    64  	//
    65  	// If left to its default value, the program will first be loaded without
    66  	// verifier output enabled. Upon error, the program load will be repeated
    67  	// with LogLevelBranch and the given (or default) LogSize value.
    68  	//
    69  	// Unless LogDisabled is set, setting this to a non-zero value will enable the verifier
    70  	// log, populating the [ebpf.Program.VerifierLog] field on successful loads
    71  	// and including detailed verifier errors if the program is rejected. This
    72  	// will always allocate an output buffer, but will result in only a single
    73  	// attempt at loading the program.
    74  	LogLevel LogLevel
    75  
    76  	// Controls the output buffer size for the verifier log, in bytes. See the
    77  	// documentation on ProgramOptions.LogLevel for details about how this value
    78  	// is used.
    79  	//
    80  	// If this value is set too low to fit the verifier log, the resulting
    81  	// [ebpf.VerifierError]'s Truncated flag will be true, and the error string
    82  	// will also contain a hint to that effect.
    83  	//
    84  	// Defaults to DefaultVerifierLogSize.
    85  	LogSize int
    86  
    87  	// Disables the verifier log completely, regardless of other options.
    88  	LogDisabled bool
    89  
    90  	// Type information used for CO-RE relocations.
    91  	//
    92  	// This is useful in environments where the kernel BTF is not available
    93  	// (containers) or where it is in a non-standard location. Defaults to
    94  	// use the kernel BTF from a well-known location if nil.
    95  	KernelTypes *btf.Spec
    96  
    97  	// Type information used for CO-RE relocations of kernel modules,
    98  	// indexed by module name.
    99  	//
   100  	// This is useful in environments where the kernel BTF is not available
   101  	// (containers) or where it is in a non-standard location. Defaults to
   102  	// use the kernel module BTF from a well-known location if nil.
   103  	KernelModuleTypes map[string]*btf.Spec
   104  }
   105  
   106  // ProgramSpec defines a Program.
   107  type ProgramSpec struct {
   108  	// Name is passed to the kernel as a debug aid. Must only contain
   109  	// alpha numeric and '_' characters.
   110  	Name string
   111  
   112  	// Type determines at which hook in the kernel a program will run.
   113  	Type ProgramType
   114  
   115  	// AttachType of the program, needed to differentiate allowed context
   116  	// accesses in some newer program types like CGroupSockAddr.
   117  	//
   118  	// Available on kernels 4.17 and later.
   119  	AttachType AttachType
   120  
   121  	// Name of a kernel data structure or function to attach to. Its
   122  	// interpretation depends on Type and AttachType.
   123  	AttachTo string
   124  
   125  	// The program to attach to. Must be provided manually.
   126  	AttachTarget *Program
   127  
   128  	// The name of the ELF section this program originated from.
   129  	SectionName string
   130  
   131  	Instructions asm.Instructions
   132  
   133  	// Flags is passed to the kernel and specifies additional program
   134  	// load attributes.
   135  	Flags uint32
   136  
   137  	// License of the program. Some helpers are only available if
   138  	// the license is deemed compatible with the GPL.
   139  	//
   140  	// See https://www.kernel.org/doc/html/latest/process/license-rules.html#id1
   141  	License string
   142  
   143  	// Version used by Kprobe programs.
   144  	//
   145  	// Deprecated on kernels 5.0 and later. Leave empty to let the library
   146  	// detect this value automatically.
   147  	KernelVersion uint32
   148  
   149  	// The byte order this program was compiled for, may be nil.
   150  	ByteOrder binary.ByteOrder
   151  }
   152  
   153  // Copy returns a copy of the spec.
   154  func (ps *ProgramSpec) Copy() *ProgramSpec {
   155  	if ps == nil {
   156  		return nil
   157  	}
   158  
   159  	cpy := *ps
   160  	cpy.Instructions = make(asm.Instructions, len(ps.Instructions))
   161  	copy(cpy.Instructions, ps.Instructions)
   162  	return &cpy
   163  }
   164  
   165  // Tag calculates the kernel tag for a series of instructions.
   166  //
   167  // Use asm.Instructions.Tag if you need to calculate for non-native endianness.
   168  func (ps *ProgramSpec) Tag() (string, error) {
   169  	return ps.Instructions.Tag(internal.NativeEndian)
   170  }
   171  
   172  // KernelModule returns the kernel module, if any, the AttachTo function is contained in.
   173  func (ps *ProgramSpec) KernelModule() (string, error) {
   174  	if ps.AttachTo == "" {
   175  		return "", nil
   176  	}
   177  
   178  	switch ps.Type {
   179  	default:
   180  		return "", nil
   181  	case Tracing:
   182  		switch ps.AttachType {
   183  		default:
   184  			return "", nil
   185  		case AttachTraceFEntry:
   186  		case AttachTraceFExit:
   187  		}
   188  		fallthrough
   189  	case Kprobe:
   190  		return kallsyms.KernelModule(ps.AttachTo)
   191  	}
   192  }
   193  
   194  // VerifierError is returned by [NewProgram] and [NewProgramWithOptions] if a
   195  // program is rejected by the verifier.
   196  //
   197  // Use [errors.As] to access the error.
   198  type VerifierError = internal.VerifierError
   199  
   200  // Program represents BPF program loaded into the kernel.
   201  //
   202  // It is not safe to close a Program which is used by other goroutines.
   203  type Program struct {
   204  	// Contains the output of the kernel verifier if enabled,
   205  	// otherwise it is empty.
   206  	VerifierLog string
   207  
   208  	fd         *sys.FD
   209  	name       string
   210  	pinnedPath string
   211  	typ        ProgramType
   212  }
   213  
   214  // NewProgram creates a new Program.
   215  //
   216  // See [NewProgramWithOptions] for details.
   217  //
   218  // Returns a [VerifierError] containing the full verifier log if the program is
   219  // rejected by the kernel.
   220  func NewProgram(spec *ProgramSpec) (*Program, error) {
   221  	return NewProgramWithOptions(spec, ProgramOptions{})
   222  }
   223  
   224  // NewProgramWithOptions creates a new Program.
   225  //
   226  // Loading a program for the first time will perform
   227  // feature detection by loading small, temporary programs.
   228  //
   229  // Returns a [VerifierError] containing the full verifier log if the program is
   230  // rejected by the kernel.
   231  func NewProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) {
   232  	if spec == nil {
   233  		return nil, errors.New("can't load a program from a nil spec")
   234  	}
   235  
   236  	prog, err := newProgramWithOptions(spec, opts)
   237  	if errors.Is(err, asm.ErrUnsatisfiedMapReference) {
   238  		return nil, fmt.Errorf("cannot load program without loading its whole collection: %w", err)
   239  	}
   240  	return prog, err
   241  }
   242  
   243  var (
   244  	coreBadLoad = []byte(fmt.Sprintf("(18) r10 = 0x%x\n", btf.COREBadRelocationSentinel))
   245  	// This log message was introduced by ebb676daa1a3 ("bpf: Print function name in
   246  	// addition to function id") which first appeared in v4.10 and has remained
   247  	// unchanged since.
   248  	coreBadCall  = []byte(fmt.Sprintf("invalid func unknown#%d\n", btf.COREBadRelocationSentinel))
   249  	kfuncBadCall = []byte(fmt.Sprintf("invalid func unknown#%d\n", kfuncCallPoisonBase))
   250  )
   251  
   252  func newProgramWithOptions(spec *ProgramSpec, opts ProgramOptions) (*Program, error) {
   253  	if len(spec.Instructions) == 0 {
   254  		return nil, errors.New("instructions cannot be empty")
   255  	}
   256  
   257  	if spec.Type == UnspecifiedProgram {
   258  		return nil, errors.New("can't load program of unspecified type")
   259  	}
   260  
   261  	if spec.ByteOrder != nil && spec.ByteOrder != internal.NativeEndian {
   262  		return nil, fmt.Errorf("can't load %s program on %s", spec.ByteOrder, internal.NativeEndian)
   263  	}
   264  
   265  	if opts.LogSize < 0 {
   266  		return nil, errors.New("ProgramOptions.LogSize must be a positive value; disable verifier logs using ProgramOptions.LogDisabled")
   267  	}
   268  
   269  	// Kernels before 5.0 (6c4fc209fcf9 "bpf: remove useless version check for prog load")
   270  	// require the version field to be set to the value of the KERNEL_VERSION
   271  	// macro for kprobe-type programs.
   272  	// Overwrite Kprobe program version if set to zero or the magic version constant.
   273  	kv := spec.KernelVersion
   274  	if spec.Type == Kprobe && (kv == 0 || kv == internal.MagicKernelVersion) {
   275  		v, err := internal.KernelVersion()
   276  		if err != nil {
   277  			return nil, fmt.Errorf("detecting kernel version: %w", err)
   278  		}
   279  		kv = v.Kernel()
   280  	}
   281  
   282  	attr := &sys.ProgLoadAttr{
   283  		ProgType:           sys.ProgType(spec.Type),
   284  		ProgFlags:          spec.Flags,
   285  		ExpectedAttachType: sys.AttachType(spec.AttachType),
   286  		License:            sys.NewStringPointer(spec.License),
   287  		KernVersion:        kv,
   288  	}
   289  
   290  	if haveObjName() == nil {
   291  		attr.ProgName = sys.NewObjName(spec.Name)
   292  	}
   293  
   294  	insns := make(asm.Instructions, len(spec.Instructions))
   295  	copy(insns, spec.Instructions)
   296  
   297  	kmodName, err := spec.KernelModule()
   298  	if err != nil {
   299  		return nil, fmt.Errorf("kernel module search: %w", err)
   300  	}
   301  
   302  	var targets []*btf.Spec
   303  	if opts.KernelTypes != nil {
   304  		targets = append(targets, opts.KernelTypes)
   305  	}
   306  	if kmodName != "" && opts.KernelModuleTypes != nil {
   307  		if modBTF, ok := opts.KernelModuleTypes[kmodName]; ok {
   308  			targets = append(targets, modBTF)
   309  		}
   310  	}
   311  
   312  	var b btf.Builder
   313  	if err := applyRelocations(insns, targets, kmodName, spec.ByteOrder, &b); err != nil {
   314  		return nil, fmt.Errorf("apply CO-RE relocations: %w", err)
   315  	}
   316  
   317  	errExtInfos := haveProgramExtInfos()
   318  	if !b.Empty() && errors.Is(errExtInfos, ErrNotSupported) {
   319  		// There is at least one CO-RE relocation which relies on a stable local
   320  		// type ID.
   321  		// Return ErrNotSupported instead of E2BIG if there is no BTF support.
   322  		return nil, errExtInfos
   323  	}
   324  
   325  	if errExtInfos == nil {
   326  		// Only add func and line info if the kernel supports it. This allows
   327  		// BPF compiled with modern toolchains to work on old kernels.
   328  		fib, lib, err := btf.MarshalExtInfos(insns, &b)
   329  		if err != nil {
   330  			return nil, fmt.Errorf("marshal ext_infos: %w", err)
   331  		}
   332  
   333  		attr.FuncInfoRecSize = btf.FuncInfoSize
   334  		attr.FuncInfoCnt = uint32(len(fib)) / btf.FuncInfoSize
   335  		attr.FuncInfo = sys.NewSlicePointer(fib)
   336  
   337  		attr.LineInfoRecSize = btf.LineInfoSize
   338  		attr.LineInfoCnt = uint32(len(lib)) / btf.LineInfoSize
   339  		attr.LineInfo = sys.NewSlicePointer(lib)
   340  	}
   341  
   342  	if !b.Empty() {
   343  		handle, err := btf.NewHandle(&b)
   344  		if err != nil {
   345  			return nil, fmt.Errorf("load BTF: %w", err)
   346  		}
   347  		defer handle.Close()
   348  
   349  		attr.ProgBtfFd = uint32(handle.FD())
   350  	}
   351  
   352  	kconfig, err := resolveKconfigReferences(insns)
   353  	if err != nil {
   354  		return nil, fmt.Errorf("resolve .kconfig: %w", err)
   355  	}
   356  	defer kconfig.Close()
   357  
   358  	if err := fixupAndValidate(insns); err != nil {
   359  		return nil, err
   360  	}
   361  
   362  	handles, err := fixupKfuncs(insns)
   363  	if err != nil {
   364  		return nil, fmt.Errorf("fixing up kfuncs: %w", err)
   365  	}
   366  	defer handles.Close()
   367  
   368  	if len(handles) > 0 {
   369  		fdArray := handles.fdArray()
   370  		attr.FdArray = sys.NewPointer(unsafe.Pointer(&fdArray[0]))
   371  	}
   372  
   373  	buf := bytes.NewBuffer(make([]byte, 0, insns.Size()))
   374  	err = insns.Marshal(buf, internal.NativeEndian)
   375  	if err != nil {
   376  		return nil, err
   377  	}
   378  
   379  	bytecode := buf.Bytes()
   380  	attr.Insns = sys.NewSlicePointer(bytecode)
   381  	attr.InsnCnt = uint32(len(bytecode) / asm.InstructionSize)
   382  
   383  	if spec.AttachTarget != nil {
   384  		targetID, err := findTargetInProgram(spec.AttachTarget, spec.AttachTo, spec.Type, spec.AttachType)
   385  		if err != nil {
   386  			return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
   387  		}
   388  
   389  		attr.AttachBtfId = targetID
   390  		attr.AttachBtfObjFd = uint32(spec.AttachTarget.FD())
   391  		defer runtime.KeepAlive(spec.AttachTarget)
   392  	} else if spec.AttachTo != "" {
   393  		module, targetID, err := findProgramTargetInKernel(spec.AttachTo, spec.Type, spec.AttachType)
   394  		if err != nil && !errors.Is(err, errUnrecognizedAttachType) {
   395  			// We ignore errUnrecognizedAttachType since AttachTo may be non-empty
   396  			// for programs that don't attach anywhere.
   397  			return nil, fmt.Errorf("attach %s/%s: %w", spec.Type, spec.AttachType, err)
   398  		}
   399  
   400  		attr.AttachBtfId = targetID
   401  		if module != nil {
   402  			attr.AttachBtfObjFd = uint32(module.FD())
   403  			defer module.Close()
   404  		}
   405  	}
   406  
   407  	if opts.LogSize == 0 {
   408  		opts.LogSize = DefaultVerifierLogSize
   409  	}
   410  
   411  	// The caller requested a specific verifier log level. Set up the log buffer.
   412  	var logBuf []byte
   413  	if !opts.LogDisabled && opts.LogLevel != 0 {
   414  		logBuf = make([]byte, opts.LogSize)
   415  		attr.LogLevel = opts.LogLevel
   416  		attr.LogSize = uint32(len(logBuf))
   417  		attr.LogBuf = sys.NewSlicePointer(logBuf)
   418  	}
   419  
   420  	fd, err := sys.ProgLoad(attr)
   421  	if err == nil {
   422  		return &Program{unix.ByteSliceToString(logBuf), fd, spec.Name, "", spec.Type}, nil
   423  	}
   424  
   425  	// An error occurred loading the program, but the caller did not explicitly
   426  	// enable the verifier log. Re-run with branch-level verifier logs enabled to
   427  	// obtain more info. Preserve the original error to return it to the caller.
   428  	// An undersized log buffer will result in ENOSPC regardless of the underlying
   429  	// cause.
   430  	var err2 error
   431  	if !opts.LogDisabled && opts.LogLevel == 0 {
   432  		logBuf = make([]byte, opts.LogSize)
   433  		attr.LogLevel = LogLevelBranch
   434  		attr.LogSize = uint32(len(logBuf))
   435  		attr.LogBuf = sys.NewSlicePointer(logBuf)
   436  
   437  		_, err2 = sys.ProgLoad(attr)
   438  	}
   439  
   440  	end := bytes.IndexByte(logBuf, 0)
   441  	if end < 0 {
   442  		end = len(logBuf)
   443  	}
   444  
   445  	tail := logBuf[max(end-256, 0):end]
   446  	switch {
   447  	case errors.Is(err, unix.EPERM):
   448  		if len(logBuf) > 0 && logBuf[0] == 0 {
   449  			// EPERM due to RLIMIT_MEMLOCK happens before the verifier, so we can
   450  			// check that the log is empty to reduce false positives.
   451  			return nil, fmt.Errorf("load program: %w (MEMLOCK may be too low, consider rlimit.RemoveMemlock)", err)
   452  		}
   453  
   454  	case errors.Is(err, unix.EINVAL):
   455  		if opts.LogSize > maxVerifierLogSize {
   456  			return nil, fmt.Errorf("load program: %w (ProgramOptions.LogSize exceeds maximum value of %d)", err, maxVerifierLogSize)
   457  		}
   458  
   459  		if bytes.Contains(tail, coreBadCall) {
   460  			err = errBadRelocation
   461  			break
   462  		} else if bytes.Contains(tail, kfuncBadCall) {
   463  			err = errUnknownKfunc
   464  			break
   465  		}
   466  
   467  	case errors.Is(err, unix.EACCES):
   468  		if bytes.Contains(tail, coreBadLoad) {
   469  			err = errBadRelocation
   470  			break
   471  		}
   472  	}
   473  
   474  	// hasFunctionReferences may be expensive, so check it last.
   475  	if (errors.Is(err, unix.EINVAL) || errors.Is(err, unix.EPERM)) &&
   476  		hasFunctionReferences(spec.Instructions) {
   477  		if err := haveBPFToBPFCalls(); err != nil {
   478  			return nil, fmt.Errorf("load program: %w", err)
   479  		}
   480  	}
   481  
   482  	truncated := errors.Is(err, unix.ENOSPC) || errors.Is(err2, unix.ENOSPC)
   483  	return nil, internal.ErrorWithLog("load program", err, logBuf, truncated)
   484  }
   485  
   486  // NewProgramFromFD creates a program from a raw fd.
   487  //
   488  // You should not use fd after calling this function.
   489  //
   490  // Requires at least Linux 4.10.
   491  func NewProgramFromFD(fd int) (*Program, error) {
   492  	f, err := sys.NewFD(fd)
   493  	if err != nil {
   494  		return nil, err
   495  	}
   496  
   497  	return newProgramFromFD(f)
   498  }
   499  
   500  // NewProgramFromID returns the program for a given id.
   501  //
   502  // Returns ErrNotExist, if there is no eBPF program with the given id.
   503  func NewProgramFromID(id ProgramID) (*Program, error) {
   504  	fd, err := sys.ProgGetFdById(&sys.ProgGetFdByIdAttr{
   505  		Id: uint32(id),
   506  	})
   507  	if err != nil {
   508  		return nil, fmt.Errorf("get program by id: %w", err)
   509  	}
   510  
   511  	return newProgramFromFD(fd)
   512  }
   513  
   514  func newProgramFromFD(fd *sys.FD) (*Program, error) {
   515  	info, err := newProgramInfoFromFd(fd)
   516  	if err != nil {
   517  		fd.Close()
   518  		return nil, fmt.Errorf("discover program type: %w", err)
   519  	}
   520  
   521  	return &Program{"", fd, info.Name, "", info.Type}, nil
   522  }
   523  
   524  func (p *Program) String() string {
   525  	if p.name != "" {
   526  		return fmt.Sprintf("%s(%s)#%v", p.typ, p.name, p.fd)
   527  	}
   528  	return fmt.Sprintf("%s(%v)", p.typ, p.fd)
   529  }
   530  
   531  // Type returns the underlying type of the program.
   532  func (p *Program) Type() ProgramType {
   533  	return p.typ
   534  }
   535  
   536  // Info returns metadata about the program.
   537  //
   538  // Requires at least 4.10.
   539  func (p *Program) Info() (*ProgramInfo, error) {
   540  	return newProgramInfoFromFd(p.fd)
   541  }
   542  
   543  // Handle returns a reference to the program's type information in the kernel.
   544  //
   545  // Returns ErrNotSupported if the kernel has no BTF support, or if there is no
   546  // BTF associated with the program.
   547  func (p *Program) Handle() (*btf.Handle, error) {
   548  	info, err := p.Info()
   549  	if err != nil {
   550  		return nil, err
   551  	}
   552  
   553  	id, ok := info.BTFID()
   554  	if !ok {
   555  		return nil, fmt.Errorf("program %s: retrieve BTF ID: %w", p, ErrNotSupported)
   556  	}
   557  
   558  	return btf.NewHandleFromID(id)
   559  }
   560  
   561  // FD gets the file descriptor of the Program.
   562  //
   563  // It is invalid to call this function after Close has been called.
   564  func (p *Program) FD() int {
   565  	return p.fd.Int()
   566  }
   567  
   568  // Clone creates a duplicate of the Program.
   569  //
   570  // Closing the duplicate does not affect the original, and vice versa.
   571  //
   572  // Cloning a nil Program returns nil.
   573  func (p *Program) Clone() (*Program, error) {
   574  	if p == nil {
   575  		return nil, nil
   576  	}
   577  
   578  	dup, err := p.fd.Dup()
   579  	if err != nil {
   580  		return nil, fmt.Errorf("can't clone program: %w", err)
   581  	}
   582  
   583  	return &Program{p.VerifierLog, dup, p.name, "", p.typ}, nil
   584  }
   585  
   586  // Pin persists the Program on the BPF virtual file system past the lifetime of
   587  // the process that created it
   588  //
   589  // Calling Pin on a previously pinned program will overwrite the path, except when
   590  // the new path already exists. Re-pinning across filesystems is not supported.
   591  //
   592  // This requires bpffs to be mounted above fileName.
   593  // See https://docs.cilium.io/en/stable/network/kubernetes/configuration/#mounting-bpffs-with-systemd
   594  func (p *Program) Pin(fileName string) error {
   595  	if err := internal.Pin(p.pinnedPath, fileName, p.fd); err != nil {
   596  		return err
   597  	}
   598  	p.pinnedPath = fileName
   599  	return nil
   600  }
   601  
   602  // Unpin removes the persisted state for the Program from the BPF virtual filesystem.
   603  //
   604  // Failed calls to Unpin will not alter the state returned by IsPinned.
   605  //
   606  // Unpinning an unpinned Program returns nil.
   607  func (p *Program) Unpin() error {
   608  	if err := internal.Unpin(p.pinnedPath); err != nil {
   609  		return err
   610  	}
   611  	p.pinnedPath = ""
   612  	return nil
   613  }
   614  
   615  // IsPinned returns true if the Program has a non-empty pinned path.
   616  func (p *Program) IsPinned() bool {
   617  	return p.pinnedPath != ""
   618  }
   619  
   620  // Close the Program's underlying file descriptor, which could unload
   621  // the program from the kernel if it is not pinned or attached to a
   622  // kernel hook.
   623  func (p *Program) Close() error {
   624  	if p == nil {
   625  		return nil
   626  	}
   627  
   628  	return p.fd.Close()
   629  }
   630  
   631  // Various options for Run'ing a Program
   632  type RunOptions struct {
   633  	// Program's data input. Required field.
   634  	//
   635  	// The kernel expects at least 14 bytes input for an ethernet header for
   636  	// XDP and SKB programs.
   637  	Data []byte
   638  	// Program's data after Program has run. Caller must allocate. Optional field.
   639  	DataOut []byte
   640  	// Program's context input. Optional field.
   641  	Context interface{}
   642  	// Program's context after Program has run. Must be a pointer or slice. Optional field.
   643  	ContextOut interface{}
   644  	// Minimum number of times to run Program. Optional field. Defaults to 1.
   645  	//
   646  	// The program may be executed more often than this due to interruptions, e.g.
   647  	// when runtime.AllThreadsSyscall is invoked.
   648  	Repeat uint32
   649  	// Optional flags.
   650  	Flags uint32
   651  	// CPU to run Program on. Optional field.
   652  	// Note not all program types support this field.
   653  	CPU uint32
   654  	// Called whenever the syscall is interrupted, and should be set to testing.B.ResetTimer
   655  	// or similar. Typically used during benchmarking. Optional field.
   656  	//
   657  	// Deprecated: use [testing.B.ReportMetric] with unit "ns/op" instead.
   658  	Reset func()
   659  }
   660  
   661  // Test runs the Program in the kernel with the given input and returns the
   662  // value returned by the eBPF program.
   663  //
   664  // Note: the kernel expects at least 14 bytes input for an ethernet header for
   665  // XDP and SKB programs.
   666  //
   667  // This function requires at least Linux 4.12.
   668  func (p *Program) Test(in []byte) (uint32, []byte, error) {
   669  	// Older kernels ignore the dataSizeOut argument when copying to user space.
   670  	// Combined with things like bpf_xdp_adjust_head() we don't really know what the final
   671  	// size will be. Hence we allocate an output buffer which we hope will always be large
   672  	// enough, and panic if the kernel wrote past the end of the allocation.
   673  	// See https://patchwork.ozlabs.org/cover/1006822/
   674  	var out []byte
   675  	if len(in) > 0 {
   676  		out = make([]byte, len(in)+outputPad)
   677  	}
   678  
   679  	opts := RunOptions{
   680  		Data:    in,
   681  		DataOut: out,
   682  		Repeat:  1,
   683  	}
   684  
   685  	ret, _, err := p.run(&opts)
   686  	if err != nil {
   687  		return ret, nil, fmt.Errorf("test program: %w", err)
   688  	}
   689  	return ret, opts.DataOut, nil
   690  }
   691  
   692  // Run runs the Program in kernel with given RunOptions.
   693  //
   694  // Note: the same restrictions from Test apply.
   695  func (p *Program) Run(opts *RunOptions) (uint32, error) {
   696  	ret, _, err := p.run(opts)
   697  	if err != nil {
   698  		return ret, fmt.Errorf("run program: %w", err)
   699  	}
   700  	return ret, nil
   701  }
   702  
   703  // Benchmark runs the Program with the given input for a number of times
   704  // and returns the time taken per iteration.
   705  //
   706  // Returns the result of the last execution of the program and the time per
   707  // run or an error. reset is called whenever the benchmark syscall is
   708  // interrupted, and should be set to testing.B.ResetTimer or similar.
   709  //
   710  // This function requires at least Linux 4.12.
   711  func (p *Program) Benchmark(in []byte, repeat int, reset func()) (uint32, time.Duration, error) {
   712  	if uint(repeat) > math.MaxUint32 {
   713  		return 0, 0, fmt.Errorf("repeat is too high")
   714  	}
   715  
   716  	opts := RunOptions{
   717  		Data:   in,
   718  		Repeat: uint32(repeat),
   719  		Reset:  reset,
   720  	}
   721  
   722  	ret, total, err := p.run(&opts)
   723  	if err != nil {
   724  		return ret, total, fmt.Errorf("benchmark program: %w", err)
   725  	}
   726  	return ret, total, nil
   727  }
   728  
   729  var haveProgRun = internal.NewFeatureTest("BPF_PROG_RUN", "4.12", func() error {
   730  	prog, err := NewProgram(&ProgramSpec{
   731  		// SocketFilter does not require privileges on newer kernels.
   732  		Type: SocketFilter,
   733  		Instructions: asm.Instructions{
   734  			asm.LoadImm(asm.R0, 0, asm.DWord),
   735  			asm.Return(),
   736  		},
   737  		License: "MIT",
   738  	})
   739  	if err != nil {
   740  		// This may be because we lack sufficient permissions, etc.
   741  		return err
   742  	}
   743  	defer prog.Close()
   744  
   745  	in := internal.EmptyBPFContext
   746  	attr := sys.ProgRunAttr{
   747  		ProgFd:     uint32(prog.FD()),
   748  		DataSizeIn: uint32(len(in)),
   749  		DataIn:     sys.NewSlicePointer(in),
   750  	}
   751  
   752  	err = sys.ProgRun(&attr)
   753  	switch {
   754  	case errors.Is(err, unix.EINVAL):
   755  		// Check for EINVAL specifically, rather than err != nil since we
   756  		// otherwise misdetect due to insufficient permissions.
   757  		return internal.ErrNotSupported
   758  
   759  	case errors.Is(err, unix.EINTR):
   760  		// We know that PROG_TEST_RUN is supported if we get EINTR.
   761  		return nil
   762  
   763  	case errors.Is(err, sys.ENOTSUPP):
   764  		// The first PROG_TEST_RUN patches shipped in 4.12 didn't include
   765  		// a test runner for SocketFilter. ENOTSUPP means PROG_TEST_RUN is
   766  		// supported, but not for the program type used in the probe.
   767  		return nil
   768  	}
   769  
   770  	return err
   771  })
   772  
   773  func (p *Program) run(opts *RunOptions) (uint32, time.Duration, error) {
   774  	if uint(len(opts.Data)) > math.MaxUint32 {
   775  		return 0, 0, fmt.Errorf("input is too long")
   776  	}
   777  
   778  	if err := haveProgRun(); err != nil {
   779  		return 0, 0, err
   780  	}
   781  
   782  	var ctxBytes []byte
   783  	if opts.Context != nil {
   784  		ctx := new(bytes.Buffer)
   785  		if err := binary.Write(ctx, internal.NativeEndian, opts.Context); err != nil {
   786  			return 0, 0, fmt.Errorf("cannot serialize context: %v", err)
   787  		}
   788  		ctxBytes = ctx.Bytes()
   789  	}
   790  
   791  	var ctxOut []byte
   792  	if opts.ContextOut != nil {
   793  		ctxOut = make([]byte, binary.Size(opts.ContextOut))
   794  	}
   795  
   796  	attr := sys.ProgRunAttr{
   797  		ProgFd:      p.fd.Uint(),
   798  		DataSizeIn:  uint32(len(opts.Data)),
   799  		DataSizeOut: uint32(len(opts.DataOut)),
   800  		DataIn:      sys.NewSlicePointer(opts.Data),
   801  		DataOut:     sys.NewSlicePointer(opts.DataOut),
   802  		Repeat:      uint32(opts.Repeat),
   803  		CtxSizeIn:   uint32(len(ctxBytes)),
   804  		CtxSizeOut:  uint32(len(ctxOut)),
   805  		CtxIn:       sys.NewSlicePointer(ctxBytes),
   806  		CtxOut:      sys.NewSlicePointer(ctxOut),
   807  		Flags:       opts.Flags,
   808  		Cpu:         opts.CPU,
   809  	}
   810  
   811  retry:
   812  	for {
   813  		err := sys.ProgRun(&attr)
   814  		if err == nil {
   815  			break retry
   816  		}
   817  
   818  		if errors.Is(err, unix.EINTR) {
   819  			if attr.Repeat <= 1 {
   820  				// Older kernels check whether enough repetitions have been
   821  				// executed only after checking for pending signals.
   822  				//
   823  				//     run signal? done? run ...
   824  				//
   825  				// As a result we can get EINTR for repeat==1 even though
   826  				// the program was run exactly once. Treat this as a
   827  				// successful run instead.
   828  				//
   829  				// Since commit 607b9cc92bd7 ("bpf: Consolidate shared test timing code")
   830  				// the conditions are reversed:
   831  				//     run done? signal? ...
   832  				break retry
   833  			}
   834  
   835  			if opts.Reset != nil {
   836  				opts.Reset()
   837  			}
   838  			continue retry
   839  		}
   840  
   841  		if errors.Is(err, sys.ENOTSUPP) {
   842  			return 0, 0, fmt.Errorf("kernel doesn't support running %s: %w", p.Type(), ErrNotSupported)
   843  		}
   844  
   845  		return 0, 0, err
   846  	}
   847  
   848  	if opts.DataOut != nil {
   849  		if int(attr.DataSizeOut) > cap(opts.DataOut) {
   850  			// Houston, we have a problem. The program created more data than we allocated,
   851  			// and the kernel wrote past the end of our buffer.
   852  			panic("kernel wrote past end of output buffer")
   853  		}
   854  		opts.DataOut = opts.DataOut[:int(attr.DataSizeOut)]
   855  	}
   856  
   857  	if len(ctxOut) != 0 {
   858  		b := bytes.NewReader(ctxOut)
   859  		if err := binary.Read(b, internal.NativeEndian, opts.ContextOut); err != nil {
   860  			return 0, 0, fmt.Errorf("failed to decode ContextOut: %v", err)
   861  		}
   862  	}
   863  
   864  	total := time.Duration(attr.Duration) * time.Nanosecond
   865  	return attr.Retval, total, nil
   866  }
   867  
   868  func unmarshalProgram(buf sysenc.Buffer) (*Program, error) {
   869  	var id uint32
   870  	if err := buf.Unmarshal(&id); err != nil {
   871  		return nil, err
   872  	}
   873  
   874  	// Looking up an entry in a nested map or prog array returns an id,
   875  	// not an fd.
   876  	return NewProgramFromID(ProgramID(id))
   877  }
   878  
   879  func marshalProgram(p *Program, length int) ([]byte, error) {
   880  	if length != 4 {
   881  		return nil, fmt.Errorf("can't marshal program to %d bytes", length)
   882  	}
   883  
   884  	buf := make([]byte, 4)
   885  	internal.NativeEndian.PutUint32(buf, p.fd.Uint())
   886  	return buf, nil
   887  }
   888  
   889  // LoadPinnedProgram loads a Program from a BPF file.
   890  //
   891  // Requires at least Linux 4.11.
   892  func LoadPinnedProgram(fileName string, opts *LoadPinOptions) (*Program, error) {
   893  	fd, err := sys.ObjGet(&sys.ObjGetAttr{
   894  		Pathname:  sys.NewStringPointer(fileName),
   895  		FileFlags: opts.Marshal(),
   896  	})
   897  	if err != nil {
   898  		return nil, err
   899  	}
   900  
   901  	info, err := newProgramInfoFromFd(fd)
   902  	if err != nil {
   903  		_ = fd.Close()
   904  		return nil, fmt.Errorf("info for %s: %w", fileName, err)
   905  	}
   906  
   907  	var progName string
   908  	if haveObjName() == nil {
   909  		progName = info.Name
   910  	} else {
   911  		progName = filepath.Base(fileName)
   912  	}
   913  
   914  	return &Program{"", fd, progName, fileName, info.Type}, nil
   915  }
   916  
   917  // SanitizeName replaces all invalid characters in name with replacement.
   918  // Passing a negative value for replacement will delete characters instead
   919  // of replacing them. Use this to automatically generate valid names for maps
   920  // and programs at runtime.
   921  //
   922  // The set of allowed characters depends on the running kernel version.
   923  // Dots are only allowed as of kernel 5.2.
   924  func SanitizeName(name string, replacement rune) string {
   925  	return strings.Map(func(char rune) rune {
   926  		if invalidBPFObjNameChar(char) {
   927  			return replacement
   928  		}
   929  		return char
   930  	}, name)
   931  }
   932  
   933  // ProgramGetNextID returns the ID of the next eBPF program.
   934  //
   935  // Returns ErrNotExist, if there is no next eBPF program.
   936  func ProgramGetNextID(startID ProgramID) (ProgramID, error) {
   937  	attr := &sys.ProgGetNextIdAttr{Id: uint32(startID)}
   938  	return ProgramID(attr.NextId), sys.ProgGetNextId(attr)
   939  }
   940  
   941  // BindMap binds map to the program and is only released once program is released.
   942  //
   943  // This may be used in cases where metadata should be associated with the program
   944  // which otherwise does not contain any references to the map.
   945  func (p *Program) BindMap(m *Map) error {
   946  	attr := &sys.ProgBindMapAttr{
   947  		ProgFd: uint32(p.FD()),
   948  		MapFd:  uint32(m.FD()),
   949  	}
   950  
   951  	return sys.ProgBindMap(attr)
   952  }
   953  
   954  var errUnrecognizedAttachType = errors.New("unrecognized attach type")
   955  
   956  // find an attach target type in the kernel.
   957  //
   958  // name, progType and attachType determine which type we need to attach to.
   959  //
   960  // The attach target may be in a loaded kernel module.
   961  // In that case the returned handle will be non-nil.
   962  // The caller is responsible for closing the handle.
   963  //
   964  // Returns errUnrecognizedAttachType if the combination of progType and attachType
   965  // is not recognised.
   966  func findProgramTargetInKernel(name string, progType ProgramType, attachType AttachType) (*btf.Handle, btf.TypeID, error) {
   967  	type match struct {
   968  		p ProgramType
   969  		a AttachType
   970  	}
   971  
   972  	var (
   973  		typeName, featureName string
   974  		target                btf.Type
   975  	)
   976  
   977  	switch (match{progType, attachType}) {
   978  	case match{LSM, AttachLSMMac}:
   979  		typeName = "bpf_lsm_" + name
   980  		featureName = name + " LSM hook"
   981  		target = (*btf.Func)(nil)
   982  	case match{Tracing, AttachTraceIter}:
   983  		typeName = "bpf_iter_" + name
   984  		featureName = name + " iterator"
   985  		target = (*btf.Func)(nil)
   986  	case match{Tracing, AttachTraceFEntry}:
   987  		typeName = name
   988  		featureName = fmt.Sprintf("fentry %s", name)
   989  		target = (*btf.Func)(nil)
   990  	case match{Tracing, AttachTraceFExit}:
   991  		typeName = name
   992  		featureName = fmt.Sprintf("fexit %s", name)
   993  		target = (*btf.Func)(nil)
   994  	case match{Tracing, AttachModifyReturn}:
   995  		typeName = name
   996  		featureName = fmt.Sprintf("fmod_ret %s", name)
   997  		target = (*btf.Func)(nil)
   998  	case match{Tracing, AttachTraceRawTp}:
   999  		typeName = fmt.Sprintf("btf_trace_%s", name)
  1000  		featureName = fmt.Sprintf("raw_tp %s", name)
  1001  		target = (*btf.Typedef)(nil)
  1002  	default:
  1003  		return nil, 0, errUnrecognizedAttachType
  1004  	}
  1005  
  1006  	spec, err := btf.LoadKernelSpec()
  1007  	if err != nil {
  1008  		return nil, 0, fmt.Errorf("load kernel spec: %w", err)
  1009  	}
  1010  
  1011  	spec, module, err := findTargetInKernel(spec, typeName, &target)
  1012  	if errors.Is(err, btf.ErrNotFound) {
  1013  		return nil, 0, &internal.UnsupportedFeatureError{Name: featureName}
  1014  	}
  1015  	// See cilium/ebpf#894. Until we can disambiguate between equally-named kernel
  1016  	// symbols, we should explicitly refuse program loads. They will not reliably
  1017  	// do what the caller intended.
  1018  	if errors.Is(err, btf.ErrMultipleMatches) {
  1019  		return nil, 0, fmt.Errorf("attaching to ambiguous kernel symbol is not supported: %w", err)
  1020  	}
  1021  	if err != nil {
  1022  		return nil, 0, fmt.Errorf("find target for %s: %w", featureName, err)
  1023  	}
  1024  
  1025  	id, err := spec.TypeID(target)
  1026  	if err != nil {
  1027  		module.Close()
  1028  		return nil, 0, err
  1029  	}
  1030  
  1031  	return module, id, nil
  1032  }
  1033  
  1034  // findTargetInKernel attempts to find a named type in the current kernel.
  1035  //
  1036  // target will point at the found type after a successful call. Searches both
  1037  // vmlinux and any loaded modules.
  1038  //
  1039  // Returns a non-nil handle if the type was found in a module, or btf.ErrNotFound
  1040  // if the type wasn't found at all.
  1041  func findTargetInKernel(kernelSpec *btf.Spec, typeName string, target *btf.Type) (*btf.Spec, *btf.Handle, error) {
  1042  	err := kernelSpec.TypeByName(typeName, target)
  1043  	if errors.Is(err, btf.ErrNotFound) {
  1044  		spec, module, err := findTargetInModule(kernelSpec, typeName, target)
  1045  		if err != nil {
  1046  			return nil, nil, fmt.Errorf("find target in modules: %w", err)
  1047  		}
  1048  		return spec, module, nil
  1049  	}
  1050  	if err != nil {
  1051  		return nil, nil, fmt.Errorf("find target in vmlinux: %w", err)
  1052  	}
  1053  	return kernelSpec, nil, err
  1054  }
  1055  
  1056  // findTargetInModule attempts to find a named type in any loaded module.
  1057  //
  1058  // base must contain the kernel's types and is used to parse kmod BTF. Modules
  1059  // are searched in the order they were loaded.
  1060  //
  1061  // Returns btf.ErrNotFound if the target can't be found in any module.
  1062  func findTargetInModule(base *btf.Spec, typeName string, target *btf.Type) (*btf.Spec, *btf.Handle, error) {
  1063  	it := new(btf.HandleIterator)
  1064  	defer it.Handle.Close()
  1065  
  1066  	for it.Next() {
  1067  		info, err := it.Handle.Info()
  1068  		if err != nil {
  1069  			return nil, nil, fmt.Errorf("get info for BTF ID %d: %w", it.ID, err)
  1070  		}
  1071  
  1072  		if !info.IsModule() {
  1073  			continue
  1074  		}
  1075  
  1076  		spec, err := it.Handle.Spec(base)
  1077  		if err != nil {
  1078  			return nil, nil, fmt.Errorf("parse types for module %s: %w", info.Name, err)
  1079  		}
  1080  
  1081  		err = spec.TypeByName(typeName, target)
  1082  		if errors.Is(err, btf.ErrNotFound) {
  1083  			continue
  1084  		}
  1085  		if err != nil {
  1086  			return nil, nil, fmt.Errorf("lookup type in module %s: %w", info.Name, err)
  1087  		}
  1088  
  1089  		return spec, it.Take(), nil
  1090  	}
  1091  	if err := it.Err(); err != nil {
  1092  		return nil, nil, fmt.Errorf("iterate modules: %w", err)
  1093  	}
  1094  
  1095  	return nil, nil, btf.ErrNotFound
  1096  }
  1097  
  1098  // find an attach target type in a program.
  1099  //
  1100  // Returns errUnrecognizedAttachType.
  1101  func findTargetInProgram(prog *Program, name string, progType ProgramType, attachType AttachType) (btf.TypeID, error) {
  1102  	type match struct {
  1103  		p ProgramType
  1104  		a AttachType
  1105  	}
  1106  
  1107  	var typeName string
  1108  	switch (match{progType, attachType}) {
  1109  	case match{Extension, AttachNone},
  1110  		match{Tracing, AttachTraceFEntry},
  1111  		match{Tracing, AttachTraceFExit}:
  1112  		typeName = name
  1113  	default:
  1114  		return 0, errUnrecognizedAttachType
  1115  	}
  1116  
  1117  	btfHandle, err := prog.Handle()
  1118  	if err != nil {
  1119  		return 0, fmt.Errorf("load target BTF: %w", err)
  1120  	}
  1121  	defer btfHandle.Close()
  1122  
  1123  	spec, err := btfHandle.Spec(nil)
  1124  	if err != nil {
  1125  		return 0, err
  1126  	}
  1127  
  1128  	var targetFunc *btf.Func
  1129  	err = spec.TypeByName(typeName, &targetFunc)
  1130  	if err != nil {
  1131  		return 0, fmt.Errorf("find target %s: %w", typeName, err)
  1132  	}
  1133  
  1134  	return spec.TypeID(targetFunc)
  1135  }