github.com/cilium/ebpf@v0.15.1-0.20240517100537-8079b37aa138/linker.go (about) 1 package ebpf 2 3 import ( 4 "debug/elf" 5 "encoding/binary" 6 "errors" 7 "fmt" 8 "io" 9 "io/fs" 10 "math" 11 "slices" 12 13 "github.com/cilium/ebpf/asm" 14 "github.com/cilium/ebpf/btf" 15 "github.com/cilium/ebpf/internal" 16 ) 17 18 // handles stores handle objects to avoid gc cleanup 19 type handles []*btf.Handle 20 21 func (hs *handles) add(h *btf.Handle) (int, error) { 22 if h == nil { 23 return 0, nil 24 } 25 26 if len(*hs) == math.MaxInt16 { 27 return 0, fmt.Errorf("can't add more than %d module FDs to fdArray", math.MaxInt16) 28 } 29 30 *hs = append(*hs, h) 31 32 // return length of slice so that indexes start at 1 33 return len(*hs), nil 34 } 35 36 func (hs handles) fdArray() []int32 { 37 // first element of fda is reserved as no module can be indexed with 0 38 fda := []int32{0} 39 for _, h := range hs { 40 fda = append(fda, int32(h.FD())) 41 } 42 43 return fda 44 } 45 46 func (hs *handles) Close() error { 47 var errs []error 48 for _, h := range *hs { 49 errs = append(errs, h.Close()) 50 } 51 return errors.Join(errs...) 52 } 53 54 // splitSymbols splits insns into subsections delimited by Symbol Instructions. 55 // insns cannot be empty and must start with a Symbol Instruction. 56 // 57 // The resulting map is indexed by Symbol name. 58 func splitSymbols(insns asm.Instructions) (map[string]asm.Instructions, error) { 59 if len(insns) == 0 { 60 return nil, errors.New("insns is empty") 61 } 62 63 currentSym := insns[0].Symbol() 64 if currentSym == "" { 65 return nil, errors.New("insns must start with a Symbol") 66 } 67 68 start := 0 69 progs := make(map[string]asm.Instructions) 70 for i, ins := range insns[1:] { 71 i := i + 1 72 73 sym := ins.Symbol() 74 if sym == "" { 75 continue 76 } 77 78 // New symbol, flush the old one out. 79 progs[currentSym] = slices.Clone(insns[start:i]) 80 81 if progs[sym] != nil { 82 return nil, fmt.Errorf("insns contains duplicate Symbol %s", sym) 83 } 84 currentSym = sym 85 start = i 86 } 87 88 if tail := insns[start:]; len(tail) > 0 { 89 progs[currentSym] = slices.Clone(tail) 90 } 91 92 return progs, nil 93 } 94 95 // The linker is responsible for resolving bpf-to-bpf calls between programs 96 // within an ELF. Each BPF program must be a self-contained binary blob, 97 // so when an instruction in one ELF program section wants to jump to 98 // a function in another, the linker needs to pull in the bytecode 99 // (and BTF info) of the target function and concatenate the instruction 100 // streams. 101 // 102 // Later on in the pipeline, all call sites are fixed up with relative jumps 103 // within this newly-created instruction stream to then finally hand off to 104 // the kernel with BPF_PROG_LOAD. 105 // 106 // Each function is denoted by an ELF symbol and the compiler takes care of 107 // register setup before each jump instruction. 108 109 // hasFunctionReferences returns true if insns contains one or more bpf2bpf 110 // function references. 111 func hasFunctionReferences(insns asm.Instructions) bool { 112 for _, i := range insns { 113 if i.IsFunctionReference() { 114 return true 115 } 116 } 117 return false 118 } 119 120 // applyRelocations collects and applies any CO-RE relocations in insns. 121 // 122 // Passing a nil target will relocate against the running kernel. insns are 123 // modified in place. 124 func applyRelocations(insns asm.Instructions, targets []*btf.Spec, kmodName string, bo binary.ByteOrder, b *btf.Builder) error { 125 var relos []*btf.CORERelocation 126 var reloInsns []*asm.Instruction 127 iter := insns.Iterate() 128 for iter.Next() { 129 if relo := btf.CORERelocationMetadata(iter.Ins); relo != nil { 130 relos = append(relos, relo) 131 reloInsns = append(reloInsns, iter.Ins) 132 } 133 } 134 135 if len(relos) == 0 { 136 return nil 137 } 138 139 if bo == nil { 140 bo = internal.NativeEndian 141 } 142 143 if len(targets) == 0 { 144 kernelTarget, err := btf.LoadKernelSpec() 145 if err != nil { 146 return fmt.Errorf("load kernel spec: %w", err) 147 } 148 targets = append(targets, kernelTarget) 149 150 if kmodName != "" { 151 kmodTarget, err := btf.LoadKernelModuleSpec(kmodName) 152 // Ignore ErrNotExists to cater to kernels which have CONFIG_DEBUG_INFO_BTF_MODULES disabled. 153 if err != nil && !errors.Is(err, fs.ErrNotExist) { 154 return fmt.Errorf("load kernel module spec: %w", err) 155 } 156 if err == nil { 157 targets = append(targets, kmodTarget) 158 } 159 } 160 } 161 162 fixups, err := btf.CORERelocate(relos, targets, bo, b.Add) 163 if err != nil { 164 return err 165 } 166 167 for i, fixup := range fixups { 168 if err := fixup.Apply(reloInsns[i]); err != nil { 169 return fmt.Errorf("fixup for %s: %w", relos[i], err) 170 } 171 } 172 173 return nil 174 } 175 176 // flattenPrograms resolves bpf-to-bpf calls for a set of programs. 177 // 178 // Links all programs in names by modifying their ProgramSpec in progs. 179 func flattenPrograms(progs map[string]*ProgramSpec, names []string) { 180 // Pre-calculate all function references. 181 refs := make(map[*ProgramSpec][]string) 182 for _, prog := range progs { 183 refs[prog] = prog.Instructions.FunctionReferences() 184 } 185 186 // Create a flattened instruction stream, but don't modify progs yet to 187 // avoid linking multiple times. 188 flattened := make([]asm.Instructions, 0, len(names)) 189 for _, name := range names { 190 flattened = append(flattened, flattenInstructions(name, progs, refs)) 191 } 192 193 // Finally, assign the flattened instructions. 194 for i, name := range names { 195 progs[name].Instructions = flattened[i] 196 } 197 } 198 199 // flattenInstructions resolves bpf-to-bpf calls for a single program. 200 // 201 // Flattens the instructions of prog by concatenating the instructions of all 202 // direct and indirect dependencies. 203 // 204 // progs contains all referenceable programs, while refs contain the direct 205 // dependencies of each program. 206 func flattenInstructions(name string, progs map[string]*ProgramSpec, refs map[*ProgramSpec][]string) asm.Instructions { 207 prog := progs[name] 208 209 insns := make(asm.Instructions, len(prog.Instructions)) 210 copy(insns, prog.Instructions) 211 212 // Add all direct references of prog to the list of to be linked programs. 213 pending := make([]string, len(refs[prog])) 214 copy(pending, refs[prog]) 215 216 // All references for which we've appended instructions. 217 linked := make(map[string]bool) 218 219 // Iterate all pending references. We can't use a range since pending is 220 // modified in the body below. 221 for len(pending) > 0 { 222 var ref string 223 ref, pending = pending[0], pending[1:] 224 225 if linked[ref] { 226 // We've already linked this ref, don't append instructions again. 227 continue 228 } 229 230 progRef := progs[ref] 231 if progRef == nil { 232 // We don't have instructions that go with this reference. This 233 // happens when calling extern functions. 234 continue 235 } 236 237 insns = append(insns, progRef.Instructions...) 238 linked[ref] = true 239 240 // Make sure we link indirect references. 241 pending = append(pending, refs[progRef]...) 242 } 243 244 return insns 245 } 246 247 // fixupAndValidate is called by the ELF reader right before marshaling the 248 // instruction stream. It performs last-minute adjustments to the program and 249 // runs some sanity checks before sending it off to the kernel. 250 func fixupAndValidate(insns asm.Instructions) error { 251 iter := insns.Iterate() 252 for iter.Next() { 253 ins := iter.Ins 254 255 // Map load was tagged with a Reference, but does not contain a Map pointer. 256 needsMap := ins.Reference() != "" || ins.Metadata.Get(kconfigMetaKey{}) != nil 257 if ins.IsLoadFromMap() && needsMap && ins.Map() == nil { 258 return fmt.Errorf("instruction %d: %w", iter.Index, asm.ErrUnsatisfiedMapReference) 259 } 260 261 fixupProbeReadKernel(ins) 262 } 263 264 return nil 265 } 266 267 // POISON_CALL_KFUNC_BASE in libbpf. 268 // https://github.com/libbpf/libbpf/blob/2778cbce609aa1e2747a69349f7f46a2f94f0522/src/libbpf.c#L5767 269 const kfuncCallPoisonBase = 2002000000 270 271 // fixupKfuncs loops over all instructions in search for kfunc calls. 272 // If at least one is found, the current kernels BTF and module BTFis are searched to set Instruction.Constant 273 // and Instruction.Offset to the correct values. 274 func fixupKfuncs(insns asm.Instructions) (_ handles, err error) { 275 closeOnError := func(c io.Closer) { 276 if err != nil { 277 c.Close() 278 } 279 } 280 281 iter := insns.Iterate() 282 for iter.Next() { 283 ins := iter.Ins 284 if metadata := ins.Metadata.Get(kfuncMetaKey{}); metadata != nil { 285 goto fixups 286 } 287 } 288 289 return nil, nil 290 291 fixups: 292 // only load the kernel spec if we found at least one kfunc call 293 kernelSpec, err := btf.LoadKernelSpec() 294 if err != nil { 295 return nil, err 296 } 297 298 fdArray := make(handles, 0) 299 defer closeOnError(&fdArray) 300 301 for { 302 ins := iter.Ins 303 304 metadata := ins.Metadata.Get(kfuncMetaKey{}) 305 if metadata == nil { 306 if !iter.Next() { 307 // break loop if this was the last instruction in the stream. 308 break 309 } 310 continue 311 } 312 313 // check meta, if no meta return err 314 kfm, _ := metadata.(*kfuncMeta) 315 if kfm == nil { 316 return nil, fmt.Errorf("kfuncMetaKey doesn't contain kfuncMeta") 317 } 318 319 target := btf.Type((*btf.Func)(nil)) 320 spec, module, err := findTargetInKernel(kernelSpec, kfm.Func.Name, &target) 321 if kfm.Binding == elf.STB_WEAK && errors.Is(err, btf.ErrNotFound) { 322 if ins.IsKfuncCall() { 323 // If the kfunc call is weak and not found, poison the call. Use a recognizable constant 324 // to make it easier to debug. And set src to zero so the verifier doesn't complain 325 // about the invalid imm/offset values before dead-code elimination. 326 ins.Constant = kfuncCallPoisonBase 327 ins.Src = 0 328 } else if ins.OpCode.IsDWordLoad() { 329 // If the kfunc DWordLoad is weak and not found, set its address to 0. 330 ins.Constant = 0 331 ins.Src = 0 332 } else { 333 return nil, fmt.Errorf("only kfunc calls and dword loads may have kfunc metadata") 334 } 335 336 iter.Next() 337 continue 338 } 339 // Error on non-weak kfunc not found. 340 if errors.Is(err, btf.ErrNotFound) { 341 return nil, fmt.Errorf("kfunc %q: %w", kfm.Func.Name, ErrNotSupported) 342 } 343 if err != nil { 344 return nil, err 345 } 346 347 idx, err := fdArray.add(module) 348 if err != nil { 349 return nil, err 350 } 351 352 if err := btf.CheckTypeCompatibility(kfm.Func.Type, target.(*btf.Func).Type); err != nil { 353 return nil, &incompatibleKfuncError{kfm.Func.Name, err} 354 } 355 356 id, err := spec.TypeID(target) 357 if err != nil { 358 return nil, err 359 } 360 361 ins.Constant = int64(id) 362 ins.Offset = int16(idx) 363 364 if !iter.Next() { 365 break 366 } 367 } 368 369 return fdArray, nil 370 } 371 372 type incompatibleKfuncError struct { 373 name string 374 err error 375 } 376 377 func (ike *incompatibleKfuncError) Error() string { 378 return fmt.Sprintf("kfunc %q: %s", ike.name, ike.err) 379 } 380 381 // fixupProbeReadKernel replaces calls to bpf_probe_read_{kernel,user}(_str) 382 // with bpf_probe_read(_str) on kernels that don't support it yet. 383 func fixupProbeReadKernel(ins *asm.Instruction) { 384 if !ins.IsBuiltinCall() { 385 return 386 } 387 388 // Kernel supports bpf_probe_read_kernel, nothing to do. 389 if haveProbeReadKernel() == nil { 390 return 391 } 392 393 switch asm.BuiltinFunc(ins.Constant) { 394 case asm.FnProbeReadKernel, asm.FnProbeReadUser: 395 ins.Constant = int64(asm.FnProbeRead) 396 case asm.FnProbeReadKernelStr, asm.FnProbeReadUserStr: 397 ins.Constant = int64(asm.FnProbeReadStr) 398 } 399 } 400 401 // resolveKconfigReferences creates and populates a .kconfig map if necessary. 402 // 403 // Returns a nil Map and no error if no references exist. 404 func resolveKconfigReferences(insns asm.Instructions) (_ *Map, err error) { 405 closeOnError := func(c io.Closer) { 406 if err != nil { 407 c.Close() 408 } 409 } 410 411 var spec *MapSpec 412 iter := insns.Iterate() 413 for iter.Next() { 414 meta, _ := iter.Ins.Metadata.Get(kconfigMetaKey{}).(*kconfigMeta) 415 if meta != nil { 416 spec = meta.Map 417 break 418 } 419 } 420 421 if spec == nil { 422 return nil, nil 423 } 424 425 cpy := spec.Copy() 426 if err := resolveKconfig(cpy); err != nil { 427 return nil, err 428 } 429 430 kconfig, err := NewMap(cpy) 431 if err != nil { 432 return nil, err 433 } 434 defer closeOnError(kconfig) 435 436 // Resolve all instructions which load from .kconfig map with actual map 437 // and offset inside it. 438 iter = insns.Iterate() 439 for iter.Next() { 440 meta, _ := iter.Ins.Metadata.Get(kconfigMetaKey{}).(*kconfigMeta) 441 if meta == nil { 442 continue 443 } 444 445 if meta.Map != spec { 446 return nil, fmt.Errorf("instruction %d: reference to multiple .kconfig maps is not allowed", iter.Index) 447 } 448 449 if err := iter.Ins.AssociateMap(kconfig); err != nil { 450 return nil, fmt.Errorf("instruction %d: %w", iter.Index, err) 451 } 452 453 // Encode a map read at the offset of the var in the datasec. 454 iter.Ins.Constant = int64(uint64(meta.Offset) << 32) 455 iter.Ins.Metadata.Set(kconfigMetaKey{}, nil) 456 } 457 458 return kconfig, nil 459 }