github.com/cilium/cilium@v1.16.2/test/verifier/verifier_test.go (about) 1 // SPDX-License-Identifier: Apache-2.0 2 // Copyright Authors of Cilium 3 4 package verifier_test 5 6 import ( 7 "bufio" 8 "bytes" 9 "errors" 10 "flag" 11 "fmt" 12 "io" 13 "math" 14 "os" 15 "os/exec" 16 "path" 17 "path/filepath" 18 "sort" 19 "strings" 20 "testing" 21 22 "github.com/cilium/ebpf" 23 "github.com/cilium/ebpf/rlimit" 24 "github.com/sirupsen/logrus" 25 "golang.org/x/sys/unix" 26 27 "github.com/cilium/cilium/pkg/bpf" 28 "github.com/cilium/cilium/pkg/datapath/linux/probes" 29 "github.com/cilium/cilium/pkg/logging" 30 ) 31 32 var ( 33 ciliumBasePath = flag.String("cilium-base-path", "", "Cilium checkout base path") 34 ciKernelVersion = flag.String("ci-kernel-version", "", "CI kernel version to assume for verifier tests (supported values: 54, 510, 61, netnext)") 35 ) 36 37 func getCIKernelVersion(t *testing.T) (string, string) { 38 t.Helper() 39 40 var uts unix.Utsname 41 if err := unix.Uname(&uts); err != nil { 42 t.Fatalf("uname: %v", err) 43 } 44 release := unix.ByteSliceToString(uts.Release[:]) 45 t.Logf("Running kernel version: %s", release) 46 47 if ciKernelVersion != nil && *ciKernelVersion != "" { 48 return *ciKernelVersion, "cli" 49 } 50 51 var ciKernel string 52 switch { 53 case strings.HasPrefix(release, "5.4"): 54 ciKernel = "54" 55 case strings.HasPrefix(release, "5.10"): 56 ciKernel = "510" 57 case strings.HasPrefix(release, "6.1"): 58 ciKernel = "61" 59 case strings.HasPrefix(release, "bpf-next"): 60 ciKernel = "netnext" 61 default: 62 t.Fatalf("detected kernel version %s not supported by verifier complexity tests, specify using -ci-kernel-version", release) 63 } 64 65 return ciKernel, "detected" 66 } 67 68 func getDatapathConfigFiles(t *testing.T, ciKernelVersion, bpfProgram string) []string { 69 t.Helper() 70 71 pattern := filepath.Join("bpf", "complexity-tests", ciKernelVersion, bpfProgram, "*.txt") 72 files, err := filepath.Glob(filepath.Join(*ciliumBasePath, pattern)) 73 if err != nil { 74 t.Fatal(err) 75 } 76 77 if len(files) == 0 { 78 t.Fatal("No files match", pattern) 79 } 80 81 return files 82 } 83 84 // readDatapathConfig turns each line in a reader into a single line with each 85 // element separated by spaces. 86 func readDatapathConfig(t *testing.T, r io.Reader) string { 87 scanner := bufio.NewScanner(r) 88 var lines []string 89 for scanner.Scan() { 90 lines = append(lines, strings.TrimSpace(scanner.Text())) 91 } 92 93 if err := scanner.Err(); err != nil { 94 t.Fatal(err) 95 } 96 97 return strings.Join(lines, " ") 98 } 99 100 // This test tries to compile BPF programs with a set of options that maximize 101 // size & complexity (as defined in bpf/complexity-tests). Programs are then 102 // loaded into the kernel to detect complexity & other verifier-related 103 // regressions. 104 func TestVerifier(t *testing.T) { 105 flag.Parse() 106 107 logging.DefaultLogger.SetLevel(logrus.DebugLevel) 108 109 if ciliumBasePath == nil || *ciliumBasePath == "" { 110 t.Skip("Please set -cilium-base-path to run verifier tests") 111 } 112 t.Logf("Cilium checkout base path: %s", *ciliumBasePath) 113 114 if err := rlimit.RemoveMemlock(); err != nil { 115 t.Fatal(err) 116 } 117 118 kernelVersion, source := getCIKernelVersion(t) 119 t.Logf("CI kernel version: %s (%s)", kernelVersion, source) 120 121 for _, bpfProgram := range []struct { 122 name string 123 macroName string 124 }{ 125 { 126 name: "bpf_lxc", 127 macroName: "MAX_LXC_OPTIONS", 128 }, 129 { 130 name: "bpf_host", 131 macroName: "MAX_HOST_OPTIONS", 132 }, 133 { 134 name: "bpf_wireguard", 135 macroName: "MAX_WIREGUARD_OPTIONS", 136 }, 137 { 138 name: "bpf_xdp", 139 macroName: "MAX_XDP_OPTIONS", 140 }, 141 { 142 name: "bpf_overlay", 143 macroName: "MAX_OVERLAY_OPTIONS", 144 }, 145 { 146 name: "bpf_sock", 147 macroName: "MAX_LB_OPTIONS", 148 }, 149 { 150 name: "bpf_network", 151 macroName: "BPF_SIMPLE_OPTIONS", 152 }, 153 } { 154 t.Run(bpfProgram.name, func(t *testing.T) { 155 initObjFile := path.Join(*ciliumBasePath, "bpf", fmt.Sprintf("%s.o", bpfProgram.name)) 156 157 fileNames := getDatapathConfigFiles(t, kernelVersion, bpfProgram.name) 158 for _, fileName := range fileNames { 159 configName := strings.TrimSuffix(filepath.Base(fileName), filepath.Ext(fileName)) 160 t.Run(configName, func(t *testing.T) { 161 file, err := os.Open(fileName) 162 if err != nil { 163 t.Fatalf("Unable to open configuration: %v", err) 164 } 165 defer file.Close() 166 167 datapathConfig := readDatapathConfig(t, file) 168 169 name := fmt.Sprintf("%s_%s", bpfProgram.name, configName) 170 cmd := exec.Command("make", "-C", "bpf", "clean", fmt.Sprintf("%s.o", bpfProgram.name)) 171 cmd.Dir = *ciliumBasePath 172 cmd.Env = append(os.Environ(), 173 fmt.Sprintf("%s=%s", bpfProgram.macroName, datapathConfig), 174 fmt.Sprintf("KERNEL=%s", kernelVersion), 175 ) 176 t.Logf("Compiling with %q", cmd.Args) 177 t.Logf("Env is %q", cmd.Env) 178 if out, err := cmd.CombinedOutput(); err != nil { 179 t.Logf("Command output:\n%s", string(out)) 180 t.Fatalf("Failed to compile bpf objects: %v", err) 181 } 182 183 objFile := filepath.Join("./", name+".o") 184 // Rename object file to avoid subsequent runs to overwrite it, 185 // so we can keep it for CI's artifact upload. 186 if err = os.Rename(initObjFile, objFile); err != nil { 187 t.Fatalf("Failed to rename %s to %s: %v", initObjFile, objFile, err) 188 } 189 190 // Parse the compiled object into a CollectionSpec. 191 spec, err := bpf.LoadCollectionSpec(objFile) 192 if err != nil { 193 t.Fatal(err) 194 } 195 196 // Delete unsupported programs from the spec. 197 for n, p := range spec.Programs { 198 err := probes.HaveAttachType(p.Type, p.AttachType) 199 if errors.Is(err, ebpf.ErrNotSupported) { 200 t.Logf("%s: skipped unsupported program/attach type (%s/%s)", n, p.Type, p.AttachType) 201 delete(spec.Programs, n) 202 continue 203 } 204 if err != nil { 205 t.Fatal(err) 206 } 207 } 208 209 // Strip all pinning flags so we don't need to specify a pin path. 210 // This creates new maps for every Collection. 211 for _, m := range spec.Maps { 212 m.Pinning = ebpf.PinNone 213 } 214 215 coll, _, err := bpf.LoadCollection(spec, &bpf.CollectionOptions{ 216 CollectionOptions: ebpf.CollectionOptions{ 217 // Enable verifier logs for successful loads. 218 // Use log level 1 since it's known by all target kernels. 219 Programs: ebpf.ProgramOptions{ 220 // Maximum log size for kernels <5.2. Some programs generate a 221 // verifier log of over 8MiB, so avoid retries due to the initial 222 // size being too small. This saves a lot of time as retrying means 223 // reloading all maps and progs in the collection. 224 LogSize: (math.MaxUint32 >> 8), // 16MiB 225 LogLevel: ebpf.LogLevelBranch, 226 }, 227 }, 228 }) 229 var ve *ebpf.VerifierError 230 if errors.As(err, &ve) { 231 // Write full verifier log to a path on disk for offline analysis. 232 var buf bytes.Buffer 233 fmt.Fprintf(&buf, "%+v", ve) 234 fullLogFile := name + "_verifier.log" 235 _ = os.WriteFile(fullLogFile, buf.Bytes(), 0444) 236 t.Log("Full verifier log at", fullLogFile) 237 238 // Print unverified instruction count. 239 t.Log("BPF unverified instruction count per program:") 240 for n, p := range spec.Programs { 241 t.Logf("\t%s: %d insns", n, len(p.Instructions)) 242 } 243 244 // Include the original err in the output since it contains the name 245 // of the program that triggered the verifier error. 246 // ebpf.VerifierError only contains the return code and verifier log 247 // buffer. 248 t.Fatalf("Error: %v\nVerifier error tail: %-10v", err, ve) 249 } 250 if err != nil { 251 t.Fatal(err) 252 } 253 defer coll.Close() 254 255 // Print verifier stats appearing on the last line of the log, e.g. 256 // 'processed 12248 insns (limit 1000000) ...'. 257 // Sort by program names for stable output. 258 names := make([]string, 0, len(coll.Programs)) 259 for n := range coll.Programs { 260 names = append(names, n) 261 } 262 sort.Strings(names) 263 for _, n := range names { 264 p := coll.Programs[n] 265 p.VerifierLog = strings.TrimRight(p.VerifierLog, "\n") 266 // Offset points at the last newline, increment by 1 to skip it. 267 // Turn a -1 into a 0 if there are no newlines in the log. 268 lastOff := strings.LastIndex(p.VerifierLog, "\n") + 1 269 t.Logf("%s: %v (%d unverified insns)", n, p.VerifierLog[lastOff:], len(spec.Programs[n].Instructions)) 270 } 271 }) 272 } 273 }) 274 } 275 }