github.com/cilium/cilium@v1.16.2/test/verifier/verifier_test.go (about)

     1  // SPDX-License-Identifier: Apache-2.0
     2  // Copyright Authors of Cilium
     3  
     4  package verifier_test
     5  
     6  import (
     7  	"bufio"
     8  	"bytes"
     9  	"errors"
    10  	"flag"
    11  	"fmt"
    12  	"io"
    13  	"math"
    14  	"os"
    15  	"os/exec"
    16  	"path"
    17  	"path/filepath"
    18  	"sort"
    19  	"strings"
    20  	"testing"
    21  
    22  	"github.com/cilium/ebpf"
    23  	"github.com/cilium/ebpf/rlimit"
    24  	"github.com/sirupsen/logrus"
    25  	"golang.org/x/sys/unix"
    26  
    27  	"github.com/cilium/cilium/pkg/bpf"
    28  	"github.com/cilium/cilium/pkg/datapath/linux/probes"
    29  	"github.com/cilium/cilium/pkg/logging"
    30  )
    31  
    32  var (
    33  	ciliumBasePath  = flag.String("cilium-base-path", "", "Cilium checkout base path")
    34  	ciKernelVersion = flag.String("ci-kernel-version", "", "CI kernel version to assume for verifier tests (supported values: 54, 510, 61, netnext)")
    35  )
    36  
    37  func getCIKernelVersion(t *testing.T) (string, string) {
    38  	t.Helper()
    39  
    40  	var uts unix.Utsname
    41  	if err := unix.Uname(&uts); err != nil {
    42  		t.Fatalf("uname: %v", err)
    43  	}
    44  	release := unix.ByteSliceToString(uts.Release[:])
    45  	t.Logf("Running kernel version: %s", release)
    46  
    47  	if ciKernelVersion != nil && *ciKernelVersion != "" {
    48  		return *ciKernelVersion, "cli"
    49  	}
    50  
    51  	var ciKernel string
    52  	switch {
    53  	case strings.HasPrefix(release, "5.4"):
    54  		ciKernel = "54"
    55  	case strings.HasPrefix(release, "5.10"):
    56  		ciKernel = "510"
    57  	case strings.HasPrefix(release, "6.1"):
    58  		ciKernel = "61"
    59  	case strings.HasPrefix(release, "bpf-next"):
    60  		ciKernel = "netnext"
    61  	default:
    62  		t.Fatalf("detected kernel version %s not supported by verifier complexity tests, specify using -ci-kernel-version", release)
    63  	}
    64  
    65  	return ciKernel, "detected"
    66  }
    67  
    68  func getDatapathConfigFiles(t *testing.T, ciKernelVersion, bpfProgram string) []string {
    69  	t.Helper()
    70  
    71  	pattern := filepath.Join("bpf", "complexity-tests", ciKernelVersion, bpfProgram, "*.txt")
    72  	files, err := filepath.Glob(filepath.Join(*ciliumBasePath, pattern))
    73  	if err != nil {
    74  		t.Fatal(err)
    75  	}
    76  
    77  	if len(files) == 0 {
    78  		t.Fatal("No files match", pattern)
    79  	}
    80  
    81  	return files
    82  }
    83  
    84  // readDatapathConfig turns each line in a reader into a single line with each
    85  // element separated by spaces.
    86  func readDatapathConfig(t *testing.T, r io.Reader) string {
    87  	scanner := bufio.NewScanner(r)
    88  	var lines []string
    89  	for scanner.Scan() {
    90  		lines = append(lines, strings.TrimSpace(scanner.Text()))
    91  	}
    92  
    93  	if err := scanner.Err(); err != nil {
    94  		t.Fatal(err)
    95  	}
    96  
    97  	return strings.Join(lines, " ")
    98  }
    99  
   100  // This test tries to compile BPF programs with a set of options that maximize
   101  // size & complexity (as defined in bpf/complexity-tests). Programs are then
   102  // loaded into the kernel to detect complexity & other verifier-related
   103  // regressions.
   104  func TestVerifier(t *testing.T) {
   105  	flag.Parse()
   106  
   107  	logging.DefaultLogger.SetLevel(logrus.DebugLevel)
   108  
   109  	if ciliumBasePath == nil || *ciliumBasePath == "" {
   110  		t.Skip("Please set -cilium-base-path to run verifier tests")
   111  	}
   112  	t.Logf("Cilium checkout base path: %s", *ciliumBasePath)
   113  
   114  	if err := rlimit.RemoveMemlock(); err != nil {
   115  		t.Fatal(err)
   116  	}
   117  
   118  	kernelVersion, source := getCIKernelVersion(t)
   119  	t.Logf("CI kernel version: %s (%s)", kernelVersion, source)
   120  
   121  	for _, bpfProgram := range []struct {
   122  		name      string
   123  		macroName string
   124  	}{
   125  		{
   126  			name:      "bpf_lxc",
   127  			macroName: "MAX_LXC_OPTIONS",
   128  		},
   129  		{
   130  			name:      "bpf_host",
   131  			macroName: "MAX_HOST_OPTIONS",
   132  		},
   133  		{
   134  			name:      "bpf_wireguard",
   135  			macroName: "MAX_WIREGUARD_OPTIONS",
   136  		},
   137  		{
   138  			name:      "bpf_xdp",
   139  			macroName: "MAX_XDP_OPTIONS",
   140  		},
   141  		{
   142  			name:      "bpf_overlay",
   143  			macroName: "MAX_OVERLAY_OPTIONS",
   144  		},
   145  		{
   146  			name:      "bpf_sock",
   147  			macroName: "MAX_LB_OPTIONS",
   148  		},
   149  		{
   150  			name:      "bpf_network",
   151  			macroName: "BPF_SIMPLE_OPTIONS",
   152  		},
   153  	} {
   154  		t.Run(bpfProgram.name, func(t *testing.T) {
   155  			initObjFile := path.Join(*ciliumBasePath, "bpf", fmt.Sprintf("%s.o", bpfProgram.name))
   156  
   157  			fileNames := getDatapathConfigFiles(t, kernelVersion, bpfProgram.name)
   158  			for _, fileName := range fileNames {
   159  				configName := strings.TrimSuffix(filepath.Base(fileName), filepath.Ext(fileName))
   160  				t.Run(configName, func(t *testing.T) {
   161  					file, err := os.Open(fileName)
   162  					if err != nil {
   163  						t.Fatalf("Unable to open configuration: %v", err)
   164  					}
   165  					defer file.Close()
   166  
   167  					datapathConfig := readDatapathConfig(t, file)
   168  
   169  					name := fmt.Sprintf("%s_%s", bpfProgram.name, configName)
   170  					cmd := exec.Command("make", "-C", "bpf", "clean", fmt.Sprintf("%s.o", bpfProgram.name))
   171  					cmd.Dir = *ciliumBasePath
   172  					cmd.Env = append(os.Environ(),
   173  						fmt.Sprintf("%s=%s", bpfProgram.macroName, datapathConfig),
   174  						fmt.Sprintf("KERNEL=%s", kernelVersion),
   175  					)
   176  					t.Logf("Compiling with %q", cmd.Args)
   177  					t.Logf("Env is %q", cmd.Env)
   178  					if out, err := cmd.CombinedOutput(); err != nil {
   179  						t.Logf("Command output:\n%s", string(out))
   180  						t.Fatalf("Failed to compile bpf objects: %v", err)
   181  					}
   182  
   183  					objFile := filepath.Join("./", name+".o")
   184  					// Rename object file to avoid subsequent runs to overwrite it,
   185  					// so we can keep it for CI's artifact upload.
   186  					if err = os.Rename(initObjFile, objFile); err != nil {
   187  						t.Fatalf("Failed to rename %s to %s: %v", initObjFile, objFile, err)
   188  					}
   189  
   190  					// Parse the compiled object into a CollectionSpec.
   191  					spec, err := bpf.LoadCollectionSpec(objFile)
   192  					if err != nil {
   193  						t.Fatal(err)
   194  					}
   195  
   196  					// Delete unsupported programs from the spec.
   197  					for n, p := range spec.Programs {
   198  						err := probes.HaveAttachType(p.Type, p.AttachType)
   199  						if errors.Is(err, ebpf.ErrNotSupported) {
   200  							t.Logf("%s: skipped unsupported program/attach type (%s/%s)", n, p.Type, p.AttachType)
   201  							delete(spec.Programs, n)
   202  							continue
   203  						}
   204  						if err != nil {
   205  							t.Fatal(err)
   206  						}
   207  					}
   208  
   209  					// Strip all pinning flags so we don't need to specify a pin path.
   210  					// This creates new maps for every Collection.
   211  					for _, m := range spec.Maps {
   212  						m.Pinning = ebpf.PinNone
   213  					}
   214  
   215  					coll, _, err := bpf.LoadCollection(spec, &bpf.CollectionOptions{
   216  						CollectionOptions: ebpf.CollectionOptions{
   217  							// Enable verifier logs for successful loads.
   218  							// Use log level 1 since it's known by all target kernels.
   219  							Programs: ebpf.ProgramOptions{
   220  								// Maximum log size for kernels <5.2. Some programs generate a
   221  								// verifier log of over 8MiB, so avoid retries due to the initial
   222  								// size being too small. This saves a lot of time as retrying means
   223  								// reloading all maps and progs in the collection.
   224  								LogSize:  (math.MaxUint32 >> 8), // 16MiB
   225  								LogLevel: ebpf.LogLevelBranch,
   226  							},
   227  						},
   228  					})
   229  					var ve *ebpf.VerifierError
   230  					if errors.As(err, &ve) {
   231  						// Write full verifier log to a path on disk for offline analysis.
   232  						var buf bytes.Buffer
   233  						fmt.Fprintf(&buf, "%+v", ve)
   234  						fullLogFile := name + "_verifier.log"
   235  						_ = os.WriteFile(fullLogFile, buf.Bytes(), 0444)
   236  						t.Log("Full verifier log at", fullLogFile)
   237  
   238  						// Print unverified instruction count.
   239  						t.Log("BPF unverified instruction count per program:")
   240  						for n, p := range spec.Programs {
   241  							t.Logf("\t%s: %d insns", n, len(p.Instructions))
   242  						}
   243  
   244  						// Include the original err in the output since it contains the name
   245  						// of the program that triggered the verifier error.
   246  						// ebpf.VerifierError only contains the return code and verifier log
   247  						// buffer.
   248  						t.Fatalf("Error: %v\nVerifier error tail: %-10v", err, ve)
   249  					}
   250  					if err != nil {
   251  						t.Fatal(err)
   252  					}
   253  					defer coll.Close()
   254  
   255  					// Print verifier stats appearing on the last line of the log, e.g.
   256  					// 'processed 12248 insns (limit 1000000) ...'.
   257  					// Sort by program names for stable output.
   258  					names := make([]string, 0, len(coll.Programs))
   259  					for n := range coll.Programs {
   260  						names = append(names, n)
   261  					}
   262  					sort.Strings(names)
   263  					for _, n := range names {
   264  						p := coll.Programs[n]
   265  						p.VerifierLog = strings.TrimRight(p.VerifierLog, "\n")
   266  						// Offset points at the last newline, increment by 1 to skip it.
   267  						// Turn a -1 into a 0 if there are no newlines in the log.
   268  						lastOff := strings.LastIndex(p.VerifierLog, "\n") + 1
   269  						t.Logf("%s: %v (%d unverified insns)", n, p.VerifierLog[lastOff:], len(spec.Programs[n].Instructions))
   270  					}
   271  				})
   272  			}
   273  		})
   274  	}
   275  }