github.com/SagerNet/gvisor@v0.0.0-20210707092255-7731c139d75c/test/packetimpact/runner/dut.go (about)

     1  // Copyright 2020 The gVisor Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  // Package runner starts docker containers and networking for a packetimpact test.
    16  package runner
    17  
    18  import (
    19  	"context"
    20  	"encoding/json"
    21  	"flag"
    22  	"fmt"
    23  	"io/ioutil"
    24  	"log"
    25  	"math/rand"
    26  	"net"
    27  	"os"
    28  	"os/exec"
    29  	"path"
    30  	"path/filepath"
    31  	"strings"
    32  	"testing"
    33  	"time"
    34  
    35  	"github.com/docker/docker/api/types/mount"
    36  	"github.com/SagerNet/gvisor/pkg/test/dockerutil"
    37  	"github.com/SagerNet/gvisor/test/packetimpact/netdevs"
    38  	"github.com/SagerNet/gvisor/test/packetimpact/testbench"
    39  )
    40  
    41  // stringList implements flag.Value.
    42  type stringList []string
    43  
    44  // String implements flag.Value.String.
    45  func (l *stringList) String() string {
    46  	return strings.Join(*l, ",")
    47  }
    48  
    49  // Set implements flag.Value.Set.
    50  func (l *stringList) Set(value string) error {
    51  	*l = append(*l, value)
    52  	return nil
    53  }
    54  
    55  var (
    56  	native          = false
    57  	testbenchBinary = ""
    58  	tshark          = false
    59  	extraTestArgs   = stringList{}
    60  	expectFailure   = false
    61  	numDUTs         = 1
    62  
    63  	// DUTAddr is the IP addres for DUT.
    64  	DUTAddr       = net.IPv4(0, 0, 0, 10)
    65  	testbenchAddr = net.IPv4(0, 0, 0, 20)
    66  )
    67  
    68  // RegisterFlags defines flags and associates them with the package-level
    69  // exported variables above. It should be called by tests in their init
    70  // functions.
    71  func RegisterFlags(fs *flag.FlagSet) {
    72  	fs.BoolVar(&native, "native", false, "whether the test should be run natively")
    73  	fs.StringVar(&testbenchBinary, "testbench_binary", "", "path to the testbench binary")
    74  	fs.BoolVar(&tshark, "tshark", false, "use more verbose tshark in logs instead of tcpdump")
    75  	fs.Var(&extraTestArgs, "extra_test_arg", "extra arguments to pass to the testbench")
    76  	fs.BoolVar(&expectFailure, "expect_failure", false, "expect that the test will fail when run")
    77  	fs.IntVar(&numDUTs, "num_duts", numDUTs, "the number of duts to create")
    78  }
    79  
    80  const (
    81  	// CtrlPort is the port that posix_server listens on.
    82  	CtrlPort uint16 = 40000
    83  	// testOutputDir is the directory in each container that holds test output.
    84  	testOutputDir = "/tmp/testoutput"
    85  )
    86  
    87  // logger implements testutil.Logger.
    88  //
    89  // Labels logs based on their source and formats multi-line logs.
    90  type logger string
    91  
    92  // Name implements testutil.Logger.Name.
    93  func (l logger) Name() string {
    94  	return string(l)
    95  }
    96  
    97  // Logf implements testutil.Logger.Logf.
    98  func (l logger) Logf(format string, args ...interface{}) {
    99  	lines := strings.Split(fmt.Sprintf(format, args...), "\n")
   100  	log.Printf("%s: %s", l, lines[0])
   101  	for _, line := range lines[1:] {
   102  		log.Printf("%*s  %s", len(l), "", line)
   103  	}
   104  }
   105  
   106  // dutInfo encapsulates all the essential information to set up testbench
   107  // container.
   108  type dutInfo struct {
   109  	dut              DUT
   110  	ctrlNet, testNet *dockerutil.Network
   111  	netInfo          *testbench.DUTTestNet
   112  	uname            *testbench.DUTUname
   113  }
   114  
   115  // setUpDUT will set up one DUT and return information for setting up the
   116  // container for testbench.
   117  func setUpDUT(ctx context.Context, t *testing.T, id int, mkDevice func(*dockerutil.Container) DUT) (dutInfo, error) {
   118  	// Create the networks needed for the test. One control network is needed
   119  	// for the gRPC control packets and one test network on which to transmit
   120  	// the test packets.
   121  	var info dutInfo
   122  	ctrlNet := dockerutil.NewNetwork(ctx, logger("ctrlNet"))
   123  	testNet := dockerutil.NewNetwork(ctx, logger("testNet"))
   124  	for _, dn := range []*dockerutil.Network{ctrlNet, testNet} {
   125  		for {
   126  			if err := createDockerNetwork(ctx, dn); err != nil {
   127  				t.Log("creating docker network:", err)
   128  				const wait = 100 * time.Millisecond
   129  				t.Logf("sleeping %s and will try creating docker network again", wait)
   130  				// This can fail if another docker network claimed the same IP so we
   131  				// will just try again.
   132  				time.Sleep(wait)
   133  				continue
   134  			}
   135  			break
   136  		}
   137  		dn := dn
   138  		t.Cleanup(func() {
   139  			if err := dn.Cleanup(ctx); err != nil {
   140  				t.Errorf("failed to cleanup network %s: %s", dn.Name, err)
   141  			}
   142  		})
   143  		// Sanity check.
   144  		if inspect, err := dn.Inspect(ctx); err != nil {
   145  			return dutInfo{}, fmt.Errorf("failed to inspect network %s: %w", dn.Name, err)
   146  		} else if inspect.Name != dn.Name {
   147  			return dutInfo{}, fmt.Errorf("name mismatch for network want: %s got: %s", dn.Name, inspect.Name)
   148  		}
   149  	}
   150  	info.ctrlNet = ctrlNet
   151  	info.testNet = testNet
   152  
   153  	// Create the Docker container for the DUT.
   154  	makeContainer := dockerutil.MakeContainer
   155  	if native {
   156  		makeContainer = dockerutil.MakeNativeContainer
   157  	}
   158  	dutContainer := makeContainer(ctx, logger(fmt.Sprintf("dut-%d", id)))
   159  	t.Cleanup(func() {
   160  		dutContainer.CleanUp(ctx)
   161  	})
   162  	info.dut = mkDevice(dutContainer)
   163  
   164  	runOpts := dockerutil.RunOpts{
   165  		Image:  "packetimpact",
   166  		CapAdd: []string{"NET_ADMIN"},
   167  	}
   168  	if _, err := MountTempDirectory(t, &runOpts, "dut-output", testOutputDir); err != nil {
   169  		return dutInfo{}, err
   170  	}
   171  
   172  	ipv4PrefixLength, _ := testNet.Subnet.Mask.Size()
   173  	remoteIPv6, remoteMAC, dutDeviceID, dutTestNetDev, err := info.dut.Prepare(ctx, t, runOpts, ctrlNet, testNet)
   174  	if err != nil {
   175  		return dutInfo{}, err
   176  	}
   177  	info.netInfo = &testbench.DUTTestNet{
   178  		RemoteMAC:        remoteMAC,
   179  		RemoteIPv4:       AddressInSubnet(DUTAddr, *testNet.Subnet),
   180  		RemoteIPv6:       remoteIPv6,
   181  		RemoteDevID:      dutDeviceID,
   182  		RemoteDevName:    dutTestNetDev,
   183  		LocalIPv4:        AddressInSubnet(testbenchAddr, *testNet.Subnet),
   184  		IPv4PrefixLength: ipv4PrefixLength,
   185  		POSIXServerIP:    AddressInSubnet(DUTAddr, *ctrlNet.Subnet),
   186  		POSIXServerPort:  CtrlPort,
   187  	}
   188  	info.uname, err = info.dut.Uname(ctx)
   189  	if err != nil {
   190  		return dutInfo{}, fmt.Errorf("failed to get uname information on DUT: %w", err)
   191  	}
   192  	return info, nil
   193  }
   194  
   195  // TestWithDUT runs a packetimpact test with the given information.
   196  func TestWithDUT(ctx context.Context, t *testing.T, mkDevice func(*dockerutil.Container) DUT) {
   197  	if testbenchBinary == "" {
   198  		t.Fatal("--testbench_binary is missing")
   199  	}
   200  	dockerutil.EnsureSupportedDockerVersion()
   201  
   202  	dutInfoChan := make(chan dutInfo, numDUTs)
   203  	errChan := make(chan error, numDUTs)
   204  	var dockerNetworks []*dockerutil.Network
   205  	var dutInfos []*testbench.DUTInfo
   206  	var duts []DUT
   207  
   208  	setUpCtx, cancelSetup := context.WithCancel(ctx)
   209  	t.Cleanup(cancelSetup)
   210  	for i := 0; i < numDUTs; i++ {
   211  		go func(i int) {
   212  			info, err := setUpDUT(setUpCtx, t, i, mkDevice)
   213  			if err != nil {
   214  				errChan <- err
   215  			} else {
   216  				dutInfoChan <- info
   217  			}
   218  		}(i)
   219  	}
   220  	for i := 0; i < numDUTs; i++ {
   221  		select {
   222  		case info := <-dutInfoChan:
   223  			dockerNetworks = append(dockerNetworks, info.ctrlNet, info.testNet)
   224  			dutInfos = append(dutInfos, &testbench.DUTInfo{
   225  				Net:   info.netInfo,
   226  				Uname: info.uname,
   227  			})
   228  			duts = append(duts, info.dut)
   229  		case err := <-errChan:
   230  			t.Fatal(err)
   231  		}
   232  	}
   233  
   234  	// Create the Docker container for the testbench.
   235  	testbenchContainer := dockerutil.MakeNativeContainer(ctx, logger("testbench"))
   236  	t.Cleanup(func() {
   237  		testbenchContainer.CleanUp(ctx)
   238  	})
   239  
   240  	runOpts := dockerutil.RunOpts{
   241  		Image:  "packetimpact",
   242  		CapAdd: []string{"NET_ADMIN"},
   243  	}
   244  	if _, err := MountTempDirectory(t, &runOpts, "testbench-output", testOutputDir); err != nil {
   245  		t.Fatal(err)
   246  	}
   247  	tbb := path.Base(testbenchBinary)
   248  	containerTestbenchBinary := filepath.Join("/packetimpact", tbb)
   249  	testbenchContainer.CopyFiles(&runOpts, "/packetimpact", filepath.Join("test/packetimpact/tests", tbb))
   250  
   251  	if err := StartContainer(
   252  		ctx,
   253  		runOpts,
   254  		testbenchContainer,
   255  		testbenchAddr,
   256  		dockerNetworks,
   257  		nil, /* sysctls */
   258  		"tail", "-f", "/dev/null",
   259  	); err != nil {
   260  		t.Fatalf("cannot start testbench container: %s", err)
   261  	}
   262  
   263  	for i := range dutInfos {
   264  		name, info, err := deviceByIP(ctx, testbenchContainer, dutInfos[i].Net.LocalIPv4)
   265  		if err != nil {
   266  			t.Fatalf("failed to get the device name associated with %s: %s", dutInfos[i].Net.LocalIPv4, err)
   267  		}
   268  		dutInfos[i].Net.LocalDevName = name
   269  		dutInfos[i].Net.LocalDevID = info.ID
   270  		dutInfos[i].Net.LocalMAC = info.MAC
   271  		localIPv6, err := getOrAssignIPv6Addr(ctx, testbenchContainer, name)
   272  		if err != nil {
   273  			t.Fatalf("failed to get IPV6 address on %s: %s", testbenchContainer.Name, err)
   274  		}
   275  		dutInfos[i].Net.LocalIPv6 = localIPv6
   276  	}
   277  	dutInfosBytes, err := json.Marshal(dutInfos)
   278  	if err != nil {
   279  		t.Fatalf("failed to marshal %v into json: %s", dutInfos, err)
   280  	}
   281  
   282  	baseSnifferArgs := []string{
   283  		"tcpdump",
   284  		"-vvv",
   285  		"--absolute-tcp-sequence-numbers",
   286  		"--packet-buffered",
   287  		// Disable DNS resolution.
   288  		"-n",
   289  		// run tcpdump as root since the output directory is owned by root. From
   290  		// `man tcpdump`:
   291  		//
   292  		// -Z user
   293  		// --relinquish-privileges=user
   294  		//        If tcpdump is running as root, after opening the capture device
   295  		//        or input savefile, change the user ID to user and the group ID to
   296  		//        the primary group of user.
   297  		// This behavior is enabled by default (-Z tcpdump), and can be
   298  		// disabled by -Z root.
   299  		"-Z", "root",
   300  	}
   301  	if tshark {
   302  		baseSnifferArgs = []string{
   303  			"tshark",
   304  			"-V",
   305  			"-o", "tcp.check_checksum:TRUE",
   306  			"-o", "udp.check_checksum:TRUE",
   307  			// Disable buffering.
   308  			"-l",
   309  			// Disable DNS resolution.
   310  			"-n",
   311  		}
   312  	}
   313  	for _, info := range dutInfos {
   314  		n := info.Net
   315  		snifferArgs := append(baseSnifferArgs, "-i", n.LocalDevName)
   316  		if !tshark {
   317  			snifferArgs = append(
   318  				snifferArgs,
   319  				"-w",
   320  				filepath.Join(testOutputDir, fmt.Sprintf("%s.pcap", n.LocalDevName)),
   321  			)
   322  		}
   323  		p, err := testbenchContainer.ExecProcess(ctx, dockerutil.ExecOpts{}, snifferArgs...)
   324  		if err != nil {
   325  			t.Fatalf("failed to start exec a sniffer on %s: %s", n.LocalDevName, err)
   326  		}
   327  		t.Cleanup(func() {
   328  			if snifferOut, err := p.Logs(); err != nil {
   329  				t.Errorf("sniffer logs failed: %s\n%s", err, snifferOut)
   330  			} else {
   331  				t.Logf("sniffer logs:\n%s", snifferOut)
   332  			}
   333  		})
   334  		// When the Linux kernel receives a SYN-ACK for a SYN it didn't send, it
   335  		// will respond with an RST. In most packetimpact tests, the SYN is sent
   336  		// by the raw socket, the kernel knows nothing about the connection, this
   337  		// behavior will break lots of TCP related packetimpact tests. To prevent
   338  		// this, we can install the following iptables rules. The raw socket that
   339  		// packetimpact tests use will still be able to see everything.
   340  		for _, bin := range []string{"iptables", "ip6tables"} {
   341  			if logs, err := testbenchContainer.Exec(ctx, dockerutil.ExecOpts{}, bin, "-A", "INPUT", "-i", n.LocalDevName, "-p", "tcp", "-j", "DROP"); err != nil {
   342  				t.Fatalf("unable to Exec %s on container %s: %s, logs from testbench:\n%s", bin, testbenchContainer.Name, err, logs)
   343  			}
   344  		}
   345  	}
   346  
   347  	t.Cleanup(func() {
   348  		// Wait 1 second before killing tcpdump to give it time to flush
   349  		// any packets. On linux tests killing it immediately can
   350  		// sometimes result in partial pcaps.
   351  		time.Sleep(1 * time.Second)
   352  		if logs, err := testbenchContainer.Exec(ctx, dockerutil.ExecOpts{}, "killall", baseSnifferArgs[0]); err != nil {
   353  			t.Errorf("failed to kill all sniffers: %s, logs: %s", err, logs)
   354  		}
   355  	})
   356  
   357  	// FIXME(b/156449515): Some piece of the system has a race. The old
   358  	// bash script version had a sleep, so we have one too. The race should
   359  	// be fixed and this sleep removed.
   360  	time.Sleep(time.Second)
   361  
   362  	// Start a packetimpact test on the test bench. The packetimpact test sends
   363  	// and receives packets and also sends POSIX socket commands to the
   364  	// posix_server to be executed on the DUT.
   365  	testArgs := []string{containerTestbenchBinary}
   366  	if testing.Verbose() {
   367  		testArgs = append(testArgs, "-test.v")
   368  	}
   369  	testArgs = append(testArgs, extraTestArgs...)
   370  	testArgs = append(testArgs,
   371  		fmt.Sprintf("--native=%t", native),
   372  		"--dut_infos_json", string(dutInfosBytes),
   373  	)
   374  	testbenchLogs, err := testbenchContainer.Exec(ctx, dockerutil.ExecOpts{}, testArgs...)
   375  	var dutLogs string
   376  	for i, dut := range duts {
   377  		logs, err := dut.Logs(ctx)
   378  		if err != nil {
   379  			logs = fmt.Sprintf("failed to fetch DUT logs: %s", err)
   380  		}
   381  		dutLogs = fmt.Sprintf(`%s====== Begin of DUT-%d Logs ======
   382  
   383  %s
   384  
   385  ====== End of DUT-%d Logs ======
   386  
   387  `, dutLogs, i, logs, i)
   388  	}
   389  	testLogs := fmt.Sprintf(`
   390  %s====== Begin of Testbench Logs ======
   391  
   392  %s
   393  
   394  ====== End of Testbench Logs ======`, dutLogs, testbenchLogs)
   395  	if (err != nil) != expectFailure {
   396  		t.Errorf(`test error: %v, expect failure: %t
   397  %s`, err, expectFailure, testLogs)
   398  	} else if expectFailure {
   399  		t.Logf(`test failed as expected: %v
   400  %s`, err, testLogs)
   401  	} else if testing.Verbose() {
   402  		t.Log(testLogs)
   403  	}
   404  }
   405  
   406  // DUT describes how to setup/teardown the dut for packetimpact tests.
   407  type DUT interface {
   408  	// Prepare prepares the dut, starts posix_server and returns the IPv6, MAC
   409  	// address, the interface ID, and the interface name for the testNet on DUT.
   410  	// The t parameter is supposed to be used for t.Cleanup. Don't use it for
   411  	// t.Fatal/FailNow functions.
   412  	Prepare(ctx context.Context, t *testing.T, runOpts dockerutil.RunOpts, ctrlNet, testNet *dockerutil.Network) (net.IP, net.HardwareAddr, uint32, string, error)
   413  
   414  	// Uname gathers information of DUT using command uname.
   415  	Uname(ctx context.Context) (*testbench.DUTUname, error)
   416  
   417  	// Logs retrieves the logs from the dut.
   418  	Logs(ctx context.Context) (string, error)
   419  }
   420  
   421  // DockerDUT describes a docker based DUT.
   422  type DockerDUT struct {
   423  	c *dockerutil.Container
   424  }
   425  
   426  // NewDockerDUT creates a docker based DUT.
   427  func NewDockerDUT(c *dockerutil.Container) DUT {
   428  	return &DockerDUT{
   429  		c: c,
   430  	}
   431  }
   432  
   433  // Prepare implements DUT.Prepare.
   434  func (dut *DockerDUT) Prepare(ctx context.Context, _ *testing.T, runOpts dockerutil.RunOpts, ctrlNet, testNet *dockerutil.Network) (net.IP, net.HardwareAddr, uint32, string, error) {
   435  	const containerPosixServerBinary = "/packetimpact/posix_server"
   436  	dut.c.CopyFiles(&runOpts, "/packetimpact", "test/packetimpact/dut/posix_server")
   437  
   438  	if err := StartContainer(
   439  		ctx,
   440  		runOpts,
   441  		dut.c,
   442  		DUTAddr,
   443  		[]*dockerutil.Network{ctrlNet, testNet},
   444  		map[string]string{
   445  			// This enables creating ICMP sockets on Linux.
   446  			"net.ipv4.ping_group_range": "0 0",
   447  		},
   448  		containerPosixServerBinary,
   449  		"--ip=0.0.0.0",
   450  		fmt.Sprintf("--port=%d", CtrlPort),
   451  	); err != nil {
   452  		return nil, nil, 0, "", fmt.Errorf("failed to start docker container for DUT: %w", err)
   453  	}
   454  
   455  	if _, err := dut.c.WaitForOutput(ctx, "Server listening.*\n", 60*time.Second); err != nil {
   456  		return nil, nil, 0, "", fmt.Errorf("%s on container %s never listened: %s", containerPosixServerBinary, dut.c.Name, err)
   457  	}
   458  
   459  	dutTestDevice, dutDeviceInfo, err := deviceByIP(ctx, dut.c, AddressInSubnet(DUTAddr, *testNet.Subnet))
   460  	if err != nil {
   461  		return nil, nil, 0, "", err
   462  	}
   463  
   464  	remoteIPv6, err := getOrAssignIPv6Addr(ctx, dut.c, dutTestDevice)
   465  	if err != nil {
   466  		return nil, nil, 0, "", fmt.Errorf("failed to get IPv6 address on %s: %s", dut.c.Name, err)
   467  	}
   468  	const testNetDev = "eth2"
   469  
   470  	return remoteIPv6, dutDeviceInfo.MAC, dutDeviceInfo.ID, testNetDev, nil
   471  }
   472  
   473  // Uname implements DUT.Uname.
   474  func (dut *DockerDUT) Uname(ctx context.Context) (*testbench.DUTUname, error) {
   475  	machine, err := dut.c.Exec(ctx, dockerutil.ExecOpts{}, "uname", "-m")
   476  	if err != nil {
   477  		return nil, err
   478  	}
   479  	kernelRelease, err := dut.c.Exec(ctx, dockerutil.ExecOpts{}, "uname", "-r")
   480  	if err != nil {
   481  		return nil, err
   482  	}
   483  	kernelVersion, err := dut.c.Exec(ctx, dockerutil.ExecOpts{}, "uname", "-v")
   484  	if err != nil {
   485  		return nil, err
   486  	}
   487  	kernelName, err := dut.c.Exec(ctx, dockerutil.ExecOpts{}, "uname", "-s")
   488  	if err != nil {
   489  		return nil, err
   490  	}
   491  	// TODO(github.com/SagerNet/issues/5586): -o is not supported on macOS.
   492  	operatingSystem, err := dut.c.Exec(ctx, dockerutil.ExecOpts{}, "uname", "-o")
   493  	if err != nil {
   494  		return nil, err
   495  	}
   496  	return &testbench.DUTUname{
   497  		Machine:         strings.TrimRight(machine, "\n"),
   498  		KernelName:      strings.TrimRight(kernelName, "\n"),
   499  		KernelRelease:   strings.TrimRight(kernelRelease, "\n"),
   500  		KernelVersion:   strings.TrimRight(kernelVersion, "\n"),
   501  		OperatingSystem: strings.TrimRight(operatingSystem, "\n"),
   502  	}, nil
   503  }
   504  
   505  // Logs implements DUT.Logs.
   506  func (dut *DockerDUT) Logs(ctx context.Context) (string, error) {
   507  	logs, err := dut.c.Logs(ctx)
   508  	if err != nil {
   509  		return "", err
   510  	}
   511  	return logs, nil
   512  }
   513  
   514  // AddNetworks connects docker network with the container and assigns the specific IP.
   515  func AddNetworks(ctx context.Context, d *dockerutil.Container, addr net.IP, networks []*dockerutil.Network) error {
   516  	for _, dn := range networks {
   517  		ip := AddressInSubnet(addr, *dn.Subnet)
   518  		// Connect to the network with the specified IP address.
   519  		if err := dn.Connect(ctx, d, ip.String(), ""); err != nil {
   520  			return fmt.Errorf("unable to connect container %s to network %s: %w", d.Name, dn.Name, err)
   521  		}
   522  	}
   523  	return nil
   524  }
   525  
   526  // AddressInSubnet combines the subnet provided with the address and returns a
   527  // new address. The return address bits come from the subnet where the mask is
   528  // 1 and from the ip address where the mask is 0.
   529  func AddressInSubnet(addr net.IP, subnet net.IPNet) net.IP {
   530  	var octets net.IP
   531  	for i := 0; i < 4; i++ {
   532  		octets = append(octets, (subnet.IP.To4()[i]&subnet.Mask[i])+(addr.To4()[i]&(^subnet.Mask[i])))
   533  	}
   534  	return octets
   535  }
   536  
   537  // devicesInfo will run "ip addr show" on the container and parse the output
   538  // to a map[string]netdevs.DeviceInfo.
   539  func devicesInfo(ctx context.Context, d *dockerutil.Container) (map[string]netdevs.DeviceInfo, error) {
   540  	out, err := d.Exec(ctx, dockerutil.ExecOpts{}, "ip", "addr", "show")
   541  	if err != nil {
   542  		return map[string]netdevs.DeviceInfo{}, fmt.Errorf("listing devices on %s container: %w\n%s", d.Name, err, out)
   543  	}
   544  	devs, err := netdevs.ParseDevices(out)
   545  	if err != nil {
   546  		return map[string]netdevs.DeviceInfo{}, fmt.Errorf("parsing devices from %s container: %w\n%s", d.Name, err, out)
   547  	}
   548  	return devs, nil
   549  }
   550  
   551  // deviceByIP finds a deviceInfo and device name from an IP address.
   552  func deviceByIP(ctx context.Context, d *dockerutil.Container, ip net.IP) (string, netdevs.DeviceInfo, error) {
   553  	devs, err := devicesInfo(ctx, d)
   554  	if err != nil {
   555  		return "", netdevs.DeviceInfo{}, err
   556  	}
   557  	testDevice, deviceInfo, err := netdevs.FindDeviceByIP(ip, devs)
   558  	if err != nil {
   559  		return "", netdevs.DeviceInfo{}, fmt.Errorf("can't find deviceInfo for container %s: %w", d.Name, err)
   560  	}
   561  	return testDevice, deviceInfo, nil
   562  }
   563  
   564  // getOrAssignIPv6Addr will try to get the IPv6 address for the interface; if an
   565  // address was not assigned, a link-local address based on MAC will be assigned
   566  // to that interface.
   567  func getOrAssignIPv6Addr(ctx context.Context, d *dockerutil.Container, iface string) (net.IP, error) {
   568  	devs, err := devicesInfo(ctx, d)
   569  	if err != nil {
   570  		return net.IP{}, err
   571  	}
   572  	info := devs[iface]
   573  	if info.IPv6Addr != nil {
   574  		return info.IPv6Addr, nil
   575  	}
   576  	if info.MAC == nil {
   577  		return nil, fmt.Errorf("unable to find MAC address of %s", iface)
   578  	}
   579  	if logs, err := d.Exec(ctx, dockerutil.ExecOpts{}, "ip", "addr", "add", netdevs.MACToIP(info.MAC).String(), "scope", "link", "dev", iface); err != nil {
   580  		return net.IP{}, fmt.Errorf("unable to ip addr add on container %s: %w, logs: %s", d.Name, err, logs)
   581  	}
   582  	// Now try again, to make sure that it worked.
   583  	devs, err = devicesInfo(ctx, d)
   584  	if err != nil {
   585  		return net.IP{}, err
   586  	}
   587  	info = devs[iface]
   588  	if info.IPv6Addr == nil {
   589  		return net.IP{}, fmt.Errorf("unable to set IPv6 address on container %s", d.Name)
   590  	}
   591  	return info.IPv6Addr, nil
   592  }
   593  
   594  // createDockerNetwork makes a randomly-named network that will start with the
   595  // namePrefix. The network will be a random /24 subnet.
   596  func createDockerNetwork(ctx context.Context, n *dockerutil.Network) error {
   597  	randSource := rand.NewSource(time.Now().UnixNano())
   598  	r1 := rand.New(randSource)
   599  	// Class C, 192.0.0.0 to 223.255.255.255, transitionally has mask 24.
   600  	ip := net.IPv4(byte(r1.Intn(224-192)+192), byte(r1.Intn(256)), byte(r1.Intn(256)), 0)
   601  	n.Subnet = &net.IPNet{
   602  		IP:   ip,
   603  		Mask: ip.DefaultMask(),
   604  	}
   605  	return n.Create(ctx)
   606  }
   607  
   608  // StartContainer will create a container instance from runOpts, connect it
   609  // with the specified docker networks and start executing the specified cmd.
   610  func StartContainer(ctx context.Context, runOpts dockerutil.RunOpts, c *dockerutil.Container, containerAddr net.IP, ns []*dockerutil.Network, sysctls map[string]string, cmd ...string) error {
   611  	conf, hostconf, netconf := c.ConfigsFrom(runOpts, cmd...)
   612  	_ = netconf
   613  	hostconf.Sysctls = map[string]string{"net.ipv6.conf.all.disable_ipv6": "0"}
   614  	for k, v := range sysctls {
   615  		hostconf.Sysctls[k] = v
   616  	}
   617  
   618  	if err := c.CreateFrom(ctx, runOpts.Image, conf, hostconf, nil); err != nil {
   619  		return fmt.Errorf("unable to create container %s: %w", c.Name, err)
   620  	}
   621  
   622  	if err := AddNetworks(ctx, c, containerAddr, ns); err != nil {
   623  		return fmt.Errorf("unable to connect the container with the networks: %w", err)
   624  	}
   625  
   626  	if err := c.Start(ctx); err != nil {
   627  		return fmt.Errorf("unable to start container %s: %w", c.Name, err)
   628  	}
   629  	return nil
   630  }
   631  
   632  // MountTempDirectory creates a temporary directory on host with the template
   633  // and then mounts it into the container under the name provided. The temporary
   634  // directory name is returned. Content in that directory will be copied to
   635  // TEST_UNDECLARED_OUTPUTS_DIR in cleanup phase.
   636  func MountTempDirectory(t *testing.T, runOpts *dockerutil.RunOpts, hostDirTemplate, containerDir string) (string, error) {
   637  	t.Helper()
   638  	tmpDir, err := ioutil.TempDir("", hostDirTemplate)
   639  	if err != nil {
   640  		return "", fmt.Errorf("failed to create a temp dir: %w", err)
   641  	}
   642  	t.Cleanup(func() {
   643  		if err := exec.Command("/bin/cp", "-r", tmpDir, os.Getenv("TEST_UNDECLARED_OUTPUTS_DIR")).Run(); err != nil {
   644  			t.Errorf("unable to copy container output files: %s", err)
   645  		}
   646  		if err := os.RemoveAll(tmpDir); err != nil {
   647  			t.Errorf("failed to remove tmpDir %s: %s", tmpDir, err)
   648  		}
   649  	})
   650  	runOpts.Mounts = append(runOpts.Mounts, mount.Mount{
   651  		Type:     mount.TypeBind,
   652  		Source:   tmpDir,
   653  		Target:   containerDir,
   654  		ReadOnly: false,
   655  	})
   656  	return tmpDir, nil
   657  }