github.com/lalkh/containerd@v1.4.3/container_linux_test.go (about)

     1  // +build linux
     2  
     3  /*
     4     Copyright The containerd Authors.
     5  
     6     Licensed under the Apache License, Version 2.0 (the "License");
     7     you may not use this file except in compliance with the License.
     8     You may obtain a copy of the License at
     9  
    10         http://www.apache.org/licenses/LICENSE-2.0
    11  
    12     Unless required by applicable law or agreed to in writing, software
    13     distributed under the License is distributed on an "AS IS" BASIS,
    14     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    15     See the License for the specific language governing permissions and
    16     limitations under the License.
    17  */
    18  
    19  package containerd
    20  
    21  import (
    22  	"bytes"
    23  	"context"
    24  	"fmt"
    25  	"io"
    26  	"io/ioutil"
    27  	"os"
    28  	"os/exec"
    29  	"path/filepath"
    30  	"runtime"
    31  	"strings"
    32  	"sync"
    33  	"syscall"
    34  	"testing"
    35  	"time"
    36  
    37  	"github.com/containerd/cgroups"
    38  	cgroupsv2 "github.com/containerd/cgroups/v2"
    39  	"github.com/containerd/containerd/cio"
    40  	"github.com/containerd/containerd/containers"
    41  	"github.com/containerd/containerd/errdefs"
    42  	"github.com/containerd/containerd/oci"
    43  	"github.com/containerd/containerd/plugin"
    44  	"github.com/containerd/containerd/runtime/linux/runctypes"
    45  	"github.com/containerd/containerd/runtime/v2/runc/options"
    46  	"github.com/containerd/containerd/sys"
    47  	specs "github.com/opencontainers/runtime-spec/specs-go"
    48  	"golang.org/x/sys/unix"
    49  )
    50  
    51  func TestTaskUpdate(t *testing.T) {
    52  	t.Parallel()
    53  
    54  	client, err := newClient(t, address)
    55  	if err != nil {
    56  		t.Fatal(err)
    57  	}
    58  	defer client.Close()
    59  
    60  	var (
    61  		ctx, cancel = testContext(t)
    62  		id          = t.Name()
    63  	)
    64  	defer cancel()
    65  
    66  	image, err := client.GetImage(ctx, testImage)
    67  	if err != nil {
    68  		t.Fatal(err)
    69  	}
    70  	limit := int64(32 * 1024 * 1024)
    71  	memory := func(_ context.Context, _ oci.Client, _ *containers.Container, s *specs.Spec) error {
    72  		s.Linux.Resources.Memory = &specs.LinuxMemory{
    73  			Limit: &limit,
    74  		}
    75  		return nil
    76  	}
    77  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image),
    78  		WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30"), memory))
    79  	if err != nil {
    80  		t.Fatal(err)
    81  	}
    82  	defer container.Delete(ctx, WithSnapshotCleanup)
    83  
    84  	task, err := container.NewTask(ctx, empty())
    85  	if err != nil {
    86  		t.Fatal(err)
    87  	}
    88  	defer task.Delete(ctx)
    89  
    90  	statusC, err := task.Wait(ctx)
    91  	if err != nil {
    92  		t.Fatal(err)
    93  	}
    94  
    95  	var (
    96  		cgroup  cgroups.Cgroup
    97  		cgroup2 *cgroupsv2.Manager
    98  	)
    99  	// check that the task has a limit of 32mb
   100  	if cgroups.Mode() == cgroups.Unified {
   101  		groupPath, err := cgroupsv2.PidGroupPath(int(task.Pid()))
   102  		if err != nil {
   103  			t.Fatal(err)
   104  		}
   105  		cgroup2, err = cgroupsv2.LoadManager("/sys/fs/cgroup", groupPath)
   106  		if err != nil {
   107  			t.Fatal(err)
   108  		}
   109  		stat, err := cgroup2.Stat()
   110  		if err != nil {
   111  			t.Fatal(err)
   112  		}
   113  		if int64(stat.Memory.UsageLimit) != limit {
   114  			t.Fatalf("expected memory limit to be set to %d but received %d", limit, stat.Memory.UsageLimit)
   115  		}
   116  	} else {
   117  		cgroup, err = cgroups.Load(cgroups.V1, cgroups.PidPath(int(task.Pid())))
   118  		if err != nil {
   119  			t.Fatal(err)
   120  		}
   121  		stat, err := cgroup.Stat(cgroups.IgnoreNotExist)
   122  		if err != nil {
   123  			t.Fatal(err)
   124  		}
   125  		if int64(stat.Memory.Usage.Limit) != limit {
   126  			t.Fatalf("expected memory limit to be set to %d but received %d", limit, stat.Memory.Usage.Limit)
   127  		}
   128  	}
   129  	limit = 64 * 1024 * 1024
   130  	if err := task.Update(ctx, WithResources(&specs.LinuxResources{
   131  		Memory: &specs.LinuxMemory{
   132  			Limit: &limit,
   133  		},
   134  	})); err != nil {
   135  		t.Error(err)
   136  	}
   137  	// check that the task has a limit of 64mb
   138  	if cgroups.Mode() == cgroups.Unified {
   139  		stat, err := cgroup2.Stat()
   140  		if err != nil {
   141  			t.Fatal(err)
   142  		}
   143  		if int64(stat.Memory.UsageLimit) != limit {
   144  			t.Errorf("expected memory limit to be set to %d but received %d", limit, stat.Memory.UsageLimit)
   145  		}
   146  	} else {
   147  		stat, err := cgroup.Stat(cgroups.IgnoreNotExist)
   148  		if err != nil {
   149  			t.Fatal(err)
   150  		}
   151  		if int64(stat.Memory.Usage.Limit) != limit {
   152  			t.Errorf("expected memory limit to be set to %d but received %d", limit, stat.Memory.Usage.Limit)
   153  		}
   154  	}
   155  	if err := task.Kill(ctx, unix.SIGKILL); err != nil {
   156  		t.Fatal(err)
   157  	}
   158  
   159  	<-statusC
   160  }
   161  
   162  func TestShimInCgroup(t *testing.T) {
   163  	t.Parallel()
   164  
   165  	client, err := newClient(t, address)
   166  	if err != nil {
   167  		t.Fatal(err)
   168  	}
   169  	defer client.Close()
   170  	var (
   171  		ctx, cancel = testContext(t)
   172  		id          = t.Name()
   173  	)
   174  	defer cancel()
   175  
   176  	image, err := client.GetImage(ctx, testImage)
   177  	if err != nil {
   178  		t.Fatal(err)
   179  	}
   180  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), oci.WithProcessArgs("sleep", "30")))
   181  	if err != nil {
   182  		t.Fatal(err)
   183  	}
   184  	defer container.Delete(ctx, WithSnapshotCleanup)
   185  	// create a cgroup for the shim to use
   186  	path := "/containerd/shim"
   187  	var (
   188  		cg  cgroups.Cgroup
   189  		cg2 *cgroupsv2.Manager
   190  	)
   191  	if cgroups.Mode() == cgroups.Unified {
   192  		cg2, err = cgroupsv2.NewManager("/sys/fs/cgroup", path, &cgroupsv2.Resources{})
   193  		if err != nil {
   194  			t.Fatal(err)
   195  		}
   196  		defer cg2.Delete()
   197  	} else {
   198  		cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(path), &specs.LinuxResources{})
   199  		if err != nil {
   200  			t.Fatal(err)
   201  		}
   202  		defer cg.Delete()
   203  	}
   204  
   205  	task, err := container.NewTask(ctx, empty(), WithShimCgroup(path))
   206  	if err != nil {
   207  		t.Fatal(err)
   208  	}
   209  	defer task.Delete(ctx)
   210  
   211  	statusC, err := task.Wait(ctx)
   212  	if err != nil {
   213  		t.Fatal(err)
   214  	}
   215  
   216  	// check to see if the shim is inside the cgroup
   217  	if cgroups.Mode() == cgroups.Unified {
   218  		processes, err := cg2.Procs(false)
   219  		if err != nil {
   220  			t.Fatal(err)
   221  		}
   222  		if len(processes) == 0 {
   223  			t.Errorf("created cgroup should have at least one process inside: %d", len(processes))
   224  		}
   225  	} else {
   226  		processes, err := cg.Processes(cgroups.Devices, false)
   227  		if err != nil {
   228  			t.Fatal(err)
   229  		}
   230  		if len(processes) == 0 {
   231  			t.Errorf("created cgroup should have at least one process inside: %d", len(processes))
   232  		}
   233  	}
   234  	if err := task.Kill(ctx, unix.SIGKILL); err != nil {
   235  		t.Fatal(err)
   236  	}
   237  
   238  	<-statusC
   239  }
   240  
   241  func TestDaemonRestart(t *testing.T) {
   242  	client, err := newClient(t, address)
   243  	if err != nil {
   244  		t.Fatal(err)
   245  	}
   246  	defer client.Close()
   247  
   248  	var (
   249  		image       Image
   250  		ctx, cancel = testContext(t)
   251  		id          = t.Name()
   252  	)
   253  	defer cancel()
   254  
   255  	image, err = client.GetImage(ctx, testImage)
   256  	if err != nil {
   257  		t.Fatal(err)
   258  	}
   259  
   260  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30")))
   261  	if err != nil {
   262  		t.Fatal(err)
   263  	}
   264  	defer container.Delete(ctx, WithSnapshotCleanup)
   265  
   266  	task, err := container.NewTask(ctx, empty())
   267  	if err != nil {
   268  		t.Fatal(err)
   269  	}
   270  	defer task.Delete(ctx)
   271  
   272  	statusC, err := task.Wait(ctx)
   273  	if err != nil {
   274  		t.Fatal(err)
   275  	}
   276  
   277  	if err := task.Start(ctx); err != nil {
   278  		t.Fatal(err)
   279  	}
   280  
   281  	var exitStatus ExitStatus
   282  	if err := ctrd.Restart(func() {
   283  		exitStatus = <-statusC
   284  	}); err != nil {
   285  		t.Fatal(err)
   286  	}
   287  
   288  	if exitStatus.Error() == nil {
   289  		t.Errorf(`first task.Wait() should have failed with "transport is closing"`)
   290  	}
   291  
   292  	waitCtx, waitCancel := context.WithTimeout(ctx, 2*time.Second)
   293  	serving, err := client.IsServing(waitCtx)
   294  	waitCancel()
   295  	if !serving {
   296  		t.Fatalf("containerd did not start within 2s: %v", err)
   297  	}
   298  
   299  	statusC, err = task.Wait(ctx)
   300  	if err != nil {
   301  		t.Fatal(err)
   302  	}
   303  
   304  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
   305  		t.Fatal(err)
   306  	}
   307  
   308  	<-statusC
   309  }
   310  
   311  func TestShimDoesNotLeakPipes(t *testing.T) {
   312  	containerdPid := ctrd.cmd.Process.Pid
   313  	initialPipes, err := numPipes(containerdPid)
   314  	if err != nil {
   315  		t.Fatal(err)
   316  	}
   317  
   318  	client, err := newClient(t, address)
   319  	if err != nil {
   320  		t.Fatal(err)
   321  	}
   322  	defer client.Close()
   323  
   324  	var (
   325  		image       Image
   326  		ctx, cancel = testContext(t)
   327  		id          = t.Name()
   328  	)
   329  	defer cancel()
   330  
   331  	image, err = client.GetImage(ctx, testImage)
   332  	if err != nil {
   333  		t.Fatal(err)
   334  	}
   335  
   336  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30")))
   337  	if err != nil {
   338  		t.Fatal(err)
   339  	}
   340  
   341  	task, err := container.NewTask(ctx, empty())
   342  	if err != nil {
   343  		t.Fatal(err)
   344  	}
   345  
   346  	exitChannel, err := task.Wait(ctx)
   347  	if err != nil {
   348  		t.Fatal(err)
   349  	}
   350  
   351  	if err := task.Start(ctx); err != nil {
   352  		t.Fatal(err)
   353  	}
   354  
   355  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
   356  		t.Fatal(err)
   357  	}
   358  
   359  	<-exitChannel
   360  
   361  	if _, err := task.Delete(ctx); err != nil {
   362  		t.Fatal(err)
   363  	}
   364  
   365  	if err := container.Delete(ctx, WithSnapshotCleanup); err != nil {
   366  		t.Fatal(err)
   367  	}
   368  
   369  	currentPipes, err := numPipes(containerdPid)
   370  	if err != nil {
   371  		t.Fatal(err)
   372  	}
   373  
   374  	if initialPipes != currentPipes {
   375  		t.Errorf("Pipes have leaked after container has been deleted. Initially there were %d pipes, after container deletion there were %d pipes", initialPipes, currentPipes)
   376  	}
   377  }
   378  
   379  func numPipes(pid int) (int, error) {
   380  	cmd := exec.Command("sh", "-c", fmt.Sprintf("lsof -p %d | grep FIFO", pid))
   381  
   382  	var stdout bytes.Buffer
   383  	cmd.Stdout = &stdout
   384  	if err := cmd.Run(); err != nil {
   385  		return 0, err
   386  	}
   387  	return strings.Count(stdout.String(), "\n"), nil
   388  }
   389  
   390  func TestDaemonReconnectsToShimIOPipesOnRestart(t *testing.T) {
   391  	client, err := newClient(t, address)
   392  	if err != nil {
   393  		t.Fatal(err)
   394  	}
   395  	defer client.Close()
   396  
   397  	var (
   398  		image       Image
   399  		ctx, cancel = testContext(t)
   400  		id          = t.Name()
   401  	)
   402  	defer cancel()
   403  
   404  	image, err = client.GetImage(ctx, testImage)
   405  	if err != nil {
   406  		t.Fatal(err)
   407  	}
   408  
   409  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30")))
   410  	if err != nil {
   411  		t.Fatal(err)
   412  	}
   413  	defer container.Delete(ctx, WithSnapshotCleanup)
   414  
   415  	task, err := container.NewTask(ctx, empty())
   416  	if err != nil {
   417  		t.Fatal(err)
   418  	}
   419  	defer task.Delete(ctx)
   420  
   421  	_, err = task.Wait(ctx)
   422  	if err != nil {
   423  		t.Fatal(err)
   424  	}
   425  
   426  	if err := task.Start(ctx); err != nil {
   427  		t.Fatal(err)
   428  	}
   429  
   430  	if err := ctrd.Restart(nil); err != nil {
   431  		t.Fatal(err)
   432  	}
   433  
   434  	waitCtx, waitCancel := context.WithTimeout(ctx, 2*time.Second)
   435  	serving, err := client.IsServing(waitCtx)
   436  	waitCancel()
   437  	if !serving {
   438  		t.Fatalf("containerd did not start within 2s: %v", err)
   439  	}
   440  
   441  	// After we restared containerd we write some messages to the log pipes, simulating shim writing stuff there.
   442  	// Then we make sure that these messages are available on the containerd log thus proving that the server reconnected to the log pipes
   443  	runtimeVersion := getRuntimeVersion()
   444  	logDirPath := getLogDirPath(runtimeVersion, id)
   445  
   446  	switch runtimeVersion {
   447  	case "v1":
   448  		writeToFile(t, filepath.Join(logDirPath, "shim.stdout.log"), fmt.Sprintf("%s writing to stdout\n", id))
   449  		writeToFile(t, filepath.Join(logDirPath, "shim.stderr.log"), fmt.Sprintf("%s writing to stderr\n", id))
   450  	case "v2":
   451  		writeToFile(t, filepath.Join(logDirPath, "log"), fmt.Sprintf("%s writing to log\n", id))
   452  	}
   453  
   454  	statusC, err := task.Wait(ctx)
   455  	if err != nil {
   456  		t.Fatal(err)
   457  	}
   458  
   459  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
   460  		t.Fatal(err)
   461  	}
   462  
   463  	<-statusC
   464  
   465  	stdioContents, err := ioutil.ReadFile(ctrdStdioFilePath)
   466  	if err != nil {
   467  		t.Fatal(err)
   468  	}
   469  
   470  	switch runtimeVersion {
   471  	case "v1":
   472  		if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to stdout", id)) {
   473  			t.Fatal("containerd did not connect to the shim stdout pipe")
   474  		}
   475  		if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to stderr", id)) {
   476  			t.Fatal("containerd did not connect to the shim stderr pipe")
   477  		}
   478  	case "v2":
   479  		if !strings.Contains(string(stdioContents), fmt.Sprintf("%s writing to log", id)) {
   480  			t.Fatal("containerd did not connect to the shim log pipe")
   481  		}
   482  	}
   483  }
   484  
   485  func writeToFile(t *testing.T, filePath, message string) {
   486  	writer, err := os.OpenFile(filePath, os.O_WRONLY, 0600)
   487  	if err != nil {
   488  		t.Fatal(err)
   489  	}
   490  	if _, err := writer.WriteString(message); err != nil {
   491  		t.Fatal(err)
   492  	}
   493  	if err := writer.Close(); err != nil {
   494  		t.Fatal(err)
   495  	}
   496  }
   497  
   498  func getLogDirPath(runtimeVersion, id string) string {
   499  	switch runtimeVersion {
   500  	case "v1":
   501  		return filepath.Join(defaultRoot, plugin.RuntimeLinuxV1, testNamespace, id)
   502  	case "v2":
   503  		return filepath.Join(defaultState, "io.containerd.runtime.v2.task", testNamespace, id)
   504  	default:
   505  		panic(fmt.Errorf("Unsupported runtime version %s", runtimeVersion))
   506  	}
   507  }
   508  
   509  func getRuntimeVersion() string {
   510  	switch rt := os.Getenv("TEST_RUNTIME"); rt {
   511  	case plugin.RuntimeLinuxV1:
   512  		return "v1"
   513  	default:
   514  		return "v2"
   515  	}
   516  }
   517  
   518  func TestContainerPTY(t *testing.T) {
   519  	t.Parallel()
   520  
   521  	client, err := newClient(t, address)
   522  	if err != nil {
   523  		t.Fatal(err)
   524  	}
   525  	defer client.Close()
   526  
   527  	var (
   528  		image       Image
   529  		ctx, cancel = testContext(t)
   530  		id          = t.Name()
   531  	)
   532  	defer cancel()
   533  
   534  	image, err = client.GetImage(ctx, testImage)
   535  	if err != nil {
   536  		t.Fatal(err)
   537  	}
   538  
   539  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), oci.WithTTY, withProcessArgs("echo", "hello")))
   540  	if err != nil {
   541  		t.Fatal(err)
   542  	}
   543  	defer container.Delete(ctx, WithSnapshotCleanup)
   544  
   545  	direct, err := newDirectIO(ctx, true)
   546  	if err != nil {
   547  		t.Fatal(err)
   548  	}
   549  	defer direct.Delete()
   550  	var (
   551  		wg  sync.WaitGroup
   552  		buf = bytes.NewBuffer(nil)
   553  	)
   554  	wg.Add(1)
   555  	go func() {
   556  		defer wg.Done()
   557  		io.Copy(buf, direct.Stdout)
   558  	}()
   559  
   560  	task, err := container.NewTask(ctx, direct.IOCreate)
   561  	if err != nil {
   562  		t.Fatal(err)
   563  	}
   564  	defer task.Delete(ctx)
   565  
   566  	status, err := task.Wait(ctx)
   567  	if err != nil {
   568  		t.Error(err)
   569  	}
   570  
   571  	if err := task.Start(ctx); err != nil {
   572  		t.Fatal(err)
   573  	}
   574  
   575  	<-status
   576  	wg.Wait()
   577  
   578  	if err := direct.Close(); err != nil {
   579  		t.Error(err)
   580  	}
   581  
   582  	out := buf.String()
   583  	if !strings.ContainsAny(fmt.Sprintf("%#q", out), `\x00`) {
   584  		t.Fatal(`expected \x00 in output`)
   585  	}
   586  }
   587  
   588  func TestContainerAttach(t *testing.T) {
   589  	t.Parallel()
   590  
   591  	if runtime.GOOS == "windows" {
   592  		// On windows, closing the write side of the pipe closes the read
   593  		// side, sending an EOF to it and preventing reopening it.
   594  		// Hence this test will always fails on windows
   595  		t.Skip("invalid logic on windows")
   596  	}
   597  
   598  	client, err := newClient(t, address)
   599  	if err != nil {
   600  		t.Fatal(err)
   601  	}
   602  	defer client.Close()
   603  
   604  	var (
   605  		image       Image
   606  		ctx, cancel = testContext(t)
   607  		id          = t.Name()
   608  	)
   609  	defer cancel()
   610  
   611  	image, err = client.GetImage(ctx, testImage)
   612  	if err != nil {
   613  		t.Fatal(err)
   614  	}
   615  
   616  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withCat()))
   617  	if err != nil {
   618  		t.Fatal(err)
   619  	}
   620  	defer container.Delete(ctx, WithSnapshotCleanup)
   621  
   622  	expected := "hello" + newLine
   623  
   624  	direct, err := newDirectIO(ctx, false)
   625  	if err != nil {
   626  		t.Fatal(err)
   627  	}
   628  	defer direct.Delete()
   629  	var (
   630  		wg  sync.WaitGroup
   631  		buf = bytes.NewBuffer(nil)
   632  	)
   633  	wg.Add(1)
   634  	go func() {
   635  		defer wg.Done()
   636  		io.Copy(buf, direct.Stdout)
   637  	}()
   638  
   639  	task, err := container.NewTask(ctx, direct.IOCreate)
   640  	if err != nil {
   641  		t.Fatal(err)
   642  	}
   643  	defer task.Delete(ctx)
   644  
   645  	status, err := task.Wait(ctx)
   646  	if err != nil {
   647  		t.Error(err)
   648  	}
   649  
   650  	if err := task.Start(ctx); err != nil {
   651  		t.Fatal(err)
   652  	}
   653  
   654  	if _, err := fmt.Fprint(direct.Stdin, expected); err != nil {
   655  		t.Error(err)
   656  	}
   657  
   658  	// load the container and re-load the task
   659  	if container, err = client.LoadContainer(ctx, id); err != nil {
   660  		t.Fatal(err)
   661  	}
   662  
   663  	if task, err = container.Task(ctx, direct.IOAttach); err != nil {
   664  		t.Fatal(err)
   665  	}
   666  
   667  	if _, err := fmt.Fprint(direct.Stdin, expected); err != nil {
   668  		t.Error(err)
   669  	}
   670  
   671  	direct.Stdin.Close()
   672  
   673  	if err := task.CloseIO(ctx, WithStdinCloser); err != nil {
   674  		t.Error(err)
   675  	}
   676  
   677  	<-status
   678  
   679  	wg.Wait()
   680  	if _, err := task.Delete(ctx); err != nil {
   681  		t.Error(err)
   682  	}
   683  
   684  	output := buf.String()
   685  
   686  	// we wrote the same thing after attach
   687  	expected = expected + expected
   688  	if output != expected {
   689  		t.Errorf("expected output %q but received %q", expected, output)
   690  	}
   691  }
   692  
   693  func newDirectIO(ctx context.Context, terminal bool) (*directIO, error) {
   694  	fifos, err := cio.NewFIFOSetInDir("", "", terminal)
   695  	if err != nil {
   696  		return nil, err
   697  	}
   698  	dio, err := cio.NewDirectIO(ctx, fifos)
   699  	if err != nil {
   700  		return nil, err
   701  	}
   702  	return &directIO{DirectIO: *dio}, nil
   703  }
   704  
   705  type directIO struct {
   706  	cio.DirectIO
   707  }
   708  
   709  // ioCreate returns IO available for use with task creation
   710  func (f *directIO) IOCreate(id string) (cio.IO, error) {
   711  	return f, nil
   712  }
   713  
   714  // ioAttach returns IO available for use with task attachment
   715  func (f *directIO) IOAttach(set *cio.FIFOSet) (cio.IO, error) {
   716  	return f, nil
   717  }
   718  
   719  func (f *directIO) Cancel() {
   720  	// nothing to cancel as all operations are handled externally
   721  }
   722  
   723  // Close closes all open fds
   724  func (f *directIO) Close() error {
   725  	err := f.Stdin.Close()
   726  	if f.Stdout != nil {
   727  		if err2 := f.Stdout.Close(); err == nil {
   728  			err = err2
   729  		}
   730  	}
   731  	if f.Stderr != nil {
   732  		if err2 := f.Stderr.Close(); err == nil {
   733  			err = err2
   734  		}
   735  	}
   736  	return err
   737  }
   738  
   739  // Delete removes the underlying directory containing fifos
   740  func (f *directIO) Delete() error {
   741  	return f.DirectIO.Close()
   742  }
   743  
   744  func TestContainerUsername(t *testing.T) {
   745  	t.Parallel()
   746  
   747  	client, err := newClient(t, address)
   748  	if err != nil {
   749  		t.Fatal(err)
   750  	}
   751  	defer client.Close()
   752  
   753  	var (
   754  		image       Image
   755  		ctx, cancel = testContext(t)
   756  		id          = t.Name()
   757  	)
   758  	defer cancel()
   759  
   760  	image, err = client.GetImage(ctx, testImage)
   761  	if err != nil {
   762  		t.Fatal(err)
   763  	}
   764  	direct, err := newDirectIO(ctx, false)
   765  	if err != nil {
   766  		t.Fatal(err)
   767  	}
   768  	defer direct.Delete()
   769  	var (
   770  		wg  sync.WaitGroup
   771  		buf = bytes.NewBuffer(nil)
   772  	)
   773  	wg.Add(1)
   774  	go func() {
   775  		defer wg.Done()
   776  		io.Copy(buf, direct.Stdout)
   777  	}()
   778  
   779  	// squid user in the alpine image has a uid of 31
   780  	container, err := client.NewContainer(ctx, id,
   781  		WithNewSnapshot(id, image),
   782  		WithNewSpec(oci.WithImageConfig(image), oci.WithUsername("squid"), oci.WithProcessArgs("id", "-u")),
   783  	)
   784  	if err != nil {
   785  		t.Fatal(err)
   786  	}
   787  	defer container.Delete(ctx, WithSnapshotCleanup)
   788  
   789  	task, err := container.NewTask(ctx, direct.IOCreate)
   790  	if err != nil {
   791  		t.Fatal(err)
   792  	}
   793  	defer task.Delete(ctx)
   794  
   795  	statusC, err := task.Wait(ctx)
   796  	if err != nil {
   797  		t.Fatal(err)
   798  	}
   799  
   800  	if err := task.Start(ctx); err != nil {
   801  		t.Fatal(err)
   802  	}
   803  	<-statusC
   804  
   805  	wg.Wait()
   806  
   807  	output := strings.TrimSuffix(buf.String(), "\n")
   808  	if output != "31" {
   809  		t.Errorf("expected squid uid to be 31 but received %q", output)
   810  	}
   811  }
   812  
   813  func TestContainerUser(t *testing.T) {
   814  	t.Parallel()
   815  	t.Run("UserNameAndGroupName", func(t *testing.T) { testContainerUser(t, "squid:squid", "31:31") })
   816  	t.Run("UserIDAndGroupName", func(t *testing.T) { testContainerUser(t, "1001:squid", "1001:31") })
   817  	t.Run("UserNameAndGroupID", func(t *testing.T) { testContainerUser(t, "squid:1002", "31:1002") })
   818  	t.Run("UserIDAndGroupID", func(t *testing.T) { testContainerUser(t, "1001:1002", "1001:1002") })
   819  }
   820  
   821  func testContainerUser(t *testing.T, userstr, expectedOutput string) {
   822  	client, err := newClient(t, address)
   823  	if err != nil {
   824  		t.Fatal(err)
   825  	}
   826  	defer client.Close()
   827  
   828  	var (
   829  		image       Image
   830  		ctx, cancel = testContext(t)
   831  		id          = strings.Replace(t.Name(), "/", "_", -1)
   832  	)
   833  	defer cancel()
   834  
   835  	image, err = client.GetImage(ctx, testImage)
   836  	if err != nil {
   837  		t.Fatal(err)
   838  	}
   839  	direct, err := newDirectIO(ctx, false)
   840  	if err != nil {
   841  		t.Fatal(err)
   842  	}
   843  	defer direct.Delete()
   844  	var (
   845  		wg  sync.WaitGroup
   846  		buf = bytes.NewBuffer(nil)
   847  	)
   848  	wg.Add(1)
   849  	go func() {
   850  		defer wg.Done()
   851  		io.Copy(buf, direct.Stdout)
   852  	}()
   853  
   854  	container, err := client.NewContainer(ctx, id,
   855  		WithNewSnapshot(id, image),
   856  		WithNewSpec(oci.WithImageConfig(image), oci.WithUser(userstr), oci.WithProcessArgs("sh", "-c", "echo $(id -u):$(id -g)")),
   857  	)
   858  	if err != nil {
   859  		t.Fatal(err)
   860  	}
   861  	defer container.Delete(ctx, WithSnapshotCleanup)
   862  
   863  	task, err := container.NewTask(ctx, direct.IOCreate)
   864  	if err != nil {
   865  		t.Fatal(err)
   866  	}
   867  	defer task.Delete(ctx)
   868  
   869  	statusC, err := task.Wait(ctx)
   870  	if err != nil {
   871  		t.Fatal(err)
   872  	}
   873  
   874  	if err := task.Start(ctx); err != nil {
   875  		t.Fatal(err)
   876  	}
   877  	<-statusC
   878  
   879  	wg.Wait()
   880  
   881  	output := strings.TrimSuffix(buf.String(), "\n")
   882  	if output != expectedOutput {
   883  		t.Errorf("expected uid:gid to be %q, but received %q", expectedOutput, output)
   884  	}
   885  }
   886  
   887  func TestContainerAttachProcess(t *testing.T) {
   888  	t.Parallel()
   889  
   890  	if runtime.GOOS == "windows" {
   891  		// On windows, closing the write side of the pipe closes the read
   892  		// side, sending an EOF to it and preventing reopening it.
   893  		// Hence this test will always fails on windows
   894  		t.Skip("invalid logic on windows")
   895  	}
   896  
   897  	client, err := newClient(t, address)
   898  	if err != nil {
   899  		t.Fatal(err)
   900  	}
   901  	defer client.Close()
   902  
   903  	var (
   904  		image       Image
   905  		ctx, cancel = testContext(t)
   906  		id          = t.Name()
   907  	)
   908  	defer cancel()
   909  
   910  	image, err = client.GetImage(ctx, testImage)
   911  	if err != nil {
   912  		t.Fatal(err)
   913  	}
   914  
   915  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "100")))
   916  	if err != nil {
   917  		t.Fatal(err)
   918  	}
   919  	defer container.Delete(ctx, WithSnapshotCleanup)
   920  
   921  	expected := "hello" + newLine
   922  
   923  	// creating IO early for easy resource cleanup
   924  	direct, err := newDirectIO(ctx, false)
   925  	if err != nil {
   926  		t.Fatal(err)
   927  	}
   928  	defer direct.Delete()
   929  	var (
   930  		wg  sync.WaitGroup
   931  		buf = bytes.NewBuffer(nil)
   932  	)
   933  	wg.Add(1)
   934  	go func() {
   935  		defer wg.Done()
   936  		io.Copy(buf, direct.Stdout)
   937  	}()
   938  
   939  	task, err := container.NewTask(ctx, empty())
   940  	if err != nil {
   941  		t.Fatal(err)
   942  	}
   943  	defer task.Delete(ctx)
   944  
   945  	status, err := task.Wait(ctx)
   946  	if err != nil {
   947  		t.Error(err)
   948  	}
   949  
   950  	if err := task.Start(ctx); err != nil {
   951  		t.Fatal(err)
   952  	}
   953  
   954  	spec, err := container.Spec(ctx)
   955  	if err != nil {
   956  		t.Fatal(err)
   957  	}
   958  
   959  	processSpec := spec.Process
   960  	processSpec.Args = []string{"cat"}
   961  	execID := t.Name() + "_exec"
   962  	process, err := task.Exec(ctx, execID, processSpec, direct.IOCreate)
   963  	if err != nil {
   964  		t.Fatal(err)
   965  	}
   966  	processStatusC, err := process.Wait(ctx)
   967  	if err != nil {
   968  		t.Fatal(err)
   969  	}
   970  
   971  	if err := process.Start(ctx); err != nil {
   972  		t.Fatal(err)
   973  	}
   974  
   975  	if _, err := fmt.Fprint(direct.Stdin, expected); err != nil {
   976  		t.Error(err)
   977  	}
   978  
   979  	if process, err = task.LoadProcess(ctx, execID, direct.IOAttach); err != nil {
   980  		t.Fatal(err)
   981  	}
   982  
   983  	if _, err := fmt.Fprint(direct.Stdin, expected); err != nil {
   984  		t.Error(err)
   985  	}
   986  
   987  	direct.Stdin.Close()
   988  
   989  	if err := process.CloseIO(ctx, WithStdinCloser); err != nil {
   990  		t.Error(err)
   991  	}
   992  
   993  	<-processStatusC
   994  
   995  	wg.Wait()
   996  
   997  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
   998  		t.Error(err)
   999  	}
  1000  
  1001  	output := buf.String()
  1002  
  1003  	// we wrote the same thing after attach
  1004  	expected = expected + expected
  1005  	if output != expected {
  1006  		t.Errorf("expected output %q but received %q", expected, output)
  1007  	}
  1008  	<-status
  1009  }
  1010  
  1011  func TestContainerLoadUnexistingProcess(t *testing.T) {
  1012  	t.Parallel()
  1013  
  1014  	if runtime.GOOS == "windows" {
  1015  		// On windows, closing the write side of the pipe closes the read
  1016  		// side, sending an EOF to it and preventing reopening it.
  1017  		// Hence this test will always fails on windows
  1018  		t.Skip("invalid logic on windows")
  1019  	}
  1020  
  1021  	client, err := newClient(t, address)
  1022  	if err != nil {
  1023  		t.Fatal(err)
  1024  	}
  1025  	defer client.Close()
  1026  
  1027  	var (
  1028  		image       Image
  1029  		ctx, cancel = testContext(t)
  1030  		id          = t.Name()
  1031  	)
  1032  	defer cancel()
  1033  
  1034  	image, err = client.GetImage(ctx, testImage)
  1035  	if err != nil {
  1036  		t.Fatal(err)
  1037  	}
  1038  
  1039  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "100")))
  1040  	if err != nil {
  1041  		t.Fatal(err)
  1042  	}
  1043  	defer container.Delete(ctx, WithSnapshotCleanup)
  1044  
  1045  	// creating IO early for easy resource cleanup
  1046  	direct, err := newDirectIO(ctx, false)
  1047  	if err != nil {
  1048  		t.Fatal(err)
  1049  	}
  1050  	defer direct.Delete()
  1051  
  1052  	task, err := container.NewTask(ctx, empty())
  1053  	if err != nil {
  1054  		t.Fatal(err)
  1055  	}
  1056  	defer task.Delete(ctx)
  1057  
  1058  	status, err := task.Wait(ctx)
  1059  	if err != nil {
  1060  		t.Error(err)
  1061  	}
  1062  
  1063  	if err := task.Start(ctx); err != nil {
  1064  		t.Fatal(err)
  1065  	}
  1066  
  1067  	if _, err = task.LoadProcess(ctx, "this-process-does-not-exist", direct.IOAttach); err == nil {
  1068  		t.Fatal("an error should have occurred when loading a process that does not exist")
  1069  	}
  1070  
  1071  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
  1072  		t.Error(err)
  1073  	}
  1074  
  1075  	<-status
  1076  }
  1077  
  1078  func TestContainerUserID(t *testing.T) {
  1079  	t.Parallel()
  1080  
  1081  	client, err := newClient(t, address)
  1082  	if err != nil {
  1083  		t.Fatal(err)
  1084  	}
  1085  	defer client.Close()
  1086  
  1087  	var (
  1088  		image       Image
  1089  		ctx, cancel = testContext(t)
  1090  		id          = t.Name()
  1091  	)
  1092  	defer cancel()
  1093  
  1094  	image, err = client.GetImage(ctx, testImage)
  1095  	if err != nil {
  1096  		t.Fatal(err)
  1097  	}
  1098  	direct, err := newDirectIO(ctx, false)
  1099  	if err != nil {
  1100  		t.Fatal(err)
  1101  	}
  1102  	defer direct.Delete()
  1103  	var (
  1104  		wg  sync.WaitGroup
  1105  		buf = bytes.NewBuffer(nil)
  1106  	)
  1107  	wg.Add(1)
  1108  	go func() {
  1109  		defer wg.Done()
  1110  		io.Copy(buf, direct.Stdout)
  1111  	}()
  1112  
  1113  	// adm user in the alpine image has a uid of 3 and gid of 4.
  1114  	container, err := client.NewContainer(ctx, id,
  1115  		WithNewSnapshot(id, image),
  1116  		WithNewSpec(oci.WithImageConfig(image), oci.WithUserID(3), oci.WithProcessArgs("sh", "-c", "echo $(id -u):$(id -g)")),
  1117  	)
  1118  	if err != nil {
  1119  		t.Fatal(err)
  1120  	}
  1121  	defer container.Delete(ctx, WithSnapshotCleanup)
  1122  
  1123  	task, err := container.NewTask(ctx, direct.IOCreate)
  1124  	if err != nil {
  1125  		t.Fatal(err)
  1126  	}
  1127  	defer task.Delete(ctx)
  1128  
  1129  	statusC, err := task.Wait(ctx)
  1130  	if err != nil {
  1131  		t.Fatal(err)
  1132  	}
  1133  
  1134  	if err := task.Start(ctx); err != nil {
  1135  		t.Fatal(err)
  1136  	}
  1137  	<-statusC
  1138  
  1139  	wg.Wait()
  1140  
  1141  	output := strings.TrimSuffix(buf.String(), "\n")
  1142  	if output != "3:4" {
  1143  		t.Errorf("expected uid:gid to be 3:4, but received %q", output)
  1144  	}
  1145  }
  1146  
  1147  func TestContainerKillAll(t *testing.T) {
  1148  	t.Parallel()
  1149  
  1150  	client, err := newClient(t, address)
  1151  	if err != nil {
  1152  		t.Fatal(err)
  1153  	}
  1154  	defer client.Close()
  1155  
  1156  	var (
  1157  		image       Image
  1158  		ctx, cancel = testContext(t)
  1159  		id          = t.Name()
  1160  	)
  1161  	defer cancel()
  1162  
  1163  	image, err = client.GetImage(ctx, testImage)
  1164  	if err != nil {
  1165  		t.Fatal(err)
  1166  	}
  1167  
  1168  	container, err := client.NewContainer(ctx, id,
  1169  		WithNewSnapshot(id, image),
  1170  		WithNewSpec(oci.WithImageConfig(image),
  1171  			withProcessArgs("sh", "-c", "top"),
  1172  			oci.WithHostNamespace(specs.PIDNamespace),
  1173  		),
  1174  	)
  1175  	if err != nil {
  1176  		t.Fatal(err)
  1177  	}
  1178  	defer container.Delete(ctx, WithSnapshotCleanup)
  1179  
  1180  	stdout := bytes.NewBuffer(nil)
  1181  	task, err := container.NewTask(ctx, cio.NewCreator(withByteBuffers(stdout)))
  1182  	if err != nil {
  1183  		t.Fatal(err)
  1184  	}
  1185  	defer task.Delete(ctx)
  1186  
  1187  	statusC, err := task.Wait(ctx)
  1188  	if err != nil {
  1189  		t.Fatal(err)
  1190  	}
  1191  
  1192  	if err := task.Start(ctx); err != nil {
  1193  		t.Fatal(err)
  1194  	}
  1195  
  1196  	if err := task.Kill(ctx, syscall.SIGKILL, WithKillAll); err != nil {
  1197  		t.Error(err)
  1198  	}
  1199  
  1200  	<-statusC
  1201  	if _, err := task.Delete(ctx); err != nil {
  1202  		t.Fatal(err)
  1203  	}
  1204  }
  1205  
  1206  func TestDaemonRestartWithRunningShim(t *testing.T) {
  1207  	client, err := newClient(t, address)
  1208  	if err != nil {
  1209  		t.Fatal(err)
  1210  	}
  1211  	defer client.Close()
  1212  
  1213  	var (
  1214  		image       Image
  1215  		ctx, cancel = testContext(t)
  1216  		id          = t.Name()
  1217  	)
  1218  	defer cancel()
  1219  
  1220  	image, err = client.GetImage(ctx, testImage)
  1221  	if err != nil {
  1222  		t.Fatal(err)
  1223  	}
  1224  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), oci.WithProcessArgs("sleep", "100")))
  1225  	if err != nil {
  1226  		t.Fatal(err)
  1227  	}
  1228  	defer container.Delete(ctx, WithSnapshotCleanup)
  1229  
  1230  	task, err := container.NewTask(ctx, empty())
  1231  	if err != nil {
  1232  		t.Fatal(err)
  1233  	}
  1234  	defer task.Delete(ctx)
  1235  
  1236  	statusC, err := task.Wait(ctx)
  1237  	if err != nil {
  1238  		t.Error(err)
  1239  	}
  1240  
  1241  	pid := task.Pid()
  1242  	if pid < 1 {
  1243  		t.Fatalf("invalid task pid %d", pid)
  1244  	}
  1245  
  1246  	if err := task.Start(ctx); err != nil {
  1247  		t.Fatal(err)
  1248  	}
  1249  
  1250  	var exitStatus ExitStatus
  1251  	if err := ctrd.Restart(func() {
  1252  		exitStatus = <-statusC
  1253  	}); err != nil {
  1254  		t.Fatal(err)
  1255  	}
  1256  
  1257  	if exitStatus.Error() == nil {
  1258  		t.Errorf(`first task.Wait() should have failed with "transport is closing"`)
  1259  	}
  1260  
  1261  	waitCtx, cancel := context.WithTimeout(ctx, 1*time.Second)
  1262  	c, err := ctrd.waitForStart(waitCtx)
  1263  	cancel()
  1264  	if err != nil {
  1265  		t.Fatal(err)
  1266  	}
  1267  	c.Close()
  1268  
  1269  	statusC, err = task.Wait(ctx)
  1270  	if err != nil {
  1271  		t.Error(err)
  1272  	}
  1273  
  1274  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
  1275  		t.Fatal(err)
  1276  	}
  1277  
  1278  	<-statusC
  1279  
  1280  	if err := unix.Kill(int(pid), 0); err != unix.ESRCH {
  1281  		t.Errorf("pid %d still exists", pid)
  1282  	}
  1283  }
  1284  
  1285  func TestContainerRuntimeOptionsv1(t *testing.T) {
  1286  	t.Parallel()
  1287  
  1288  	client, err := newClient(t, address)
  1289  	if err != nil {
  1290  		t.Fatal(err)
  1291  	}
  1292  	defer client.Close()
  1293  
  1294  	var (
  1295  		image       Image
  1296  		ctx, cancel = testContext(t)
  1297  		id          = t.Name()
  1298  	)
  1299  	defer cancel()
  1300  
  1301  	image, err = client.GetImage(ctx, testImage)
  1302  	if err != nil {
  1303  		t.Fatal(err)
  1304  	}
  1305  
  1306  	container, err := client.NewContainer(
  1307  		ctx, id,
  1308  		WithNewSnapshot(id, image),
  1309  		WithNewSpec(oci.WithImageConfig(image), withExitStatus(7)),
  1310  		WithRuntime(plugin.RuntimeLinuxV1, &runctypes.RuncOptions{Runtime: "no-runc"}),
  1311  	)
  1312  	if err != nil {
  1313  		t.Fatal(err)
  1314  	}
  1315  	defer container.Delete(ctx, WithSnapshotCleanup)
  1316  
  1317  	task, err := container.NewTask(ctx, empty())
  1318  	if err == nil {
  1319  		t.Errorf("task creation should have failed")
  1320  		task.Delete(ctx)
  1321  		return
  1322  	}
  1323  	if !strings.Contains(err.Error(), `"no-runc"`) {
  1324  		t.Errorf("task creation should have failed because of lack of executable. Instead failed with: %v", err.Error())
  1325  	}
  1326  }
  1327  
  1328  func TestContainerRuntimeOptionsv2(t *testing.T) {
  1329  	t.Parallel()
  1330  
  1331  	client, err := newClient(t, address)
  1332  	if err != nil {
  1333  		t.Fatal(err)
  1334  	}
  1335  	defer client.Close()
  1336  
  1337  	var (
  1338  		image       Image
  1339  		ctx, cancel = testContext(t)
  1340  		id          = t.Name()
  1341  	)
  1342  	defer cancel()
  1343  
  1344  	image, err = client.GetImage(ctx, testImage)
  1345  	if err != nil {
  1346  		t.Fatal(err)
  1347  	}
  1348  
  1349  	container, err := client.NewContainer(
  1350  		ctx, id,
  1351  		WithNewSnapshot(id, image),
  1352  		WithNewSpec(oci.WithImageConfig(image), withExitStatus(7)),
  1353  		WithRuntime(plugin.RuntimeRuncV1, &options.Options{BinaryName: "no-runc"}),
  1354  	)
  1355  	if err != nil {
  1356  		t.Fatal(err)
  1357  	}
  1358  	defer container.Delete(ctx, WithSnapshotCleanup)
  1359  
  1360  	task, err := container.NewTask(ctx, empty())
  1361  	if err == nil {
  1362  		t.Errorf("task creation should have failed")
  1363  		task.Delete(ctx)
  1364  		return
  1365  	}
  1366  	if !strings.Contains(err.Error(), `"no-runc"`) {
  1367  		t.Errorf("task creation should have failed because of lack of executable. Instead failed with: %v", err.Error())
  1368  	}
  1369  }
  1370  
  1371  func initContainerAndCheckChildrenDieOnKill(t *testing.T, opts ...oci.SpecOpts) {
  1372  	client, err := newClient(t, address)
  1373  	if err != nil {
  1374  		t.Fatal(err)
  1375  	}
  1376  	defer client.Close()
  1377  
  1378  	var (
  1379  		image       Image
  1380  		ctx, cancel = testContext(t)
  1381  		id          = t.Name()
  1382  	)
  1383  	defer cancel()
  1384  
  1385  	image, err = client.GetImage(ctx, testImage)
  1386  	if err != nil {
  1387  		t.Fatal(err)
  1388  	}
  1389  
  1390  	opts = append(opts, oci.WithImageConfig(image))
  1391  	opts = append(opts, withProcessArgs("sh", "-c", "sleep 42; echo hi"))
  1392  
  1393  	container, err := client.NewContainer(ctx, id,
  1394  		WithNewSnapshot(id, image),
  1395  		WithNewSpec(opts...),
  1396  	)
  1397  	if err != nil {
  1398  		t.Fatal(err)
  1399  	}
  1400  	defer container.Delete(ctx, WithSnapshotCleanup)
  1401  
  1402  	stdout := bytes.NewBuffer(nil)
  1403  	task, err := container.NewTask(ctx, cio.NewCreator(withByteBuffers(stdout)))
  1404  	if err != nil {
  1405  		t.Fatal(err)
  1406  	}
  1407  	defer task.Delete(ctx)
  1408  
  1409  	statusC, err := task.Wait(ctx)
  1410  	if err != nil {
  1411  		t.Fatal(err)
  1412  	}
  1413  
  1414  	if err := task.Start(ctx); err != nil {
  1415  		t.Fatal(err)
  1416  	}
  1417  
  1418  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
  1419  		t.Error(err)
  1420  	}
  1421  
  1422  	// Give the shim time to reap the init process and kill the orphans
  1423  	select {
  1424  	case <-statusC:
  1425  	case <-time.After(100 * time.Millisecond):
  1426  	}
  1427  
  1428  	b, err := exec.Command("ps", "ax").CombinedOutput()
  1429  	if err != nil {
  1430  		t.Fatal(err)
  1431  	}
  1432  
  1433  	if strings.Contains(string(b), "sleep 42") {
  1434  		t.Fatalf("killing init didn't kill all its children:\n%v", string(b))
  1435  	}
  1436  
  1437  	if _, err := task.Delete(ctx, WithProcessKill); err != nil {
  1438  		t.Error(err)
  1439  	}
  1440  }
  1441  
  1442  func TestContainerKillInitPidHost(t *testing.T) {
  1443  	initContainerAndCheckChildrenDieOnKill(t, oci.WithHostNamespace(specs.PIDNamespace))
  1444  }
  1445  
  1446  func TestContainerKillInitKillsChildWhenNotHostPid(t *testing.T) {
  1447  	initContainerAndCheckChildrenDieOnKill(t)
  1448  }
  1449  
  1450  func TestUserNamespaces(t *testing.T) {
  1451  	t.Parallel()
  1452  	t.Run("WritableRootFS", func(t *testing.T) { testUserNamespaces(t, false) })
  1453  	// see #1373 and runc#1572
  1454  	t.Run("ReadonlyRootFS", func(t *testing.T) { testUserNamespaces(t, true) })
  1455  }
  1456  
  1457  func checkUserNS(t *testing.T) {
  1458  	cmd := exec.Command("true")
  1459  	cmd.SysProcAttr = &syscall.SysProcAttr{
  1460  		Cloneflags: syscall.CLONE_NEWUSER,
  1461  	}
  1462  
  1463  	if err := cmd.Run(); err != nil {
  1464  		t.Skip("User namespaces are unavailable")
  1465  	}
  1466  }
  1467  
  1468  func testUserNamespaces(t *testing.T, readonlyRootFS bool) {
  1469  	checkUserNS(t)
  1470  
  1471  	client, err := newClient(t, address)
  1472  	if err != nil {
  1473  		t.Fatal(err)
  1474  	}
  1475  	defer client.Close()
  1476  
  1477  	var (
  1478  		image       Image
  1479  		ctx, cancel = testContext(t)
  1480  		id          = strings.Replace(t.Name(), "/", "-", -1)
  1481  	)
  1482  	defer cancel()
  1483  
  1484  	image, err = client.GetImage(ctx, testImage)
  1485  	if err != nil {
  1486  		t.Fatal(err)
  1487  	}
  1488  
  1489  	opts := []NewContainerOpts{WithNewSpec(oci.WithImageConfig(image),
  1490  		withExitStatus(7),
  1491  		oci.WithUserNamespace([]specs.LinuxIDMapping{
  1492  			{
  1493  				ContainerID: 0,
  1494  				HostID:      1000,
  1495  				Size:        10000,
  1496  			},
  1497  		}, []specs.LinuxIDMapping{
  1498  			{
  1499  				ContainerID: 0,
  1500  				HostID:      2000,
  1501  				Size:        10000,
  1502  			},
  1503  		}),
  1504  	)}
  1505  	if readonlyRootFS {
  1506  		opts = append([]NewContainerOpts{WithRemappedSnapshotView(id, image, 1000, 2000)}, opts...)
  1507  	} else {
  1508  		opts = append([]NewContainerOpts{WithRemappedSnapshot(id, image, 1000, 2000)}, opts...)
  1509  	}
  1510  
  1511  	container, err := client.NewContainer(ctx, id, opts...)
  1512  	if err != nil {
  1513  		t.Fatal(err)
  1514  	}
  1515  	defer container.Delete(ctx, WithSnapshotCleanup)
  1516  
  1517  	var copts interface{}
  1518  	if CheckRuntime(client.runtime, "io.containerd.runc") {
  1519  		copts = &options.Options{
  1520  			IoUid: 1000,
  1521  			IoGid: 2000,
  1522  		}
  1523  	} else {
  1524  		copts = &runctypes.CreateOptions{
  1525  			IoUid: 1000,
  1526  			IoGid: 2000,
  1527  		}
  1528  	}
  1529  
  1530  	task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStdio), func(_ context.Context, client *Client, r *TaskInfo) error {
  1531  		r.Options = copts
  1532  		return nil
  1533  	})
  1534  	if err != nil {
  1535  		t.Fatal(err)
  1536  	}
  1537  	defer task.Delete(ctx)
  1538  
  1539  	statusC, err := task.Wait(ctx)
  1540  	if err != nil {
  1541  		t.Fatal(err)
  1542  	}
  1543  
  1544  	if pid := task.Pid(); pid < 1 {
  1545  		t.Errorf("invalid task pid %d", pid)
  1546  	}
  1547  	if err := task.Start(ctx); err != nil {
  1548  		t.Error(err)
  1549  		task.Delete(ctx)
  1550  		return
  1551  	}
  1552  	status := <-statusC
  1553  	code, _, err := status.Result()
  1554  	if err != nil {
  1555  		t.Fatal(err)
  1556  	}
  1557  	if code != 7 {
  1558  		t.Errorf("expected status 7 from wait but received %d", code)
  1559  	}
  1560  	deleteStatus, err := task.Delete(ctx)
  1561  	if err != nil {
  1562  		t.Fatal(err)
  1563  	}
  1564  	if ec := deleteStatus.ExitCode(); ec != 7 {
  1565  		t.Errorf("expected status 7 from delete but received %d", ec)
  1566  	}
  1567  }
  1568  
  1569  func TestTaskResize(t *testing.T) {
  1570  	t.Parallel()
  1571  
  1572  	client, err := newClient(t, address)
  1573  	if err != nil {
  1574  		t.Fatal(err)
  1575  	}
  1576  	defer client.Close()
  1577  
  1578  	var (
  1579  		image       Image
  1580  		ctx, cancel = testContext(t)
  1581  		id          = t.Name()
  1582  	)
  1583  	defer cancel()
  1584  
  1585  	image, err = client.GetImage(ctx, testImage)
  1586  	if err != nil {
  1587  		t.Fatal(err)
  1588  	}
  1589  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withExitStatus(7)))
  1590  	if err != nil {
  1591  		t.Fatal(err)
  1592  	}
  1593  	defer container.Delete(ctx, WithSnapshotCleanup)
  1594  
  1595  	task, err := container.NewTask(ctx, empty())
  1596  	if err != nil {
  1597  		t.Fatal(err)
  1598  	}
  1599  	defer task.Delete(ctx)
  1600  
  1601  	statusC, err := task.Wait(ctx)
  1602  	if err != nil {
  1603  		t.Fatal(err)
  1604  	}
  1605  	if err := task.Resize(ctx, 32, 32); err != nil {
  1606  		t.Fatal(err)
  1607  	}
  1608  	task.Kill(ctx, syscall.SIGKILL)
  1609  	<-statusC
  1610  }
  1611  
  1612  func TestContainerImage(t *testing.T) {
  1613  	t.Parallel()
  1614  
  1615  	ctx, cancel := testContext(t)
  1616  	defer cancel()
  1617  	id := t.Name()
  1618  
  1619  	client, err := newClient(t, address)
  1620  	if err != nil {
  1621  		t.Fatal(err)
  1622  	}
  1623  	defer client.Close()
  1624  
  1625  	image, err := client.GetImage(ctx, testImage)
  1626  	if err != nil {
  1627  		t.Fatal(err)
  1628  	}
  1629  
  1630  	container, err := client.NewContainer(ctx, id, WithNewSpec(), WithImage(image))
  1631  	if err != nil {
  1632  		t.Fatal(err)
  1633  	}
  1634  	defer container.Delete(ctx)
  1635  
  1636  	i, err := container.Image(ctx)
  1637  	if err != nil {
  1638  		t.Fatal(err)
  1639  	}
  1640  	if i.Name() != image.Name() {
  1641  		t.Fatalf("expected container image name %s but received %s", image.Name(), i.Name())
  1642  	}
  1643  }
  1644  
  1645  func TestContainerNoImage(t *testing.T) {
  1646  	t.Parallel()
  1647  
  1648  	ctx, cancel := testContext(t)
  1649  	defer cancel()
  1650  	id := t.Name()
  1651  
  1652  	client, err := newClient(t, address)
  1653  	if err != nil {
  1654  		t.Fatal(err)
  1655  	}
  1656  	defer client.Close()
  1657  
  1658  	container, err := client.NewContainer(ctx, id, WithNewSpec())
  1659  	if err != nil {
  1660  		t.Fatal(err)
  1661  	}
  1662  	defer container.Delete(ctx)
  1663  
  1664  	_, err = container.Image(ctx)
  1665  	if err == nil {
  1666  		t.Fatal("error should not be nil when container is created without an image")
  1667  	}
  1668  	if !errdefs.IsNotFound(err) {
  1669  		t.Fatalf("expected error to be %s but received %s", errdefs.ErrNotFound, err)
  1670  	}
  1671  }
  1672  
  1673  func TestUIDNoGID(t *testing.T) {
  1674  	t.Parallel()
  1675  
  1676  	ctx, cancel := testContext(t)
  1677  	defer cancel()
  1678  	id := t.Name()
  1679  
  1680  	client, err := newClient(t, address)
  1681  	if err != nil {
  1682  		t.Fatal(err)
  1683  	}
  1684  	defer client.Close()
  1685  	image, err := client.GetImage(ctx, testImage)
  1686  	if err != nil {
  1687  		t.Fatal(err)
  1688  	}
  1689  
  1690  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithUserID(1000)))
  1691  	if err != nil {
  1692  		t.Fatal(err)
  1693  	}
  1694  	defer container.Delete(ctx)
  1695  
  1696  	spec, err := container.Spec(ctx)
  1697  	if err != nil {
  1698  		t.Fatal(err)
  1699  	}
  1700  	if uid := spec.Process.User.UID; uid != 1000 {
  1701  		t.Fatalf("expected uid 1000 but received %d", uid)
  1702  	}
  1703  	if gid := spec.Process.User.GID; gid != 0 {
  1704  		t.Fatalf("expected gid 0 but received %d", gid)
  1705  	}
  1706  }
  1707  
  1708  func TestBindLowPortNonRoot(t *testing.T) {
  1709  	t.Parallel()
  1710  
  1711  	client, err := newClient(t, address)
  1712  	if err != nil {
  1713  		t.Fatal(err)
  1714  	}
  1715  	defer client.Close()
  1716  
  1717  	var (
  1718  		image       Image
  1719  		ctx, cancel = testContext(t)
  1720  		id          = t.Name()
  1721  	)
  1722  	defer cancel()
  1723  
  1724  	image, err = client.GetImage(ctx, testImage)
  1725  	if err != nil {
  1726  		t.Fatal(err)
  1727  	}
  1728  	container, err := client.NewContainer(ctx, id,
  1729  		WithNewSnapshot(id, image),
  1730  		WithNewSpec(oci.WithImageConfig(image), withProcessArgs("nc", "-l", "-p", "80"), oci.WithUIDGID(1000, 1000)),
  1731  	)
  1732  	if err != nil {
  1733  		t.Fatal(err)
  1734  	}
  1735  	defer container.Delete(ctx, WithSnapshotCleanup)
  1736  
  1737  	task, err := container.NewTask(ctx, empty())
  1738  	if err != nil {
  1739  		t.Fatal(err)
  1740  	}
  1741  	defer task.Delete(ctx)
  1742  
  1743  	statusC, err := task.Wait(ctx)
  1744  	if err != nil {
  1745  		t.Fatal(err)
  1746  	}
  1747  
  1748  	if err := task.Start(ctx); err != nil {
  1749  		t.Fatal(err)
  1750  	}
  1751  	status := <-statusC
  1752  	code, _, err := status.Result()
  1753  	if err != nil {
  1754  		t.Fatal(err)
  1755  	}
  1756  	if code != 1 {
  1757  		t.Errorf("expected status 1 from wait but received %d", code)
  1758  	}
  1759  	if _, err := task.Delete(ctx); err != nil {
  1760  		t.Fatal(err)
  1761  	}
  1762  }
  1763  
  1764  func TestBindLowPortNonOpt(t *testing.T) {
  1765  	t.Parallel()
  1766  
  1767  	client, err := newClient(t, address)
  1768  	if err != nil {
  1769  		t.Fatal(err)
  1770  	}
  1771  	defer client.Close()
  1772  
  1773  	var (
  1774  		image       Image
  1775  		ctx, cancel = testContext(t)
  1776  		id          = t.Name()
  1777  	)
  1778  	defer cancel()
  1779  
  1780  	image, err = client.GetImage(ctx, testImage)
  1781  	if err != nil {
  1782  		t.Fatal(err)
  1783  	}
  1784  	container, err := client.NewContainer(ctx, id,
  1785  		WithNewSnapshot(id, image),
  1786  		WithNewSpec(oci.WithImageConfig(image), withProcessArgs("nc", "-l", "-p", "80"), oci.WithUIDGID(1000, 1000), oci.WithAmbientCapabilities([]string{"CAP_NET_BIND_SERVICE"})),
  1787  	)
  1788  	if err != nil {
  1789  		t.Fatal(err)
  1790  	}
  1791  	defer container.Delete(ctx, WithSnapshotCleanup)
  1792  
  1793  	task, err := container.NewTask(ctx, empty())
  1794  	if err != nil {
  1795  		t.Fatal(err)
  1796  	}
  1797  	defer task.Delete(ctx)
  1798  
  1799  	statusC, err := task.Wait(ctx)
  1800  	if err != nil {
  1801  		t.Fatal(err)
  1802  	}
  1803  
  1804  	if err := task.Start(ctx); err != nil {
  1805  		t.Fatal(err)
  1806  	}
  1807  	go func() {
  1808  		time.Sleep(2 * time.Second)
  1809  		task.Kill(ctx, unix.SIGTERM)
  1810  	}()
  1811  	status := <-statusC
  1812  	code, _, err := status.Result()
  1813  	if err != nil {
  1814  		t.Fatal(err)
  1815  	}
  1816  	// 128 + sigterm
  1817  	if code != 143 {
  1818  		t.Errorf("expected status 143 from wait but received %d", code)
  1819  	}
  1820  	if _, err := task.Delete(ctx); err != nil {
  1821  		t.Fatal(err)
  1822  	}
  1823  }
  1824  
  1825  func TestContainerNoSTDIN(t *testing.T) {
  1826  	t.Parallel()
  1827  
  1828  	client, err := newClient(t, address)
  1829  	if err != nil {
  1830  		t.Fatal(err)
  1831  	}
  1832  	defer client.Close()
  1833  
  1834  	var (
  1835  		image       Image
  1836  		ctx, cancel = testContext(t)
  1837  		id          = t.Name()
  1838  	)
  1839  	defer cancel()
  1840  
  1841  	image, err = client.GetImage(ctx, testImage)
  1842  	if err != nil {
  1843  		t.Fatal(err)
  1844  	}
  1845  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withExitStatus(0)))
  1846  	if err != nil {
  1847  		t.Fatal(err)
  1848  	}
  1849  	defer container.Delete(ctx, WithSnapshotCleanup)
  1850  
  1851  	task, err := container.NewTask(ctx, cio.NewCreator(cio.WithStreams(nil, ioutil.Discard, ioutil.Discard)))
  1852  	if err != nil {
  1853  		t.Fatal(err)
  1854  	}
  1855  	defer task.Delete(ctx)
  1856  
  1857  	statusC, err := task.Wait(ctx)
  1858  	if err != nil {
  1859  		t.Fatal(err)
  1860  	}
  1861  	if err := task.Start(ctx); err != nil {
  1862  		t.Fatal(err)
  1863  	}
  1864  	status := <-statusC
  1865  	code, _, err := status.Result()
  1866  	if err != nil {
  1867  		t.Fatal(err)
  1868  	}
  1869  	if code != 0 {
  1870  		t.Errorf("expected status 0 from wait but received %d", code)
  1871  	}
  1872  }
  1873  
  1874  func TestShimOOMScore(t *testing.T) {
  1875  	containerdPid := ctrd.cmd.Process.Pid
  1876  	containerdScore, err := sys.GetOOMScoreAdj(containerdPid)
  1877  	if err != nil {
  1878  		t.Fatal(err)
  1879  	}
  1880  
  1881  	client, err := newClient(t, address)
  1882  	if err != nil {
  1883  		t.Fatal(err)
  1884  	}
  1885  	defer client.Close()
  1886  
  1887  	var (
  1888  		image       Image
  1889  		ctx, cancel = testContext(t)
  1890  		id          = t.Name()
  1891  	)
  1892  	defer cancel()
  1893  
  1894  	path := "/containerd/oomshim"
  1895  	var (
  1896  		cg  cgroups.Cgroup
  1897  		cg2 *cgroupsv2.Manager
  1898  	)
  1899  	if cgroups.Mode() == cgroups.Unified {
  1900  		cg2, err = cgroupsv2.NewManager("/sys/fs/cgroup", path, &cgroupsv2.Resources{})
  1901  		if err != nil {
  1902  			t.Fatal(err)
  1903  		}
  1904  		defer cg2.Delete()
  1905  	} else {
  1906  		cg, err = cgroups.New(cgroups.V1, cgroups.StaticPath(path), &specs.LinuxResources{})
  1907  		if err != nil {
  1908  			t.Fatal(err)
  1909  		}
  1910  		defer cg.Delete()
  1911  	}
  1912  
  1913  	image, err = client.GetImage(ctx, testImage)
  1914  	if err != nil {
  1915  		t.Fatal(err)
  1916  	}
  1917  
  1918  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), withProcessArgs("sleep", "30")))
  1919  	if err != nil {
  1920  		t.Fatal(err)
  1921  	}
  1922  	defer container.Delete(ctx, WithSnapshotCleanup)
  1923  
  1924  	task, err := container.NewTask(ctx, empty(), WithShimCgroup(path))
  1925  	if err != nil {
  1926  		t.Fatal(err)
  1927  	}
  1928  	defer task.Delete(ctx)
  1929  
  1930  	statusC, err := task.Wait(ctx)
  1931  	if err != nil {
  1932  		t.Fatal(err)
  1933  	}
  1934  
  1935  	expectedScore := containerdScore + 1
  1936  	// find the shim's pid
  1937  	if cgroups.Mode() == cgroups.Unified {
  1938  		processes, err := cg2.Procs(false)
  1939  		if err != nil {
  1940  			t.Fatal(err)
  1941  		}
  1942  		for _, pid := range processes {
  1943  			score, err := sys.GetOOMScoreAdj(int(pid))
  1944  			if err != nil {
  1945  				t.Fatal(err)
  1946  			}
  1947  			if score != expectedScore {
  1948  				t.Errorf("expected score %d but got %d for shim process", expectedScore, score)
  1949  			}
  1950  		}
  1951  	} else {
  1952  		processes, err := cg.Processes(cgroups.Devices, false)
  1953  		if err != nil {
  1954  			t.Fatal(err)
  1955  		}
  1956  		for _, p := range processes {
  1957  			score, err := sys.GetOOMScoreAdj(p.Pid)
  1958  			if err != nil {
  1959  				t.Fatal(err)
  1960  			}
  1961  			if score != expectedScore {
  1962  				t.Errorf("expected score %d but got %d for shim process", expectedScore, score)
  1963  			}
  1964  		}
  1965  	}
  1966  
  1967  	if err := task.Kill(ctx, unix.SIGKILL); err != nil {
  1968  		t.Fatal(err)
  1969  	}
  1970  
  1971  	<-statusC
  1972  }
  1973  
  1974  func TestTaskSpec(t *testing.T) {
  1975  	t.Parallel()
  1976  
  1977  	client, err := newClient(t, address)
  1978  	if err != nil {
  1979  		t.Fatal(err)
  1980  	}
  1981  	defer client.Close()
  1982  
  1983  	var (
  1984  		image       Image
  1985  		ctx, cancel = testContext(t)
  1986  		id          = t.Name()
  1987  	)
  1988  	defer cancel()
  1989  
  1990  	image, err = client.GetImage(ctx, testImage)
  1991  	if err != nil {
  1992  		t.Fatal(err)
  1993  	}
  1994  
  1995  	container, err := client.NewContainer(ctx, id, WithNewSnapshot(id, image), WithNewSpec(oci.WithImageConfig(image), longCommand))
  1996  	if err != nil {
  1997  		t.Fatal(err)
  1998  	}
  1999  	defer container.Delete(ctx, WithSnapshotCleanup)
  2000  
  2001  	task, err := container.NewTask(ctx, empty())
  2002  	if err != nil {
  2003  		t.Fatal(err)
  2004  	}
  2005  	defer task.Delete(ctx)
  2006  
  2007  	statusC, err := task.Wait(ctx)
  2008  	if err != nil {
  2009  		t.Fatal(err)
  2010  	}
  2011  
  2012  	spec, err := task.Spec(ctx)
  2013  	if err != nil {
  2014  		t.Fatal(err)
  2015  	}
  2016  	if spec == nil {
  2017  		t.Fatal("spec from task is nil")
  2018  	}
  2019  	direct, err := newDirectIO(ctx, false)
  2020  	if err != nil {
  2021  		t.Fatal(err)
  2022  	}
  2023  	defer direct.Delete()
  2024  
  2025  	lt, err := container.Task(ctx, direct.IOAttach)
  2026  	if err != nil {
  2027  		t.Fatal(err)
  2028  	}
  2029  
  2030  	spec, err = lt.Spec(ctx)
  2031  	if err != nil {
  2032  		t.Fatal(err)
  2033  	}
  2034  	if spec == nil {
  2035  		t.Fatal("spec from loaded task is nil")
  2036  	}
  2037  
  2038  	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
  2039  		t.Fatal(err)
  2040  	}
  2041  	<-statusC
  2042  }