github.com/ilhicas/nomad@v0.11.8/drivers/docker/driver_unix_test.go (about)

     1  // +build darwin dragonfly freebsd linux netbsd openbsd solaris
     2  
     3  package docker
     4  
     5  import (
     6  	"fmt"
     7  	"io"
     8  	"os"
     9  	"path/filepath"
    10  	"runtime"
    11  	"sort"
    12  	"strconv"
    13  	"strings"
    14  	"testing"
    15  	"time"
    16  
    17  	docker "github.com/fsouza/go-dockerclient"
    18  	"github.com/hashicorp/nomad/client/allocdir"
    19  	"github.com/hashicorp/nomad/client/testutil"
    20  	"github.com/hashicorp/nomad/helper/freeport"
    21  	"github.com/hashicorp/nomad/helper/uuid"
    22  	"github.com/hashicorp/nomad/plugins/drivers"
    23  	dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils"
    24  	tu "github.com/hashicorp/nomad/testutil"
    25  	"github.com/stretchr/testify/assert"
    26  	"github.com/stretchr/testify/require"
    27  )
    28  
    29  func TestDockerDriver_User(t *testing.T) {
    30  	if !tu.IsCI() {
    31  		t.Parallel()
    32  	}
    33  	testutil.DockerCompatible(t)
    34  	task, cfg, ports := dockerTask(t)
    35  	defer freeport.Return(ports)
    36  	task.User = "alice"
    37  	cfg.Command = "/bin/sleep"
    38  	cfg.Args = []string{"10000"}
    39  	require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
    40  
    41  	d := dockerDriverHarness(t, nil)
    42  	cleanup := d.MkAllocDir(task, true)
    43  	defer cleanup()
    44  	copyImage(t, task.TaskDir(), "busybox.tar")
    45  
    46  	_, _, err := d.StartTask(task)
    47  	if err == nil {
    48  		d.DestroyTask(task.ID, true)
    49  		t.Fatalf("Should've failed")
    50  	}
    51  
    52  	if !strings.Contains(err.Error(), "alice") {
    53  		t.Fatalf("Expected failure string not found, found %q instead", err.Error())
    54  	}
    55  }
    56  
    57  func TestDockerDriver_NetworkAliases_Bridge(t *testing.T) {
    58  	if !tu.IsCI() {
    59  		t.Parallel()
    60  	}
    61  	testutil.DockerCompatible(t)
    62  	require := require.New(t)
    63  
    64  	// Because go-dockerclient doesn't provide api for query network aliases, just check that
    65  	// a container can be created with a 'network_aliases' property
    66  
    67  	// Create network, network-scoped alias is supported only for containers in user defined networks
    68  	client := newTestDockerClient(t)
    69  	networkOpts := docker.CreateNetworkOptions{Name: "foobar", Driver: "bridge"}
    70  	network, err := client.CreateNetwork(networkOpts)
    71  	require.NoError(err)
    72  	defer client.RemoveNetwork(network.ID)
    73  
    74  	expected := []string{"foobar"}
    75  	taskCfg := newTaskConfig("", busyboxLongRunningCmd)
    76  	taskCfg.NetworkMode = network.Name
    77  	taskCfg.NetworkAliases = expected
    78  	task := &drivers.TaskConfig{
    79  		ID:        uuid.Generate(),
    80  		Name:      "busybox",
    81  		Resources: basicResources,
    82  	}
    83  	require.NoError(task.EncodeConcreteDriverConfig(&taskCfg))
    84  
    85  	d := dockerDriverHarness(t, nil)
    86  	cleanup := d.MkAllocDir(task, true)
    87  	defer cleanup()
    88  	copyImage(t, task.TaskDir(), "busybox.tar")
    89  
    90  	_, _, err = d.StartTask(task)
    91  	require.NoError(err)
    92  	require.NoError(d.WaitUntilStarted(task.ID, 5*time.Second))
    93  
    94  	defer d.DestroyTask(task.ID, true)
    95  
    96  	dockerDriver, ok := d.Impl().(*Driver)
    97  	require.True(ok)
    98  
    99  	handle, ok := dockerDriver.tasks.Get(task.ID)
   100  	require.True(ok)
   101  
   102  	_, err = client.InspectContainer(handle.containerID)
   103  	require.NoError(err)
   104  }
   105  
   106  func TestDockerDriver_NetworkMode_Host(t *testing.T) {
   107  	if !tu.IsCI() {
   108  		t.Parallel()
   109  	}
   110  	testutil.DockerCompatible(t)
   111  	expected := "host"
   112  
   113  	taskCfg := newTaskConfig("", busyboxLongRunningCmd)
   114  	taskCfg.NetworkMode = expected
   115  
   116  	task := &drivers.TaskConfig{
   117  		ID:        uuid.Generate(),
   118  		Name:      "busybox-demo",
   119  		Resources: basicResources,
   120  	}
   121  	require.NoError(t, task.EncodeConcreteDriverConfig(&taskCfg))
   122  
   123  	d := dockerDriverHarness(t, nil)
   124  	cleanup := d.MkAllocDir(task, true)
   125  	defer cleanup()
   126  	copyImage(t, task.TaskDir(), "busybox.tar")
   127  
   128  	_, _, err := d.StartTask(task)
   129  	require.NoError(t, err)
   130  
   131  	require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second))
   132  
   133  	defer d.DestroyTask(task.ID, true)
   134  
   135  	dockerDriver, ok := d.Impl().(*Driver)
   136  	require.True(t, ok)
   137  
   138  	handle, ok := dockerDriver.tasks.Get(task.ID)
   139  	require.True(t, ok)
   140  
   141  	container, err := client.InspectContainer(handle.containerID)
   142  	if err != nil {
   143  		t.Fatalf("err: %v", err)
   144  	}
   145  
   146  	actual := container.HostConfig.NetworkMode
   147  	require.Equal(t, expected, actual)
   148  }
   149  
   150  func TestDockerDriver_CPUCFSPeriod(t *testing.T) {
   151  	if !tu.IsCI() {
   152  		t.Parallel()
   153  	}
   154  	testutil.DockerCompatible(t)
   155  
   156  	task, cfg, ports := dockerTask(t)
   157  	defer freeport.Return(ports)
   158  	cfg.CPUHardLimit = true
   159  	cfg.CPUCFSPeriod = 1000000
   160  	require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   161  
   162  	client, _, handle, cleanup := dockerSetup(t, task, nil)
   163  	defer cleanup()
   164  
   165  	waitForExist(t, client, handle.containerID)
   166  
   167  	container, err := client.InspectContainer(handle.containerID)
   168  	require.NoError(t, err)
   169  
   170  	require.Equal(t, cfg.CPUCFSPeriod, container.HostConfig.CPUPeriod)
   171  }
   172  
   173  func TestDockerDriver_Sysctl_Ulimit(t *testing.T) {
   174  	testutil.DockerCompatible(t)
   175  	task, cfg, ports := dockerTask(t)
   176  	defer freeport.Return(ports)
   177  	expectedUlimits := map[string]string{
   178  		"nproc":  "4242",
   179  		"nofile": "2048:4096",
   180  	}
   181  	cfg.Sysctl = map[string]string{
   182  		"net.core.somaxconn": "16384",
   183  	}
   184  	cfg.Ulimit = expectedUlimits
   185  	require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   186  
   187  	client, d, handle, cleanup := dockerSetup(t, task, nil)
   188  	defer cleanup()
   189  	require.NoError(t, d.WaitUntilStarted(task.ID, 5*time.Second))
   190  
   191  	container, err := client.InspectContainer(handle.containerID)
   192  	assert.Nil(t, err, "unexpected error: %v", err)
   193  
   194  	want := "16384"
   195  	got := container.HostConfig.Sysctls["net.core.somaxconn"]
   196  	assert.Equal(t, want, got, "Wrong net.core.somaxconn config for docker job. Expect: %s, got: %s", want, got)
   197  
   198  	expectedUlimitLen := 2
   199  	actualUlimitLen := len(container.HostConfig.Ulimits)
   200  	assert.Equal(t, want, got, "Wrong number of ulimit configs for docker job. Expect: %d, got: %d", expectedUlimitLen, actualUlimitLen)
   201  
   202  	for _, got := range container.HostConfig.Ulimits {
   203  		if expectedStr, ok := expectedUlimits[got.Name]; !ok {
   204  			t.Errorf("%s config unexpected for docker job.", got.Name)
   205  		} else {
   206  			if !strings.Contains(expectedStr, ":") {
   207  				expectedStr = expectedStr + ":" + expectedStr
   208  			}
   209  
   210  			splitted := strings.SplitN(expectedStr, ":", 2)
   211  			soft, _ := strconv.Atoi(splitted[0])
   212  			hard, _ := strconv.Atoi(splitted[1])
   213  			assert.Equal(t, int64(soft), got.Soft, "Wrong soft %s ulimit for docker job. Expect: %d, got: %d", got.Name, soft, got.Soft)
   214  			assert.Equal(t, int64(hard), got.Hard, "Wrong hard %s ulimit for docker job. Expect: %d, got: %d", got.Name, hard, got.Hard)
   215  
   216  		}
   217  	}
   218  }
   219  
   220  func TestDockerDriver_Sysctl_Ulimit_Errors(t *testing.T) {
   221  	testutil.DockerCompatible(t)
   222  	brokenConfigs := []map[string]string{
   223  		{
   224  			"nofile": "",
   225  		},
   226  		{
   227  			"nofile": "abc:1234",
   228  		},
   229  		{
   230  			"nofile": "1234:abc",
   231  		},
   232  	}
   233  
   234  	testCases := []struct {
   235  		ulimitConfig map[string]string
   236  		err          error
   237  	}{
   238  		{brokenConfigs[0], fmt.Errorf("Malformed ulimit specification nofile: \"\", cannot be empty")},
   239  		{brokenConfigs[1], fmt.Errorf("Malformed soft ulimit nofile: abc:1234")},
   240  		{brokenConfigs[2], fmt.Errorf("Malformed hard ulimit nofile: 1234:abc")},
   241  	}
   242  
   243  	for _, tc := range testCases {
   244  		task, cfg, ports := dockerTask(t)
   245  		cfg.Ulimit = tc.ulimitConfig
   246  		require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   247  
   248  		d := dockerDriverHarness(t, nil)
   249  		cleanup := d.MkAllocDir(task, true)
   250  		defer cleanup()
   251  		copyImage(t, task.TaskDir(), "busybox.tar")
   252  
   253  		_, _, err := d.StartTask(task)
   254  		require.NotNil(t, err, "Expected non nil error")
   255  		require.Contains(t, err.Error(), tc.err.Error())
   256  		freeport.Return(ports)
   257  	}
   258  }
   259  
   260  // This test does not run on Windows due to stricter path validation in the
   261  // negative case for non existent mount paths. We should write a similar test
   262  // for windows.
   263  func TestDockerDriver_BindMountsHonorVolumesEnabledFlag(t *testing.T) {
   264  	t.Parallel()
   265  
   266  	testutil.DockerCompatible(t)
   267  
   268  	allocDir := "/tmp/nomad/alloc-dir"
   269  
   270  	cases := []struct {
   271  		name            string
   272  		requiresVolumes bool
   273  
   274  		volumeDriver string
   275  		volumes      []string
   276  
   277  		expectedVolumes []string
   278  	}{
   279  		{
   280  			name:            "basic plugin",
   281  			requiresVolumes: true,
   282  			volumeDriver:    "nfs",
   283  			volumes:         []string{"test-path:/tmp/taskpath"},
   284  			expectedVolumes: []string{"test-path:/tmp/taskpath"},
   285  		},
   286  		{
   287  			name:            "absolute default driver",
   288  			requiresVolumes: true,
   289  			volumeDriver:    "",
   290  			volumes:         []string{"/abs/test-path:/tmp/taskpath"},
   291  			expectedVolumes: []string{"/abs/test-path:/tmp/taskpath"},
   292  		},
   293  		{
   294  			name:            "absolute local driver",
   295  			requiresVolumes: true,
   296  			volumeDriver:    "local",
   297  			volumes:         []string{"/abs/test-path:/tmp/taskpath"},
   298  			expectedVolumes: []string{"/abs/test-path:/tmp/taskpath"},
   299  		},
   300  		{
   301  			name:            "relative default driver",
   302  			requiresVolumes: false,
   303  			volumeDriver:    "",
   304  			volumes:         []string{"test-path:/tmp/taskpath"},
   305  			expectedVolumes: []string{"/tmp/nomad/alloc-dir/demo/test-path:/tmp/taskpath"},
   306  		},
   307  		{
   308  			name:            "named volume local driver",
   309  			requiresVolumes: true,
   310  			volumeDriver:    "local",
   311  			volumes:         []string{"test-path:/tmp/taskpath"},
   312  			expectedVolumes: []string{"test-path:/tmp/taskpath"},
   313  		},
   314  		{
   315  			name:            "relative outside task-dir default driver",
   316  			requiresVolumes: false,
   317  			volumeDriver:    "",
   318  			volumes:         []string{"../test-path:/tmp/taskpath"},
   319  			expectedVolumes: []string{"/tmp/nomad/alloc-dir/test-path:/tmp/taskpath"},
   320  		},
   321  		{
   322  			name:            "relative outside alloc-dir default driver",
   323  			requiresVolumes: true,
   324  			volumeDriver:    "",
   325  			volumes:         []string{"../../test-path:/tmp/taskpath"},
   326  			expectedVolumes: []string{"/tmp/nomad/test-path:/tmp/taskpath"},
   327  		},
   328  		{
   329  			name:            "clean path local driver",
   330  			requiresVolumes: true,
   331  			volumeDriver:    "local",
   332  			volumes:         []string{"/tmp/nomad/../test-path:/tmp/taskpath"},
   333  			expectedVolumes: []string{"/tmp/test-path:/tmp/taskpath"},
   334  		},
   335  	}
   336  
   337  	t.Run("with volumes enabled", func(t *testing.T) {
   338  		dh := dockerDriverHarness(t, nil)
   339  		driver := dh.Impl().(*Driver)
   340  		driver.config.Volumes.Enabled = true
   341  
   342  		for _, c := range cases {
   343  			t.Run(c.name, func(t *testing.T) {
   344  				task, cfg, ports := dockerTask(t)
   345  				defer freeport.Return(ports)
   346  				cfg.VolumeDriver = c.volumeDriver
   347  				cfg.Volumes = c.volumes
   348  
   349  				task.AllocDir = allocDir
   350  				task.Name = "demo"
   351  
   352  				require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   353  
   354  				cc, err := driver.createContainerConfig(task, cfg, "org/repo:0.1")
   355  				require.NoError(t, err)
   356  
   357  				for _, v := range c.expectedVolumes {
   358  					require.Contains(t, cc.HostConfig.Binds, v)
   359  				}
   360  			})
   361  		}
   362  	})
   363  
   364  	t.Run("with volumes disabled", func(t *testing.T) {
   365  		dh := dockerDriverHarness(t, nil)
   366  		driver := dh.Impl().(*Driver)
   367  		driver.config.Volumes.Enabled = false
   368  
   369  		for _, c := range cases {
   370  			t.Run(c.name, func(t *testing.T) {
   371  				task, cfg, ports := dockerTask(t)
   372  				defer freeport.Return(ports)
   373  				cfg.VolumeDriver = c.volumeDriver
   374  				cfg.Volumes = c.volumes
   375  
   376  				task.AllocDir = allocDir
   377  				task.Name = "demo"
   378  
   379  				require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   380  
   381  				cc, err := driver.createContainerConfig(task, cfg, "org/repo:0.1")
   382  				if c.requiresVolumes {
   383  					require.Error(t, err, "volumes are not enabled")
   384  				} else {
   385  					require.NoError(t, err)
   386  
   387  					for _, v := range c.expectedVolumes {
   388  						require.Contains(t, cc.HostConfig.Binds, v)
   389  					}
   390  				}
   391  			})
   392  		}
   393  	})
   394  }
   395  
   396  // This test does not run on windows due to differences in the definition of
   397  // an absolute path, changing path expansion behaviour. A similar test should
   398  // be written for windows.
   399  func TestDockerDriver_MountsSerialization(t *testing.T) {
   400  	t.Parallel()
   401  	testutil.DockerCompatible(t)
   402  
   403  	allocDir := "/tmp/nomad/alloc-dir"
   404  
   405  	cases := []struct {
   406  		name            string
   407  		requiresVolumes bool
   408  		passedMounts    []DockerMount
   409  		expectedMounts  []docker.HostMount
   410  	}{
   411  		{
   412  			name:            "basic volume",
   413  			requiresVolumes: true,
   414  			passedMounts: []DockerMount{
   415  				{
   416  					Target:   "/nomad",
   417  					ReadOnly: true,
   418  					Source:   "test",
   419  				},
   420  			},
   421  			expectedMounts: []docker.HostMount{
   422  				{
   423  					Type:          "volume",
   424  					Target:        "/nomad",
   425  					Source:        "test",
   426  					ReadOnly:      true,
   427  					VolumeOptions: &docker.VolumeOptions{},
   428  				},
   429  			},
   430  		},
   431  		{
   432  			name: "basic bind",
   433  			passedMounts: []DockerMount{
   434  				{
   435  					Type:   "bind",
   436  					Target: "/nomad",
   437  					Source: "test",
   438  				},
   439  			},
   440  			expectedMounts: []docker.HostMount{
   441  				{
   442  					Type:        "bind",
   443  					Target:      "/nomad",
   444  					Source:      "/tmp/nomad/alloc-dir/demo/test",
   445  					BindOptions: &docker.BindOptions{},
   446  				},
   447  			},
   448  		},
   449  		{
   450  			name:            "basic absolute bind",
   451  			requiresVolumes: true,
   452  			passedMounts: []DockerMount{
   453  				{
   454  					Type:   "bind",
   455  					Target: "/nomad",
   456  					Source: "/tmp/test",
   457  				},
   458  			},
   459  			expectedMounts: []docker.HostMount{
   460  				{
   461  					Type:        "bind",
   462  					Target:      "/nomad",
   463  					Source:      "/tmp/test",
   464  					BindOptions: &docker.BindOptions{},
   465  				},
   466  			},
   467  		},
   468  		{
   469  			name:            "bind relative outside",
   470  			requiresVolumes: true,
   471  			passedMounts: []DockerMount{
   472  				{
   473  					Type:   "bind",
   474  					Target: "/nomad",
   475  					Source: "../../test",
   476  				},
   477  			},
   478  			expectedMounts: []docker.HostMount{
   479  				{
   480  					Type:        "bind",
   481  					Target:      "/nomad",
   482  					Source:      "/tmp/nomad/test",
   483  					BindOptions: &docker.BindOptions{},
   484  				},
   485  			},
   486  		},
   487  		{
   488  			name:            "basic tmpfs",
   489  			requiresVolumes: false,
   490  			passedMounts: []DockerMount{
   491  				{
   492  					Type:   "tmpfs",
   493  					Target: "/nomad",
   494  					TmpfsOptions: DockerTmpfsOptions{
   495  						SizeBytes: 321,
   496  						Mode:      0666,
   497  					},
   498  				},
   499  			},
   500  			expectedMounts: []docker.HostMount{
   501  				{
   502  					Type:   "tmpfs",
   503  					Target: "/nomad",
   504  					TempfsOptions: &docker.TempfsOptions{
   505  						SizeBytes: 321,
   506  						Mode:      0666,
   507  					},
   508  				},
   509  			},
   510  		},
   511  	}
   512  
   513  	t.Run("with volumes enabled", func(t *testing.T) {
   514  		dh := dockerDriverHarness(t, nil)
   515  		driver := dh.Impl().(*Driver)
   516  		driver.config.Volumes.Enabled = true
   517  
   518  		for _, c := range cases {
   519  			t.Run(c.name, func(t *testing.T) {
   520  				task, cfg, ports := dockerTask(t)
   521  				defer freeport.Return(ports)
   522  				cfg.Mounts = c.passedMounts
   523  
   524  				task.AllocDir = allocDir
   525  				task.Name = "demo"
   526  
   527  				require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   528  
   529  				cc, err := driver.createContainerConfig(task, cfg, "org/repo:0.1")
   530  				require.NoError(t, err)
   531  				require.EqualValues(t, c.expectedMounts, cc.HostConfig.Mounts)
   532  			})
   533  		}
   534  	})
   535  
   536  	t.Run("with volumes disabled", func(t *testing.T) {
   537  		dh := dockerDriverHarness(t, nil)
   538  		driver := dh.Impl().(*Driver)
   539  		driver.config.Volumes.Enabled = false
   540  
   541  		for _, c := range cases {
   542  			t.Run(c.name, func(t *testing.T) {
   543  				task, cfg, ports := dockerTask(t)
   544  				defer freeport.Return(ports)
   545  				cfg.Mounts = c.passedMounts
   546  
   547  				task.AllocDir = allocDir
   548  				task.Name = "demo"
   549  
   550  				require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   551  
   552  				cc, err := driver.createContainerConfig(task, cfg, "org/repo:0.1")
   553  				if c.requiresVolumes {
   554  					require.Error(t, err, "volumes are not enabled")
   555  				} else {
   556  					require.NoError(t, err)
   557  					require.EqualValues(t, c.expectedMounts, cc.HostConfig.Mounts)
   558  				}
   559  			})
   560  		}
   561  	})
   562  }
   563  
   564  // TestDockerDriver_CreateContainerConfig_MountsCombined asserts that
   565  // devices and mounts set by device managers/plugins are honored
   566  // and present in docker.CreateContainerOptions, and that it is appended
   567  // to any devices/mounts a user sets in the task config.
   568  func TestDockerDriver_CreateContainerConfig_MountsCombined(t *testing.T) {
   569  	t.Parallel()
   570  	testutil.DockerCompatible(t)
   571  
   572  	task, cfg, ports := dockerTask(t)
   573  	defer freeport.Return(ports)
   574  
   575  	task.Devices = []*drivers.DeviceConfig{
   576  		{
   577  			HostPath:    "/dev/fuse",
   578  			TaskPath:    "/container/dev/task-fuse",
   579  			Permissions: "rw",
   580  		},
   581  	}
   582  	task.Mounts = []*drivers.MountConfig{
   583  		{
   584  			HostPath: "/tmp/task-mount",
   585  			TaskPath: "/container/tmp/task-mount",
   586  			Readonly: true,
   587  		},
   588  	}
   589  
   590  	cfg.Devices = []DockerDevice{
   591  		{
   592  			HostPath:          "/dev/stdout",
   593  			ContainerPath:     "/container/dev/cfg-stdout",
   594  			CgroupPermissions: "rwm",
   595  		},
   596  	}
   597  	cfg.Mounts = []DockerMount{
   598  		{
   599  			Type:     "bind",
   600  			Source:   "/tmp/cfg-mount",
   601  			Target:   "/container/tmp/cfg-mount",
   602  			ReadOnly: false,
   603  		},
   604  	}
   605  
   606  	require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   607  
   608  	dh := dockerDriverHarness(t, nil)
   609  	driver := dh.Impl().(*Driver)
   610  	driver.config.Volumes.Enabled = true
   611  
   612  	c, err := driver.createContainerConfig(task, cfg, "org/repo:0.1")
   613  	require.NoError(t, err)
   614  	expectedMounts := []docker.HostMount{
   615  		{
   616  			Type:     "bind",
   617  			Source:   "/tmp/cfg-mount",
   618  			Target:   "/container/tmp/cfg-mount",
   619  			ReadOnly: false,
   620  			BindOptions: &docker.BindOptions{
   621  				Propagation: "",
   622  			},
   623  		},
   624  		{
   625  			Type:     "bind",
   626  			Source:   "/tmp/task-mount",
   627  			Target:   "/container/tmp/task-mount",
   628  			ReadOnly: true,
   629  			BindOptions: &docker.BindOptions{
   630  				Propagation: "rprivate",
   631  			},
   632  		},
   633  	}
   634  
   635  	if runtime.GOOS != "linux" {
   636  		expectedMounts[0].BindOptions = &docker.BindOptions{}
   637  		expectedMounts[1].BindOptions = &docker.BindOptions{}
   638  	}
   639  
   640  	foundMounts := c.HostConfig.Mounts
   641  	sort.Slice(foundMounts, func(i, j int) bool {
   642  		return foundMounts[i].Target < foundMounts[j].Target
   643  	})
   644  	require.EqualValues(t, expectedMounts, foundMounts)
   645  
   646  	expectedDevices := []docker.Device{
   647  		{
   648  			PathOnHost:        "/dev/stdout",
   649  			PathInContainer:   "/container/dev/cfg-stdout",
   650  			CgroupPermissions: "rwm",
   651  		},
   652  		{
   653  			PathOnHost:        "/dev/fuse",
   654  			PathInContainer:   "/container/dev/task-fuse",
   655  			CgroupPermissions: "rw",
   656  		},
   657  	}
   658  
   659  	foundDevices := c.HostConfig.Devices
   660  	sort.Slice(foundDevices, func(i, j int) bool {
   661  		return foundDevices[i].PathInContainer < foundDevices[j].PathInContainer
   662  	})
   663  	require.EqualValues(t, expectedDevices, foundDevices)
   664  }
   665  
   666  // TestDockerDriver_Cleanup ensures Cleanup removes only downloaded images.
   667  // Doesn't run on windows because it requires an image variant
   668  func TestDockerDriver_Cleanup(t *testing.T) {
   669  	testutil.DockerCompatible(t)
   670  
   671  	// using a small image and an specific point release to avoid accidental conflicts with other tasks
   672  	cfg := newTaskConfig("", []string{"sleep", "100"})
   673  	cfg.Image = "busybox:1.29.2"
   674  	cfg.LoadImage = ""
   675  	task := &drivers.TaskConfig{
   676  		ID:        uuid.Generate(),
   677  		Name:      "cleanup_test",
   678  		Resources: basicResources,
   679  	}
   680  
   681  	require.NoError(t, task.EncodeConcreteDriverConfig(cfg))
   682  
   683  	client, driver, handle, cleanup := dockerSetup(t, task, map[string]interface{}{
   684  		"gc": map[string]interface{}{
   685  			"image":       true,
   686  			"image_delay": "1ms",
   687  		},
   688  	})
   689  	defer cleanup()
   690  
   691  	require.NoError(t, driver.WaitUntilStarted(task.ID, 5*time.Second))
   692  	// Cleanup
   693  	require.NoError(t, driver.DestroyTask(task.ID, true))
   694  
   695  	// Ensure image was removed
   696  	tu.WaitForResult(func() (bool, error) {
   697  		if _, err := client.InspectImage(cfg.Image); err == nil {
   698  			return false, fmt.Errorf("image exists but should have been removed. Does another %v container exist?", cfg.Image)
   699  		}
   700  
   701  		return true, nil
   702  	}, func(err error) {
   703  		require.NoError(t, err)
   704  	})
   705  
   706  	// The image doesn't exist which shouldn't be an error when calling
   707  	// Cleanup, so call it again to make sure.
   708  	require.NoError(t, driver.Impl().(*Driver).cleanupImage(handle))
   709  }
   710  
   711  // Tests that images prefixed with "https://" are supported
   712  func TestDockerDriver_Start_Image_HTTPS(t *testing.T) {
   713  	if !tu.IsCI() {
   714  		t.Parallel()
   715  	}
   716  	testutil.DockerCompatible(t)
   717  
   718  	taskCfg := TaskConfig{
   719  		Image: "https://gcr.io/google_containers/pause:0.8.0",
   720  	}
   721  	task := &drivers.TaskConfig{
   722  		ID:        uuid.Generate(),
   723  		Name:      "pause",
   724  		AllocID:   uuid.Generate(),
   725  		Resources: basicResources,
   726  	}
   727  	require.NoError(t, task.EncodeConcreteDriverConfig(&taskCfg))
   728  
   729  	d := dockerDriverHarness(t, nil)
   730  	cleanup := d.MkAllocDir(task, true)
   731  	defer cleanup()
   732  
   733  	_, _, err := d.StartTask(task)
   734  	require.NoError(t, err)
   735  
   736  	d.DestroyTask(task.ID, true)
   737  }
   738  
   739  func newTaskConfig(variant string, command []string) TaskConfig {
   740  	// busyboxImageID is the ID stored in busybox.tar
   741  	busyboxImageID := "busybox:1.29.3"
   742  
   743  	image := busyboxImageID
   744  	loadImage := "busybox.tar"
   745  	if variant != "" {
   746  		image = fmt.Sprintf("%s-%s", busyboxImageID, variant)
   747  		loadImage = fmt.Sprintf("busybox_%s.tar", variant)
   748  	}
   749  
   750  	return TaskConfig{
   751  		Image:     image,
   752  		LoadImage: loadImage,
   753  		Command:   command[0],
   754  		Args:      command[1:],
   755  	}
   756  }
   757  
   758  func copyImage(t *testing.T, taskDir *allocdir.TaskDir, image string) {
   759  	dst := filepath.Join(taskDir.LocalDir, image)
   760  	copyFile(filepath.Join("./test-resources/docker", image), dst, t)
   761  }
   762  
   763  // copyFile moves an existing file to the destination
   764  func copyFile(src, dst string, t *testing.T) {
   765  	in, err := os.Open(src)
   766  	if err != nil {
   767  		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
   768  	}
   769  	defer in.Close()
   770  	out, err := os.Create(dst)
   771  	if err != nil {
   772  		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
   773  	}
   774  	defer func() {
   775  		if err := out.Close(); err != nil {
   776  			t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
   777  		}
   778  	}()
   779  	if _, err = io.Copy(out, in); err != nil {
   780  		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
   781  	}
   782  	if err := out.Sync(); err != nil {
   783  		t.Fatalf("copying %v -> %v failed: %v", src, dst, err)
   784  	}
   785  }
   786  
   787  func TestDocker_ExecTaskStreaming(t *testing.T) {
   788  	if !tu.IsCI() {
   789  		t.Parallel()
   790  	}
   791  	testutil.DockerCompatible(t)
   792  
   793  	taskCfg := newTaskConfig("", []string{"/bin/sleep", "1000"})
   794  	task := &drivers.TaskConfig{
   795  		ID:        uuid.Generate(),
   796  		Name:      "nc-demo",
   797  		AllocID:   uuid.Generate(),
   798  		Resources: basicResources,
   799  	}
   800  	require.NoError(t, task.EncodeConcreteDriverConfig(&taskCfg))
   801  
   802  	d := dockerDriverHarness(t, nil)
   803  	cleanup := d.MkAllocDir(task, true)
   804  	defer cleanup()
   805  	copyImage(t, task.TaskDir(), "busybox.tar")
   806  
   807  	_, _, err := d.StartTask(task)
   808  	require.NoError(t, err)
   809  
   810  	defer d.DestroyTask(task.ID, true)
   811  
   812  	dtestutil.ExecTaskStreamingConformanceTests(t, d, task.ID)
   813  
   814  }