github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/drivers/exec/driver_unix_test.go (about) 1 //go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris 2 3 package exec 4 5 import ( 6 "context" 7 "fmt" 8 "strings" 9 "testing" 10 "time" 11 12 "github.com/hashicorp/nomad/ci" 13 "github.com/hashicorp/nomad/client/lib/cgutil" 14 ctestutils "github.com/hashicorp/nomad/client/testutil" 15 "github.com/hashicorp/nomad/drivers/shared/capabilities" 16 "github.com/hashicorp/nomad/drivers/shared/executor" 17 "github.com/hashicorp/nomad/helper/testlog" 18 "github.com/hashicorp/nomad/helper/uuid" 19 basePlug "github.com/hashicorp/nomad/plugins/base" 20 "github.com/hashicorp/nomad/plugins/drivers" 21 dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" 22 "github.com/hashicorp/nomad/testutil" 23 "github.com/stretchr/testify/require" 24 "golang.org/x/sys/unix" 25 ) 26 27 func TestExecDriver_StartWaitStop(t *testing.T) { 28 ci.Parallel(t) 29 ctestutils.ExecCompatible(t) 30 31 ctx, cancel := context.WithCancel(context.Background()) 32 defer cancel() 33 34 d := NewExecDriver(ctx, testlog.HCLogger(t)) 35 harness := dtestutil.NewDriverHarness(t, d) 36 allocID := uuid.Generate() 37 task := &drivers.TaskConfig{ 38 AllocID: allocID, 39 ID: uuid.Generate(), 40 Name: "test", 41 Resources: testResources(allocID, "test"), 42 } 43 44 taskConfig := map[string]interface{}{ 45 "command": "/bin/sleep", 46 "args": []string{"600"}, 47 } 48 require.NoError(t, task.EncodeConcreteDriverConfig(&taskConfig)) 49 50 cleanup := harness.MkAllocDir(task, false) 51 defer cleanup() 52 53 handle, _, err := harness.StartTask(task) 54 defer harness.DestroyTask(task.ID, true) 55 require.NoError(t, err) 56 57 ch, err := harness.WaitTask(context.Background(), handle.Config.ID) 58 require.NoError(t, err) 59 60 require.NoError(t, harness.WaitUntilStarted(task.ID, 1*time.Second)) 61 62 go func() { 63 harness.StopTask(task.ID, 2*time.Second, "SIGKILL") 64 }() 65 66 select { 67 case result := <-ch: 68 require.Equal(t, int(unix.SIGKILL), result.Signal) 69 case <-time.After(10 * time.Second): 70 require.Fail(t, "timeout waiting for task to shutdown") 71 } 72 73 // Ensure that the task is marked as dead, but account 74 // for WaitTask() closing channel before internal state is updated 75 testutil.WaitForResult(func() (bool, error) { 76 status, err := harness.InspectTask(task.ID) 77 if err != nil { 78 return false, fmt.Errorf("inspecting task failed: %v", err) 79 } 80 if status.State != drivers.TaskStateExited { 81 return false, fmt.Errorf("task hasn't exited yet; status: %v", status.State) 82 } 83 84 return true, nil 85 }, func(err error) { 86 require.NoError(t, err) 87 }) 88 } 89 90 func TestExec_ExecTaskStreaming(t *testing.T) { 91 ci.Parallel(t) 92 93 ctx, cancel := context.WithCancel(context.Background()) 94 defer cancel() 95 96 d := NewExecDriver(ctx, testlog.HCLogger(t)) 97 harness := dtestutil.NewDriverHarness(t, d) 98 defer harness.Kill() 99 100 task := &drivers.TaskConfig{ 101 ID: uuid.Generate(), 102 Name: "sleep", 103 } 104 105 if cgutil.UseV2 { 106 allocID := uuid.Generate() 107 task.AllocID = allocID 108 task.Resources = testResources(allocID, "sleep") 109 } 110 111 cleanup := harness.MkAllocDir(task, false) 112 defer cleanup() 113 114 tc := &TaskConfig{ 115 Command: "/bin/sleep", 116 Args: []string{"9000"}, 117 } 118 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 119 120 _, _, err := harness.StartTask(task) 121 require.NoError(t, err) 122 defer d.DestroyTask(task.ID, true) 123 124 dtestutil.ExecTaskStreamingConformanceTests(t, harness, task.ID) 125 } 126 127 // Tests that a given DNSConfig properly configures dns 128 func TestExec_dnsConfig(t *testing.T) { 129 ci.Parallel(t) 130 ctestutils.RequireRoot(t) 131 ctestutils.ExecCompatible(t) 132 133 ctx, cancel := context.WithCancel(context.Background()) 134 defer cancel() 135 136 d := NewExecDriver(ctx, testlog.HCLogger(t)) 137 harness := dtestutil.NewDriverHarness(t, d) 138 defer harness.Kill() 139 140 cases := []struct { 141 name string 142 cfg *drivers.DNSConfig 143 }{ 144 { 145 name: "nil DNSConfig", 146 }, 147 { 148 name: "basic", 149 cfg: &drivers.DNSConfig{ 150 Servers: []string{"1.1.1.1", "1.0.0.1"}, 151 }, 152 }, 153 { 154 name: "full", 155 cfg: &drivers.DNSConfig{ 156 Servers: []string{"1.1.1.1", "1.0.0.1"}, 157 Searches: []string{"local.test", "node.consul"}, 158 Options: []string{"ndots:2", "edns0"}, 159 }, 160 }, 161 } 162 163 for _, c := range cases { 164 task := &drivers.TaskConfig{ 165 ID: uuid.Generate(), 166 Name: "sleep", 167 DNS: c.cfg, 168 } 169 170 if cgutil.UseV2 { 171 allocID := uuid.Generate() 172 task.Resources = testResources(allocID, "sleep") 173 } 174 175 cleanup := harness.MkAllocDir(task, false) 176 defer cleanup() 177 178 tc := &TaskConfig{ 179 Command: "/bin/sleep", 180 Args: []string{"9000"}, 181 } 182 require.NoError(t, task.EncodeConcreteDriverConfig(&tc)) 183 184 _, _, err := harness.StartTask(task) 185 require.NoError(t, err) 186 defer d.DestroyTask(task.ID, true) 187 188 dtestutil.TestTaskDNSConfig(t, harness, task.ID, c.cfg) 189 } 190 } 191 192 func TestExecDriver_Capabilities(t *testing.T) { 193 ci.Parallel(t) 194 ctestutils.ExecCompatible(t) 195 196 task := &drivers.TaskConfig{ 197 ID: uuid.Generate(), 198 Name: "sleep", 199 } 200 201 if cgutil.UseV2 { 202 allocID := uuid.Generate() 203 task.AllocID = allocID 204 task.Resources = testResources(allocID, "sleep") 205 } 206 207 for _, tc := range []struct { 208 Name string 209 CapAdd []string 210 CapDrop []string 211 AllowList string 212 StartError string 213 }{ 214 { 215 Name: "default-allowlist-add-allowed", 216 CapAdd: []string{"fowner", "mknod"}, 217 CapDrop: []string{"ALL"}, 218 }, 219 { 220 Name: "default-allowlist-add-forbidden", 221 CapAdd: []string{"net_admin"}, 222 StartError: "net_admin", 223 }, 224 { 225 Name: "default-allowlist-drop-existing", 226 CapDrop: []string{"FOWNER", "MKNOD", "NET_RAW"}, 227 }, 228 { 229 Name: "restrictive-allowlist-drop-all", 230 CapDrop: []string{"ALL"}, 231 AllowList: "FOWNER,MKNOD", 232 }, 233 { 234 Name: "restrictive-allowlist-add-allowed", 235 CapAdd: []string{"fowner", "mknod"}, 236 CapDrop: []string{"ALL"}, 237 AllowList: "fowner,mknod", 238 }, 239 { 240 Name: "restrictive-allowlist-add-forbidden", 241 CapAdd: []string{"net_admin", "mknod"}, 242 CapDrop: []string{"ALL"}, 243 AllowList: "fowner,mknod", 244 StartError: "net_admin", 245 }, 246 { 247 Name: "restrictive-allowlist-add-multiple-forbidden", 248 CapAdd: []string{"net_admin", "mknod", "CAP_SYS_TIME"}, 249 CapDrop: []string{"ALL"}, 250 AllowList: "fowner,mknod", 251 StartError: "net_admin, sys_time", 252 }, 253 { 254 Name: "permissive-allowlist", 255 CapAdd: []string{"net_admin", "mknod"}, 256 AllowList: "ALL", 257 }, 258 { 259 Name: "permissive-allowlist-add-all", 260 CapAdd: []string{"all"}, 261 AllowList: "ALL", 262 }, 263 } { 264 t.Run(tc.Name, func(t *testing.T) { 265 ctx, cancel := context.WithCancel(context.Background()) 266 defer cancel() 267 268 d := NewExecDriver(ctx, testlog.HCLogger(t)) 269 harness := dtestutil.NewDriverHarness(t, d) 270 defer harness.Kill() 271 272 config := &Config{ 273 NoPivotRoot: true, 274 DefaultModePID: executor.IsolationModePrivate, 275 DefaultModeIPC: executor.IsolationModePrivate, 276 } 277 278 if tc.AllowList != "" { 279 config.AllowCaps = strings.Split(tc.AllowList, ",") 280 } else { 281 // inherit HCL defaults if not set 282 config.AllowCaps = capabilities.NomadDefaults().Slice(true) 283 } 284 285 var data []byte 286 require.NoError(t, basePlug.MsgPackEncode(&data, config)) 287 baseConfig := &basePlug.Config{PluginConfig: data} 288 require.NoError(t, harness.SetConfig(baseConfig)) 289 290 cleanup := harness.MkAllocDir(task, false) 291 defer cleanup() 292 293 tCfg := &TaskConfig{ 294 Command: "/bin/sleep", 295 Args: []string{"9000"}, 296 } 297 if len(tc.CapAdd) > 0 { 298 tCfg.CapAdd = tc.CapAdd 299 } 300 if len(tc.CapDrop) > 0 { 301 tCfg.CapDrop = tc.CapDrop 302 } 303 require.NoError(t, task.EncodeConcreteDriverConfig(&tCfg)) 304 305 // check the start error against expectations 306 _, _, err := harness.StartTask(task) 307 if err == nil && tc.StartError != "" { 308 t.Fatalf("Expected error in start: %v", tc.StartError) 309 } else if err != nil { 310 if tc.StartError == "" { 311 require.NoError(t, err) 312 } else { 313 require.Contains(t, err.Error(), tc.StartError) 314 } 315 return 316 } 317 318 _ = d.DestroyTask(task.ID, true) 319 }) 320 } 321 }