github.com/janma/nomad@v0.11.3/drivers/rawexec/driver_test.go (about) 1 package rawexec 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "runtime" 10 "strconv" 11 "sync" 12 "syscall" 13 "testing" 14 "time" 15 16 ctestutil "github.com/hashicorp/nomad/client/testutil" 17 "github.com/hashicorp/nomad/helper/pluginutils/hclutils" 18 "github.com/hashicorp/nomad/helper/testlog" 19 "github.com/hashicorp/nomad/helper/testtask" 20 "github.com/hashicorp/nomad/helper/uuid" 21 basePlug "github.com/hashicorp/nomad/plugins/base" 22 "github.com/hashicorp/nomad/plugins/drivers" 23 dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" 24 pstructs "github.com/hashicorp/nomad/plugins/shared/structs" 25 "github.com/hashicorp/nomad/testutil" 26 "github.com/stretchr/testify/require" 27 ) 28 29 func TestMain(m *testing.M) { 30 if !testtask.Run() { 31 os.Exit(m.Run()) 32 } 33 } 34 35 func newEnabledRawExecDriver(t *testing.T) *Driver { 36 ctx, cancel := context.WithCancel(context.Background()) 37 t.Cleanup(func() { cancel() }) 38 39 d := NewRawExecDriver(ctx, testlog.HCLogger(t)).(*Driver) 40 d.config.Enabled = true 41 return d 42 } 43 44 func TestRawExecDriver_SetConfig(t *testing.T) { 45 t.Parallel() 46 require := require.New(t) 47 48 ctx, cancel := context.WithCancel(context.Background()) 49 defer cancel() 50 51 d := NewRawExecDriver(ctx, testlog.HCLogger(t)) 52 harness := dtestutil.NewDriverHarness(t, d) 53 defer harness.Kill() 54 55 bconfig := &basePlug.Config{} 56 57 // Disable raw exec. 58 config := &Config{} 59 60 var data []byte 61 require.NoError(basePlug.MsgPackEncode(&data, config)) 62 bconfig.PluginConfig = data 63 require.NoError(harness.SetConfig(bconfig)) 64 require.Exactly(config, d.(*Driver).config) 65 66 config.Enabled = true 67 config.NoCgroups = true 68 data = []byte{} 69 require.NoError(basePlug.MsgPackEncode(&data, config)) 70 bconfig.PluginConfig = data 71 require.NoError(harness.SetConfig(bconfig)) 72 require.Exactly(config, d.(*Driver).config) 73 74 config.NoCgroups = false 75 data = []byte{} 76 require.NoError(basePlug.MsgPackEncode(&data, config)) 77 bconfig.PluginConfig = data 78 require.NoError(harness.SetConfig(bconfig)) 79 require.Exactly(config, d.(*Driver).config) 80 } 81 82 func TestRawExecDriver_Fingerprint(t *testing.T) { 83 t.Parallel() 84 85 fingerprintTest := func(config *Config, expected *drivers.Fingerprint) func(t *testing.T) { 86 return func(t *testing.T) { 87 require := require.New(t) 88 d := newEnabledRawExecDriver(t) 89 harness := dtestutil.NewDriverHarness(t, d) 90 defer harness.Kill() 91 92 var data []byte 93 require.NoError(basePlug.MsgPackEncode(&data, config)) 94 bconfig := &basePlug.Config{ 95 PluginConfig: data, 96 } 97 require.NoError(harness.SetConfig(bconfig)) 98 99 fingerCh, err := harness.Fingerprint(context.Background()) 100 require.NoError(err) 101 select { 102 case result := <-fingerCh: 103 require.Equal(expected, result) 104 case <-time.After(time.Duration(testutil.TestMultiplier()) * time.Second): 105 require.Fail("timeout receiving fingerprint") 106 } 107 } 108 } 109 110 cases := []struct { 111 Name string 112 Conf Config 113 Expected drivers.Fingerprint 114 }{ 115 { 116 Name: "Disabled", 117 Conf: Config{ 118 Enabled: false, 119 }, 120 Expected: drivers.Fingerprint{ 121 Attributes: nil, 122 Health: drivers.HealthStateUndetected, 123 HealthDescription: "disabled", 124 }, 125 }, 126 { 127 Name: "Enabled", 128 Conf: Config{ 129 Enabled: true, 130 }, 131 Expected: drivers.Fingerprint{ 132 Attributes: map[string]*pstructs.Attribute{"driver.raw_exec": pstructs.NewBoolAttribute(true)}, 133 Health: drivers.HealthStateHealthy, 134 HealthDescription: drivers.DriverHealthy, 135 }, 136 }, 137 } 138 139 for _, tc := range cases { 140 t.Run(tc.Name, fingerprintTest(&tc.Conf, &tc.Expected)) 141 } 142 } 143 144 func TestRawExecDriver_StartWait(t *testing.T) { 145 t.Parallel() 146 require := require.New(t) 147 148 d := newEnabledRawExecDriver(t) 149 harness := dtestutil.NewDriverHarness(t, d) 150 defer harness.Kill() 151 task := &drivers.TaskConfig{ 152 ID: uuid.Generate(), 153 Name: "test", 154 } 155 156 tc := &TaskConfig{ 157 Command: testtask.Path(), 158 Args: []string{"sleep", "10ms"}, 159 } 160 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 161 testtask.SetTaskConfigEnv(task) 162 163 cleanup := harness.MkAllocDir(task, false) 164 defer cleanup() 165 166 handle, _, err := harness.StartTask(task) 167 require.NoError(err) 168 169 ch, err := harness.WaitTask(context.Background(), handle.Config.ID) 170 require.NoError(err) 171 172 var result *drivers.ExitResult 173 select { 174 case result = <-ch: 175 case <-time.After(5 * time.Second): 176 t.Fatal("timed out") 177 } 178 179 require.Zero(result.ExitCode) 180 require.Zero(result.Signal) 181 require.False(result.OOMKilled) 182 require.NoError(result.Err) 183 require.NoError(harness.DestroyTask(task.ID, true)) 184 } 185 186 func TestRawExecDriver_StartWaitRecoverWaitStop(t *testing.T) { 187 t.Parallel() 188 require := require.New(t) 189 190 d := newEnabledRawExecDriver(t) 191 harness := dtestutil.NewDriverHarness(t, d) 192 defer harness.Kill() 193 194 // Disable cgroups so test works without root 195 config := &Config{NoCgroups: true, Enabled: true} 196 var data []byte 197 require.NoError(basePlug.MsgPackEncode(&data, config)) 198 bconfig := &basePlug.Config{PluginConfig: data} 199 require.NoError(harness.SetConfig(bconfig)) 200 201 task := &drivers.TaskConfig{ 202 ID: uuid.Generate(), 203 Name: "sleep", 204 } 205 tc := &TaskConfig{ 206 Command: testtask.Path(), 207 Args: []string{"sleep", "100s"}, 208 } 209 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 210 211 testtask.SetTaskConfigEnv(task) 212 cleanup := harness.MkAllocDir(task, false) 213 defer cleanup() 214 215 handle, _, err := harness.StartTask(task) 216 require.NoError(err) 217 218 ch, err := harness.WaitTask(context.Background(), task.ID) 219 require.NoError(err) 220 221 var waitDone bool 222 var wg sync.WaitGroup 223 wg.Add(1) 224 go func() { 225 defer wg.Done() 226 result := <-ch 227 require.Error(result.Err) 228 waitDone = true 229 }() 230 231 originalStatus, err := d.InspectTask(task.ID) 232 require.NoError(err) 233 234 d.tasks.Delete(task.ID) 235 236 wg.Wait() 237 require.True(waitDone) 238 _, err = d.InspectTask(task.ID) 239 require.Equal(drivers.ErrTaskNotFound, err) 240 241 err = d.RecoverTask(handle) 242 require.NoError(err) 243 244 status, err := d.InspectTask(task.ID) 245 require.NoError(err) 246 require.Exactly(originalStatus, status) 247 248 ch, err = harness.WaitTask(context.Background(), task.ID) 249 require.NoError(err) 250 251 wg.Add(1) 252 waitDone = false 253 go func() { 254 defer wg.Done() 255 result := <-ch 256 require.NoError(result.Err) 257 require.NotZero(result.ExitCode) 258 require.Equal(9, result.Signal) 259 waitDone = true 260 }() 261 262 time.Sleep(300 * time.Millisecond) 263 require.NoError(d.StopTask(task.ID, 0, "SIGKILL")) 264 wg.Wait() 265 require.NoError(d.DestroyTask(task.ID, false)) 266 require.True(waitDone) 267 } 268 269 func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) { 270 t.Parallel() 271 require := require.New(t) 272 273 d := newEnabledRawExecDriver(t) 274 harness := dtestutil.NewDriverHarness(t, d) 275 defer harness.Kill() 276 277 task := &drivers.TaskConfig{ 278 ID: uuid.Generate(), 279 Name: "sleep", 280 } 281 282 cleanup := harness.MkAllocDir(task, false) 283 defer cleanup() 284 285 exp := []byte("win") 286 file := "output.txt" 287 outPath := fmt.Sprintf(`%s/%s`, task.TaskDir().SharedAllocDir, file) 288 289 tc := &TaskConfig{ 290 Command: testtask.Path(), 291 Args: []string{"sleep", "1s", "write", string(exp), outPath}, 292 } 293 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 294 testtask.SetTaskConfigEnv(task) 295 296 _, _, err := harness.StartTask(task) 297 require.NoError(err) 298 299 // Task should terminate quickly 300 waitCh, err := harness.WaitTask(context.Background(), task.ID) 301 require.NoError(err) 302 303 select { 304 case res := <-waitCh: 305 require.NoError(res.Err) 306 require.True(res.Successful()) 307 case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): 308 require.Fail("WaitTask timeout") 309 } 310 311 // Check that data was written to the shared alloc directory. 312 outputFile := filepath.Join(task.TaskDir().SharedAllocDir, file) 313 act, err := ioutil.ReadFile(outputFile) 314 require.NoError(err) 315 require.Exactly(exp, act) 316 require.NoError(harness.DestroyTask(task.ID, true)) 317 } 318 319 // This test creates a process tree such that without cgroups tracking the 320 // processes cleanup of the children would not be possible. Thus the test 321 // asserts that the processes get killed properly when using cgroups. 322 func TestRawExecDriver_Start_Kill_Wait_Cgroup(t *testing.T) { 323 ctestutil.ExecCompatible(t) 324 t.Parallel() 325 require := require.New(t) 326 pidFile := "pid" 327 328 d := newEnabledRawExecDriver(t) 329 harness := dtestutil.NewDriverHarness(t, d) 330 defer harness.Kill() 331 332 task := &drivers.TaskConfig{ 333 ID: uuid.Generate(), 334 Name: "sleep", 335 User: "root", 336 } 337 338 cleanup := harness.MkAllocDir(task, false) 339 defer cleanup() 340 341 tc := &TaskConfig{ 342 Command: testtask.Path(), 343 Args: []string{"fork/exec", pidFile, "pgrp", "0", "sleep", "20s"}, 344 } 345 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 346 testtask.SetTaskConfigEnv(task) 347 348 _, _, err := harness.StartTask(task) 349 require.NoError(err) 350 351 // Find the process 352 var pidData []byte 353 testutil.WaitForResult(func() (bool, error) { 354 var err error 355 pidData, err = ioutil.ReadFile(filepath.Join(task.TaskDir().Dir, pidFile)) 356 if err != nil { 357 return false, err 358 } 359 360 if len(pidData) == 0 { 361 return false, fmt.Errorf("pidFile empty") 362 } 363 364 return true, nil 365 }, func(err error) { 366 require.NoError(err) 367 }) 368 369 pid, err := strconv.Atoi(string(pidData)) 370 require.NoError(err, "failed to read pidData: %s", string(pidData)) 371 372 // Check the pid is up 373 process, err := os.FindProcess(pid) 374 require.NoError(err) 375 require.NoError(process.Signal(syscall.Signal(0))) 376 377 var wg sync.WaitGroup 378 wg.Add(1) 379 go func() { 380 defer wg.Done() 381 time.Sleep(1 * time.Second) 382 err := harness.StopTask(task.ID, 0, "") 383 384 // Can't rely on the ordering between wait and kill on CI/travis... 385 if !testutil.IsCI() { 386 require.NoError(err) 387 } 388 }() 389 390 // Task should terminate quickly 391 waitCh, err := harness.WaitTask(context.Background(), task.ID) 392 require.NoError(err) 393 select { 394 case res := <-waitCh: 395 require.False(res.Successful()) 396 case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): 397 require.Fail("WaitTask timeout") 398 } 399 400 testutil.WaitForResult(func() (bool, error) { 401 if err := process.Signal(syscall.Signal(0)); err == nil { 402 return false, fmt.Errorf("process should not exist: %v", pid) 403 } 404 405 return true, nil 406 }, func(err error) { 407 require.NoError(err) 408 }) 409 410 wg.Wait() 411 require.NoError(harness.DestroyTask(task.ID, true)) 412 } 413 414 func TestRawExecDriver_Exec(t *testing.T) { 415 t.Parallel() 416 require := require.New(t) 417 418 d := newEnabledRawExecDriver(t) 419 harness := dtestutil.NewDriverHarness(t, d) 420 defer harness.Kill() 421 422 task := &drivers.TaskConfig{ 423 ID: uuid.Generate(), 424 Name: "sleep", 425 } 426 427 cleanup := harness.MkAllocDir(task, false) 428 defer cleanup() 429 430 tc := &TaskConfig{ 431 Command: testtask.Path(), 432 Args: []string{"sleep", "9000s"}, 433 } 434 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 435 testtask.SetTaskConfigEnv(task) 436 437 _, _, err := harness.StartTask(task) 438 require.NoError(err) 439 440 if runtime.GOOS == "windows" { 441 // Exec a command that should work 442 res, err := harness.ExecTask(task.ID, []string{"cmd.exe", "/c", "echo", "hello"}, 1*time.Second) 443 require.NoError(err) 444 require.True(res.ExitResult.Successful()) 445 require.Equal(string(res.Stdout), "hello\r\n") 446 447 // Exec a command that should fail 448 res, err = harness.ExecTask(task.ID, []string{"cmd.exe", "/c", "stat", "notarealfile123abc"}, 1*time.Second) 449 require.NoError(err) 450 require.False(res.ExitResult.Successful()) 451 require.Contains(string(res.Stdout), "not recognized") 452 } else { 453 // Exec a command that should work 454 res, err := harness.ExecTask(task.ID, []string{"/usr/bin/stat", "/tmp"}, 1*time.Second) 455 require.NoError(err) 456 require.True(res.ExitResult.Successful()) 457 require.True(len(res.Stdout) > 100) 458 459 // Exec a command that should fail 460 res, err = harness.ExecTask(task.ID, []string{"/usr/bin/stat", "notarealfile123abc"}, 1*time.Second) 461 require.NoError(err) 462 require.False(res.ExitResult.Successful()) 463 require.Contains(string(res.Stdout), "No such file or directory") 464 } 465 466 require.NoError(harness.DestroyTask(task.ID, true)) 467 } 468 469 func TestConfig_ParseAllHCL(t *testing.T) { 470 cfgStr := ` 471 config { 472 command = "/bin/bash" 473 args = ["-c", "echo hello"] 474 }` 475 476 expected := &TaskConfig{ 477 Command: "/bin/bash", 478 Args: []string{"-c", "echo hello"}, 479 } 480 481 var tc *TaskConfig 482 hclutils.NewConfigParser(taskConfigSpec).ParseHCL(t, cfgStr, &tc) 483 484 require.EqualValues(t, expected, tc) 485 } 486 487 func TestRawExecDriver_Disabled(t *testing.T) { 488 t.Parallel() 489 require := require.New(t) 490 491 d := newEnabledRawExecDriver(t) 492 d.config.Enabled = false 493 494 harness := dtestutil.NewDriverHarness(t, d) 495 defer harness.Kill() 496 task := &drivers.TaskConfig{ 497 ID: uuid.Generate(), 498 Name: "test", 499 } 500 501 handle, _, err := harness.StartTask(task) 502 require.Error(err) 503 require.Contains(err.Error(), errDisabledDriver.Error()) 504 require.Nil(handle) 505 }