github.com/iqoqo/nomad@v0.11.3-0.20200911112621-d7021c74d101/drivers/rawexec/driver_test.go (about) 1 package rawexec 2 3 import ( 4 "context" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "runtime" 10 "strconv" 11 "sync" 12 "syscall" 13 "testing" 14 "time" 15 16 ctestutil "github.com/hashicorp/nomad/client/testutil" 17 "github.com/hashicorp/nomad/helper/pluginutils/hclutils" 18 "github.com/hashicorp/nomad/helper/testlog" 19 "github.com/hashicorp/nomad/helper/testtask" 20 "github.com/hashicorp/nomad/helper/uuid" 21 basePlug "github.com/hashicorp/nomad/plugins/base" 22 "github.com/hashicorp/nomad/plugins/drivers" 23 dtestutil "github.com/hashicorp/nomad/plugins/drivers/testutils" 24 pstructs "github.com/hashicorp/nomad/plugins/shared/structs" 25 "github.com/hashicorp/nomad/testutil" 26 "github.com/stretchr/testify/require" 27 ) 28 29 func TestMain(m *testing.M) { 30 if !testtask.Run() { 31 os.Exit(m.Run()) 32 } 33 } 34 35 func newEnabledRawExecDriver(t *testing.T) *Driver { 36 d := NewRawExecDriver(testlog.HCLogger(t)).(*Driver) 37 d.config.Enabled = true 38 return d 39 } 40 41 func TestRawExecDriver_SetConfig(t *testing.T) { 42 t.Parallel() 43 require := require.New(t) 44 45 d := NewRawExecDriver(testlog.HCLogger(t)) 46 harness := dtestutil.NewDriverHarness(t, d) 47 defer harness.Kill() 48 49 bconfig := &basePlug.Config{} 50 51 // Disable raw exec. 52 config := &Config{} 53 54 var data []byte 55 require.NoError(basePlug.MsgPackEncode(&data, config)) 56 bconfig.PluginConfig = data 57 require.NoError(harness.SetConfig(bconfig)) 58 require.Exactly(config, d.(*Driver).config) 59 60 config.Enabled = true 61 config.NoCgroups = true 62 data = []byte{} 63 require.NoError(basePlug.MsgPackEncode(&data, config)) 64 bconfig.PluginConfig = data 65 require.NoError(harness.SetConfig(bconfig)) 66 require.Exactly(config, d.(*Driver).config) 67 68 config.NoCgroups = false 69 data = []byte{} 70 require.NoError(basePlug.MsgPackEncode(&data, config)) 71 bconfig.PluginConfig = data 72 require.NoError(harness.SetConfig(bconfig)) 73 require.Exactly(config, d.(*Driver).config) 74 } 75 76 func TestRawExecDriver_Fingerprint(t *testing.T) { 77 t.Parallel() 78 79 fingerprintTest := func(config *Config, expected *drivers.Fingerprint) func(t *testing.T) { 80 return func(t *testing.T) { 81 require := require.New(t) 82 d := newEnabledRawExecDriver(t) 83 harness := dtestutil.NewDriverHarness(t, d) 84 defer harness.Kill() 85 86 var data []byte 87 require.NoError(basePlug.MsgPackEncode(&data, config)) 88 bconfig := &basePlug.Config{ 89 PluginConfig: data, 90 } 91 require.NoError(harness.SetConfig(bconfig)) 92 93 fingerCh, err := harness.Fingerprint(context.Background()) 94 require.NoError(err) 95 select { 96 case result := <-fingerCh: 97 require.Equal(expected, result) 98 case <-time.After(time.Duration(testutil.TestMultiplier()) * time.Second): 99 require.Fail("timeout receiving fingerprint") 100 } 101 } 102 } 103 104 cases := []struct { 105 Name string 106 Conf Config 107 Expected drivers.Fingerprint 108 }{ 109 { 110 Name: "Disabled", 111 Conf: Config{ 112 Enabled: false, 113 }, 114 Expected: drivers.Fingerprint{ 115 Attributes: nil, 116 Health: drivers.HealthStateUndetected, 117 HealthDescription: "disabled", 118 }, 119 }, 120 { 121 Name: "Enabled", 122 Conf: Config{ 123 Enabled: true, 124 }, 125 Expected: drivers.Fingerprint{ 126 Attributes: map[string]*pstructs.Attribute{"driver.raw_exec": pstructs.NewBoolAttribute(true)}, 127 Health: drivers.HealthStateHealthy, 128 HealthDescription: drivers.DriverHealthy, 129 }, 130 }, 131 } 132 133 for _, tc := range cases { 134 t.Run(tc.Name, fingerprintTest(&tc.Conf, &tc.Expected)) 135 } 136 } 137 138 func TestRawExecDriver_StartWait(t *testing.T) { 139 t.Parallel() 140 require := require.New(t) 141 142 d := newEnabledRawExecDriver(t) 143 harness := dtestutil.NewDriverHarness(t, d) 144 defer harness.Kill() 145 task := &drivers.TaskConfig{ 146 ID: uuid.Generate(), 147 Name: "test", 148 } 149 150 tc := &TaskConfig{ 151 Command: testtask.Path(), 152 Args: []string{"sleep", "10ms"}, 153 } 154 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 155 testtask.SetTaskConfigEnv(task) 156 157 cleanup := harness.MkAllocDir(task, false) 158 defer cleanup() 159 160 handle, _, err := harness.StartTask(task) 161 require.NoError(err) 162 163 ch, err := harness.WaitTask(context.Background(), handle.Config.ID) 164 require.NoError(err) 165 166 var result *drivers.ExitResult 167 select { 168 case result = <-ch: 169 case <-time.After(5 * time.Second): 170 t.Fatal("timed out") 171 } 172 173 require.Zero(result.ExitCode) 174 require.Zero(result.Signal) 175 require.False(result.OOMKilled) 176 require.NoError(result.Err) 177 require.NoError(harness.DestroyTask(task.ID, true)) 178 } 179 180 func TestRawExecDriver_StartWaitRecoverWaitStop(t *testing.T) { 181 t.Parallel() 182 require := require.New(t) 183 184 d := newEnabledRawExecDriver(t) 185 harness := dtestutil.NewDriverHarness(t, d) 186 defer harness.Kill() 187 188 // Disable cgroups so test works without root 189 config := &Config{NoCgroups: true, Enabled: true} 190 var data []byte 191 require.NoError(basePlug.MsgPackEncode(&data, config)) 192 bconfig := &basePlug.Config{PluginConfig: data} 193 require.NoError(harness.SetConfig(bconfig)) 194 195 task := &drivers.TaskConfig{ 196 ID: uuid.Generate(), 197 Name: "sleep", 198 } 199 tc := &TaskConfig{ 200 Command: testtask.Path(), 201 Args: []string{"sleep", "100s"}, 202 } 203 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 204 205 testtask.SetTaskConfigEnv(task) 206 cleanup := harness.MkAllocDir(task, false) 207 defer cleanup() 208 209 handle, _, err := harness.StartTask(task) 210 require.NoError(err) 211 212 ch, err := harness.WaitTask(context.Background(), task.ID) 213 require.NoError(err) 214 215 var waitDone bool 216 var wg sync.WaitGroup 217 wg.Add(1) 218 go func() { 219 defer wg.Done() 220 result := <-ch 221 require.Error(result.Err) 222 waitDone = true 223 }() 224 225 originalStatus, err := d.InspectTask(task.ID) 226 require.NoError(err) 227 228 d.tasks.Delete(task.ID) 229 230 wg.Wait() 231 require.True(waitDone) 232 _, err = d.InspectTask(task.ID) 233 require.Equal(drivers.ErrTaskNotFound, err) 234 235 err = d.RecoverTask(handle) 236 require.NoError(err) 237 238 status, err := d.InspectTask(task.ID) 239 require.NoError(err) 240 require.Exactly(originalStatus, status) 241 242 ch, err = harness.WaitTask(context.Background(), task.ID) 243 require.NoError(err) 244 245 wg.Add(1) 246 waitDone = false 247 go func() { 248 defer wg.Done() 249 result := <-ch 250 require.NoError(result.Err) 251 require.NotZero(result.ExitCode) 252 require.Equal(9, result.Signal) 253 waitDone = true 254 }() 255 256 time.Sleep(300 * time.Millisecond) 257 require.NoError(d.StopTask(task.ID, 0, "SIGKILL")) 258 wg.Wait() 259 require.NoError(d.DestroyTask(task.ID, false)) 260 require.True(waitDone) 261 } 262 263 func TestRawExecDriver_Start_Wait_AllocDir(t *testing.T) { 264 t.Parallel() 265 require := require.New(t) 266 267 d := newEnabledRawExecDriver(t) 268 harness := dtestutil.NewDriverHarness(t, d) 269 defer harness.Kill() 270 271 task := &drivers.TaskConfig{ 272 ID: uuid.Generate(), 273 Name: "sleep", 274 } 275 276 cleanup := harness.MkAllocDir(task, false) 277 defer cleanup() 278 279 exp := []byte("win") 280 file := "output.txt" 281 outPath := fmt.Sprintf(`%s/%s`, task.TaskDir().SharedAllocDir, file) 282 283 tc := &TaskConfig{ 284 Command: testtask.Path(), 285 Args: []string{"sleep", "1s", "write", string(exp), outPath}, 286 } 287 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 288 testtask.SetTaskConfigEnv(task) 289 290 _, _, err := harness.StartTask(task) 291 require.NoError(err) 292 293 // Task should terminate quickly 294 waitCh, err := harness.WaitTask(context.Background(), task.ID) 295 require.NoError(err) 296 297 select { 298 case res := <-waitCh: 299 require.NoError(res.Err) 300 require.True(res.Successful()) 301 case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): 302 require.Fail("WaitTask timeout") 303 } 304 305 // Check that data was written to the shared alloc directory. 306 outputFile := filepath.Join(task.TaskDir().SharedAllocDir, file) 307 act, err := ioutil.ReadFile(outputFile) 308 require.NoError(err) 309 require.Exactly(exp, act) 310 require.NoError(harness.DestroyTask(task.ID, true)) 311 } 312 313 // This test creates a process tree such that without cgroups tracking the 314 // processes cleanup of the children would not be possible. Thus the test 315 // asserts that the processes get killed properly when using cgroups. 316 func TestRawExecDriver_Start_Kill_Wait_Cgroup(t *testing.T) { 317 ctestutil.ExecCompatible(t) 318 t.Parallel() 319 require := require.New(t) 320 pidFile := "pid" 321 322 d := newEnabledRawExecDriver(t) 323 harness := dtestutil.NewDriverHarness(t, d) 324 defer harness.Kill() 325 326 task := &drivers.TaskConfig{ 327 ID: uuid.Generate(), 328 Name: "sleep", 329 User: "root", 330 } 331 332 cleanup := harness.MkAllocDir(task, false) 333 defer cleanup() 334 335 tc := &TaskConfig{ 336 Command: testtask.Path(), 337 Args: []string{"fork/exec", pidFile, "pgrp", "0", "sleep", "20s"}, 338 } 339 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 340 testtask.SetTaskConfigEnv(task) 341 342 _, _, err := harness.StartTask(task) 343 require.NoError(err) 344 345 // Find the process 346 var pidData []byte 347 testutil.WaitForResult(func() (bool, error) { 348 var err error 349 pidData, err = ioutil.ReadFile(filepath.Join(task.TaskDir().Dir, pidFile)) 350 if err != nil { 351 return false, err 352 } 353 354 if len(pidData) == 0 { 355 return false, fmt.Errorf("pidFile empty") 356 } 357 358 return true, nil 359 }, func(err error) { 360 require.NoError(err) 361 }) 362 363 pid, err := strconv.Atoi(string(pidData)) 364 require.NoError(err, "failed to read pidData: %s", string(pidData)) 365 366 // Check the pid is up 367 process, err := os.FindProcess(pid) 368 require.NoError(err) 369 require.NoError(process.Signal(syscall.Signal(0))) 370 371 var wg sync.WaitGroup 372 wg.Add(1) 373 go func() { 374 defer wg.Done() 375 time.Sleep(1 * time.Second) 376 err := harness.StopTask(task.ID, 0, "") 377 378 // Can't rely on the ordering between wait and kill on CI/travis... 379 if !testutil.IsCI() { 380 require.NoError(err) 381 } 382 }() 383 384 // Task should terminate quickly 385 waitCh, err := harness.WaitTask(context.Background(), task.ID) 386 require.NoError(err) 387 select { 388 case res := <-waitCh: 389 require.False(res.Successful()) 390 case <-time.After(time.Duration(testutil.TestMultiplier()*5) * time.Second): 391 require.Fail("WaitTask timeout") 392 } 393 394 testutil.WaitForResult(func() (bool, error) { 395 if err := process.Signal(syscall.Signal(0)); err == nil { 396 return false, fmt.Errorf("process should not exist: %v", pid) 397 } 398 399 return true, nil 400 }, func(err error) { 401 require.NoError(err) 402 }) 403 404 wg.Wait() 405 require.NoError(harness.DestroyTask(task.ID, true)) 406 } 407 408 func TestRawExecDriver_Exec(t *testing.T) { 409 t.Parallel() 410 require := require.New(t) 411 412 d := newEnabledRawExecDriver(t) 413 harness := dtestutil.NewDriverHarness(t, d) 414 defer harness.Kill() 415 416 task := &drivers.TaskConfig{ 417 ID: uuid.Generate(), 418 Name: "sleep", 419 } 420 421 cleanup := harness.MkAllocDir(task, false) 422 defer cleanup() 423 424 tc := &TaskConfig{ 425 Command: testtask.Path(), 426 Args: []string{"sleep", "9000s"}, 427 } 428 require.NoError(task.EncodeConcreteDriverConfig(&tc)) 429 testtask.SetTaskConfigEnv(task) 430 431 _, _, err := harness.StartTask(task) 432 require.NoError(err) 433 434 if runtime.GOOS == "windows" { 435 // Exec a command that should work 436 res, err := harness.ExecTask(task.ID, []string{"cmd.exe", "/c", "echo", "hello"}, 1*time.Second) 437 require.NoError(err) 438 require.True(res.ExitResult.Successful()) 439 require.Equal(string(res.Stdout), "hello\r\n") 440 441 // Exec a command that should fail 442 res, err = harness.ExecTask(task.ID, []string{"cmd.exe", "/c", "stat", "notarealfile123abc"}, 1*time.Second) 443 require.NoError(err) 444 require.False(res.ExitResult.Successful()) 445 require.Contains(string(res.Stdout), "not recognized") 446 } else { 447 // Exec a command that should work 448 res, err := harness.ExecTask(task.ID, []string{"/usr/bin/stat", "/tmp"}, 1*time.Second) 449 require.NoError(err) 450 require.True(res.ExitResult.Successful()) 451 require.True(len(res.Stdout) > 100) 452 453 // Exec a command that should fail 454 res, err = harness.ExecTask(task.ID, []string{"/usr/bin/stat", "notarealfile123abc"}, 1*time.Second) 455 require.NoError(err) 456 require.False(res.ExitResult.Successful()) 457 require.Contains(string(res.Stdout), "No such file or directory") 458 } 459 460 require.NoError(harness.DestroyTask(task.ID, true)) 461 } 462 463 func TestConfig_ParseAllHCL(t *testing.T) { 464 cfgStr := ` 465 config { 466 command = "/bin/bash" 467 args = ["-c", "echo hello"] 468 }` 469 470 expected := &TaskConfig{ 471 Command: "/bin/bash", 472 Args: []string{"-c", "echo hello"}, 473 } 474 475 var tc *TaskConfig 476 hclutils.NewConfigParser(taskConfigSpec).ParseHCL(t, cfgStr, &tc) 477 478 require.EqualValues(t, expected, tc) 479 } 480 481 func TestRawExecDriver_Disabled(t *testing.T) { 482 t.Parallel() 483 require := require.New(t) 484 485 d := newEnabledRawExecDriver(t) 486 d.config.Enabled = false 487 488 harness := dtestutil.NewDriverHarness(t, d) 489 defer harness.Kill() 490 task := &drivers.TaskConfig{ 491 ID: uuid.Generate(), 492 Name: "test", 493 } 494 495 handle, _, err := harness.StartTask(task) 496 require.Error(err) 497 require.Contains(err.Error(), errDisabledDriver.Error()) 498 require.Nil(handle) 499 }