github.com/blixtra/nomad@v0.7.2-0.20171221000451-da9a1d7bb050/client/driver/docker_test.go (about) 1 package driver 2 3 import ( 4 "fmt" 5 "io/ioutil" 6 "math/rand" 7 "os" 8 "path/filepath" 9 "reflect" 10 "runtime" 11 "runtime/debug" 12 "sort" 13 "strconv" 14 "strings" 15 "testing" 16 "time" 17 18 docker "github.com/fsouza/go-dockerclient" 19 "github.com/hashicorp/consul/lib/freeport" 20 sockaddr "github.com/hashicorp/go-sockaddr" 21 "github.com/hashicorp/nomad/client/allocdir" 22 "github.com/hashicorp/nomad/client/config" 23 "github.com/hashicorp/nomad/client/driver/env" 24 cstructs "github.com/hashicorp/nomad/client/structs" 25 "github.com/hashicorp/nomad/client/testutil" 26 "github.com/hashicorp/nomad/helper/uuid" 27 "github.com/hashicorp/nomad/nomad/mock" 28 "github.com/hashicorp/nomad/nomad/structs" 29 tu "github.com/hashicorp/nomad/testutil" 30 "github.com/stretchr/testify/assert" 31 ) 32 33 func dockerIsRemote(t *testing.T) bool { 34 client, err := docker.NewClientFromEnv() 35 if err != nil { 36 return false 37 } 38 39 // Technically this could be a local tcp socket but for testing purposes 40 // we'll just assume that tcp is only used for remote connections. 41 if client.Endpoint()[0:3] == "tcp" { 42 return true 43 } 44 return false 45 } 46 47 // Returns a task with a reserved and dynamic port. The ports are returned 48 // respectively. 49 func dockerTask(t *testing.T) (*structs.Task, int, int) { 50 ports := freeport.GetT(t, 2) 51 dockerReserved := ports[0] 52 dockerDynamic := ports[1] 53 return &structs.Task{ 54 Name: "redis-demo", 55 Driver: "docker", 56 Config: map[string]interface{}{ 57 "image": "busybox", 58 "load": "busybox.tar", 59 "command": "/bin/nc", 60 "args": []string{"-l", "127.0.0.1", "-p", "0"}, 61 }, 62 LogConfig: &structs.LogConfig{ 63 MaxFiles: 10, 64 MaxFileSizeMB: 10, 65 }, 66 Resources: &structs.Resources{ 67 MemoryMB: 256, 68 CPU: 512, 69 Networks: []*structs.NetworkResource{ 70 { 71 IP: "127.0.0.1", 72 ReservedPorts: []structs.Port{{Label: "main", Value: dockerReserved}}, 73 DynamicPorts: []structs.Port{{Label: "REDIS", Value: dockerDynamic}}, 74 }, 75 }, 76 }, 77 }, dockerReserved, dockerDynamic 78 } 79 80 // dockerSetup does all of the basic setup you need to get a running docker 81 // process up and running for testing. Use like: 82 // 83 // task := taskTemplate() 84 // // do custom task configuration 85 // client, handle, cleanup := dockerSetup(t, task) 86 // defer cleanup() 87 // // do test stuff 88 // 89 // If there is a problem during setup this function will abort or skip the test 90 // and indicate the reason. 91 func dockerSetup(t *testing.T, task *structs.Task) (*docker.Client, *DockerHandle, func()) { 92 client := newTestDockerClient(t) 93 return dockerSetupWithClient(t, task, client) 94 } 95 96 func testDockerDriverContexts(t *testing.T, task *structs.Task) *testContext { 97 tctx := testDriverContexts(t, task) 98 99 // Drop the delay 100 tctx.DriverCtx.config.Options = make(map[string]string) 101 tctx.DriverCtx.config.Options[dockerImageRemoveDelayConfigOption] = "1s" 102 103 return tctx 104 } 105 106 func dockerSetupWithClient(t *testing.T, task *structs.Task, client *docker.Client) (*docker.Client, *DockerHandle, func()) { 107 t.Helper() 108 tctx := testDockerDriverContexts(t, task) 109 //tctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 110 driver := NewDockerDriver(tctx.DriverCtx) 111 copyImage(t, tctx.ExecCtx.TaskDir, "busybox.tar") 112 113 presp, err := driver.Prestart(tctx.ExecCtx, task) 114 if err != nil { 115 driver.Cleanup(tctx.ExecCtx, presp.CreatedResources) 116 tctx.AllocDir.Destroy() 117 t.Fatalf("error in prestart: %v", err) 118 } 119 // Update the exec ctx with the driver network env vars 120 tctx.ExecCtx.TaskEnv = tctx.EnvBuilder.SetDriverNetwork(presp.Network).Build() 121 122 sresp, err := driver.Start(tctx.ExecCtx, task) 123 if err != nil { 124 driver.Cleanup(tctx.ExecCtx, presp.CreatedResources) 125 tctx.AllocDir.Destroy() 126 t.Fatalf("Failed to start driver: %s\nStack\n%s", err, debug.Stack()) 127 } 128 129 if sresp.Handle == nil { 130 driver.Cleanup(tctx.ExecCtx, presp.CreatedResources) 131 tctx.AllocDir.Destroy() 132 t.Fatalf("handle is nil\nStack\n%s", debug.Stack()) 133 } 134 135 // At runtime this is handled by TaskRunner 136 tctx.ExecCtx.TaskEnv = tctx.EnvBuilder.SetDriverNetwork(sresp.Network).Build() 137 138 cleanup := func() { 139 driver.Cleanup(tctx.ExecCtx, presp.CreatedResources) 140 sresp.Handle.Kill() 141 tctx.AllocDir.Destroy() 142 } 143 144 return client, sresp.Handle.(*DockerHandle), cleanup 145 } 146 147 func newTestDockerClient(t *testing.T) *docker.Client { 148 t.Helper() 149 if !testutil.DockerIsConnected(t) { 150 t.Skip("Docker not connected") 151 } 152 153 client, err := docker.NewClientFromEnv() 154 if err != nil { 155 t.Fatalf("Failed to initialize client: %s\nStack\n%s", err, debug.Stack()) 156 } 157 return client 158 } 159 160 // This test should always pass, even if docker daemon is not available 161 func TestDockerDriver_Fingerprint(t *testing.T) { 162 if !tu.IsTravis() { 163 t.Parallel() 164 } 165 ctx := testDockerDriverContexts(t, &structs.Task{Name: "foo", Driver: "docker", Resources: basicResources}) 166 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 167 defer ctx.AllocDir.Destroy() 168 d := NewDockerDriver(ctx.DriverCtx) 169 node := &structs.Node{ 170 Attributes: make(map[string]string), 171 } 172 apply, err := d.Fingerprint(&config.Config{}, node) 173 if err != nil { 174 t.Fatalf("err: %v", err) 175 } 176 if apply != testutil.DockerIsConnected(t) { 177 t.Fatalf("Fingerprinter should detect when docker is available") 178 } 179 if node.Attributes["driver.docker"] != "1" { 180 t.Log("Docker daemon not available. The remainder of the docker tests will be skipped.") 181 } 182 t.Logf("Found docker version %s", node.Attributes["driver.docker.version"]) 183 } 184 185 // TestDockerDriver_Fingerprint_Bridge asserts that if Docker is running we set 186 // the bridge network's IP as a node attribute. See #2785 187 func TestDockerDriver_Fingerprint_Bridge(t *testing.T) { 188 if !tu.IsTravis() { 189 t.Parallel() 190 } 191 if !testutil.DockerIsConnected(t) { 192 t.Skip("requires Docker") 193 } 194 if runtime.GOOS != "linux" { 195 t.Skip("expect only on linux") 196 } 197 198 // This seems fragile, so we might need to reconsider this test if it 199 // proves flaky 200 expectedAddr, err := sockaddr.GetInterfaceIP("docker0") 201 if err != nil { 202 t.Fatalf("unable to get ip for docker0: %v", err) 203 } 204 if expectedAddr == "" { 205 t.Fatalf("unable to get ip for docker bridge") 206 } 207 208 conf := testConfig(t) 209 conf.Node = mock.Node() 210 dd := NewDockerDriver(NewDriverContext("", "", conf, conf.Node, testLogger(), nil)) 211 ok, err := dd.Fingerprint(conf, conf.Node) 212 if err != nil { 213 t.Fatalf("error fingerprinting docker: %v", err) 214 } 215 if !ok { 216 t.Fatalf("expected Docker to be enabled but false was returned") 217 } 218 219 if found := conf.Node.Attributes["driver.docker.bridge_ip"]; found != expectedAddr { 220 t.Fatalf("expected bridge ip %q but found: %q", expectedAddr, found) 221 } 222 t.Logf("docker bridge ip: %q", conf.Node.Attributes["driver.docker.bridge_ip"]) 223 } 224 225 func TestDockerDriver_StartOpen_Wait(t *testing.T) { 226 if !tu.IsTravis() { 227 t.Parallel() 228 } 229 if !testutil.DockerIsConnected(t) { 230 t.Skip("Docker not connected") 231 } 232 233 task := &structs.Task{ 234 Name: "nc-demo", 235 Driver: "docker", 236 Config: map[string]interface{}{ 237 "load": "busybox.tar", 238 "image": "busybox", 239 "command": "/bin/nc", 240 "args": []string{"-l", "127.0.0.1", "-p", "0"}, 241 }, 242 LogConfig: &structs.LogConfig{ 243 MaxFiles: 10, 244 MaxFileSizeMB: 10, 245 }, 246 Resources: basicResources, 247 } 248 249 ctx := testDockerDriverContexts(t, task) 250 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 251 defer ctx.AllocDir.Destroy() 252 d := NewDockerDriver(ctx.DriverCtx) 253 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 254 255 _, err := d.Prestart(ctx.ExecCtx, task) 256 if err != nil { 257 t.Fatalf("error in prestart: %v", err) 258 } 259 260 resp, err := d.Start(ctx.ExecCtx, task) 261 if err != nil { 262 t.Fatalf("err: %v", err) 263 } 264 if resp.Handle == nil { 265 t.Fatalf("missing handle") 266 } 267 defer resp.Handle.Kill() 268 269 // Attempt to open 270 resp2, err := d.Open(ctx.ExecCtx, resp.Handle.ID()) 271 if err != nil { 272 t.Fatalf("err: %v", err) 273 } 274 if resp2 == nil { 275 t.Fatalf("missing handle") 276 } 277 } 278 279 func TestDockerDriver_Start_Wait(t *testing.T) { 280 if !tu.IsTravis() { 281 t.Parallel() 282 } 283 if !testutil.DockerIsConnected(t) { 284 t.Skip("Docker not connected") 285 } 286 task := &structs.Task{ 287 Name: "nc-demo", 288 Driver: "docker", 289 Config: map[string]interface{}{ 290 "load": "busybox.tar", 291 "image": "busybox", 292 "command": "/bin/echo", 293 "args": []string{"hello"}, 294 }, 295 Resources: &structs.Resources{ 296 MemoryMB: 256, 297 CPU: 512, 298 }, 299 LogConfig: &structs.LogConfig{ 300 MaxFiles: 10, 301 MaxFileSizeMB: 10, 302 }, 303 } 304 305 _, handle, cleanup := dockerSetup(t, task) 306 defer cleanup() 307 308 // Update should be a no-op 309 err := handle.Update(task) 310 if err != nil { 311 t.Fatalf("err: %v", err) 312 } 313 314 select { 315 case res := <-handle.WaitCh(): 316 if !res.Successful() { 317 t.Fatalf("err: %v", res) 318 } 319 case <-time.After(time.Duration(tu.TestMultiplier()*5) * time.Second): 320 t.Fatalf("timeout") 321 } 322 } 323 324 // TestDockerDriver_Start_StoppedContainer asserts that Nomad will detect a 325 // stopped task container, remove it, and start a new container. 326 // 327 // See https://github.com/hashicorp/nomad/issues/3419 328 func TestDockerDriver_Start_StoppedContainer(t *testing.T) { 329 if !tu.IsTravis() { 330 t.Parallel() 331 } 332 if !testutil.DockerIsConnected(t) { 333 t.Skip("Docker not connected") 334 } 335 task := &structs.Task{ 336 Name: "nc-demo", 337 Driver: "docker", 338 Config: map[string]interface{}{ 339 "load": "busybox.tar", 340 "image": "busybox", 341 "command": "sleep", 342 "args": []string{"9000"}, 343 }, 344 Resources: &structs.Resources{ 345 MemoryMB: 100, 346 CPU: 100, 347 }, 348 LogConfig: &structs.LogConfig{ 349 MaxFiles: 1, 350 MaxFileSizeMB: 10, 351 }, 352 } 353 354 tctx := testDockerDriverContexts(t, task) 355 defer tctx.AllocDir.Destroy() 356 357 copyImage(t, tctx.ExecCtx.TaskDir, "busybox.tar") 358 client := newTestDockerClient(t) 359 driver := NewDockerDriver(tctx.DriverCtx).(*DockerDriver) 360 driverConfig := &DockerDriverConfig{ImageName: "busybox", LoadImage: "busybox.tar"} 361 if _, err := driver.loadImage(driverConfig, client, tctx.ExecCtx.TaskDir); err != nil { 362 t.Fatalf("error loading image: %v", err) 363 } 364 365 // Create a container of the same name but don't start it. This mimics 366 // the case of dockerd getting restarted and stopping containers while 367 // Nomad is watching them. 368 opts := docker.CreateContainerOptions{ 369 Name: fmt.Sprintf("%s-%s", task.Name, tctx.DriverCtx.allocID), 370 Config: &docker.Config{ 371 Image: "busybox", 372 Cmd: []string{"sleep", "9000"}, 373 }, 374 } 375 if _, err := client.CreateContainer(opts); err != nil { 376 t.Fatalf("error creating initial container: %v", err) 377 } 378 379 // Now assert that the driver can still start normally 380 presp, err := driver.Prestart(tctx.ExecCtx, task) 381 if err != nil { 382 driver.Cleanup(tctx.ExecCtx, presp.CreatedResources) 383 t.Fatalf("error in prestart: %v", err) 384 } 385 defer driver.Cleanup(tctx.ExecCtx, presp.CreatedResources) 386 387 sresp, err := driver.Start(tctx.ExecCtx, task) 388 if err != nil { 389 t.Fatalf("failed to start driver: %s", err) 390 } 391 handle := sresp.Handle.(*DockerHandle) 392 waitForExist(t, client, handle) 393 handle.Kill() 394 } 395 396 func TestDockerDriver_Start_LoadImage(t *testing.T) { 397 if !tu.IsTravis() { 398 t.Parallel() 399 } 400 if !testutil.DockerIsConnected(t) { 401 t.Skip("Docker not connected") 402 } 403 task := &structs.Task{ 404 Name: "busybox-demo", 405 Driver: "docker", 406 Config: map[string]interface{}{ 407 "image": "busybox", 408 "load": "busybox.tar", 409 "command": "/bin/sh", 410 "args": []string{ 411 "-c", 412 "echo hello > $NOMAD_TASK_DIR/output", 413 }, 414 }, 415 LogConfig: &structs.LogConfig{ 416 MaxFiles: 10, 417 MaxFileSizeMB: 10, 418 }, 419 Resources: &structs.Resources{ 420 MemoryMB: 256, 421 CPU: 512, 422 }, 423 } 424 425 ctx := testDockerDriverContexts(t, task) 426 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 427 defer ctx.AllocDir.Destroy() 428 d := NewDockerDriver(ctx.DriverCtx) 429 430 // Copy the image into the task's directory 431 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 432 433 _, err := d.Prestart(ctx.ExecCtx, task) 434 if err != nil { 435 t.Fatalf("error in prestart: %v", err) 436 } 437 resp, err := d.Start(ctx.ExecCtx, task) 438 if err != nil { 439 t.Fatalf("err: %v", err) 440 } 441 defer resp.Handle.Kill() 442 443 select { 444 case res := <-resp.Handle.WaitCh(): 445 if !res.Successful() { 446 t.Fatalf("err: %v", res) 447 } 448 case <-time.After(time.Duration(tu.TestMultiplier()*5) * time.Second): 449 t.Fatalf("timeout") 450 } 451 452 // Check that data was written to the shared alloc directory. 453 outputFile := filepath.Join(ctx.ExecCtx.TaskDir.LocalDir, "output") 454 act, err := ioutil.ReadFile(outputFile) 455 if err != nil { 456 t.Fatalf("Couldn't read expected output: %v", err) 457 } 458 459 exp := "hello" 460 if strings.TrimSpace(string(act)) != exp { 461 t.Fatalf("Command outputted %v; want %v", act, exp) 462 } 463 464 } 465 466 func TestDockerDriver_Start_BadPull_Recoverable(t *testing.T) { 467 if !tu.IsTravis() { 468 t.Parallel() 469 } 470 if !testutil.DockerIsConnected(t) { 471 t.Skip("Docker not connected") 472 } 473 task := &structs.Task{ 474 Name: "busybox-demo", 475 Driver: "docker", 476 Config: map[string]interface{}{ 477 "image": "127.0.1.1:32121/foo", // bad path 478 "command": "/bin/echo", 479 "args": []string{ 480 "hello", 481 }, 482 }, 483 LogConfig: &structs.LogConfig{ 484 MaxFiles: 10, 485 MaxFileSizeMB: 10, 486 }, 487 Resources: &structs.Resources{ 488 MemoryMB: 256, 489 CPU: 512, 490 }, 491 } 492 493 ctx := testDockerDriverContexts(t, task) 494 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 495 defer ctx.AllocDir.Destroy() 496 d := NewDockerDriver(ctx.DriverCtx) 497 498 _, err := d.Prestart(ctx.ExecCtx, task) 499 if err == nil { 500 t.Fatalf("want error in prestart: %v", err) 501 } 502 503 if rerr, ok := err.(*structs.RecoverableError); !ok { 504 t.Fatalf("want recoverable error: %+v", err) 505 } else if !rerr.IsRecoverable() { 506 t.Fatalf("error not recoverable: %+v", err) 507 } 508 } 509 510 func TestDockerDriver_Start_Wait_AllocDir(t *testing.T) { 511 if !tu.IsTravis() { 512 t.Parallel() 513 } 514 // This test requires that the alloc dir be mounted into docker as a volume. 515 // Because this cannot happen when docker is run remotely, e.g. when running 516 // docker in a VM, we skip this when we detect Docker is being run remotely. 517 if !testutil.DockerIsConnected(t) || dockerIsRemote(t) { 518 t.Skip("Docker not connected") 519 } 520 521 exp := []byte{'w', 'i', 'n'} 522 file := "output.txt" 523 task := &structs.Task{ 524 Name: "nc-demo", 525 Driver: "docker", 526 Config: map[string]interface{}{ 527 "image": "busybox", 528 "load": "busybox.tar", 529 "command": "/bin/sh", 530 "args": []string{ 531 "-c", 532 fmt.Sprintf(`sleep 1; echo -n %s > $%s/%s`, 533 string(exp), env.AllocDir, file), 534 }, 535 }, 536 LogConfig: &structs.LogConfig{ 537 MaxFiles: 10, 538 MaxFileSizeMB: 10, 539 }, 540 Resources: &structs.Resources{ 541 MemoryMB: 256, 542 CPU: 512, 543 }, 544 } 545 546 ctx := testDockerDriverContexts(t, task) 547 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 548 defer ctx.AllocDir.Destroy() 549 d := NewDockerDriver(ctx.DriverCtx) 550 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 551 552 _, err := d.Prestart(ctx.ExecCtx, task) 553 if err != nil { 554 t.Fatalf("error in prestart: %v", err) 555 } 556 resp, err := d.Start(ctx.ExecCtx, task) 557 if err != nil { 558 t.Fatalf("err: %v", err) 559 } 560 defer resp.Handle.Kill() 561 562 select { 563 case res := <-resp.Handle.WaitCh(): 564 if !res.Successful() { 565 t.Fatalf("err: %v", res) 566 } 567 case <-time.After(time.Duration(tu.TestMultiplier()*5) * time.Second): 568 t.Fatalf("timeout") 569 } 570 571 // Check that data was written to the shared alloc directory. 572 outputFile := filepath.Join(ctx.AllocDir.SharedDir, file) 573 act, err := ioutil.ReadFile(outputFile) 574 if err != nil { 575 t.Fatalf("Couldn't read expected output: %v", err) 576 } 577 578 if !reflect.DeepEqual(act, exp) { 579 t.Fatalf("Command outputted %v; want %v", act, exp) 580 } 581 } 582 583 func TestDockerDriver_Start_Kill_Wait(t *testing.T) { 584 if !tu.IsTravis() { 585 t.Parallel() 586 } 587 if !testutil.DockerIsConnected(t) { 588 t.Skip("Docker not connected") 589 } 590 task := &structs.Task{ 591 Name: "nc-demo", 592 Driver: "docker", 593 Config: map[string]interface{}{ 594 "image": "busybox", 595 "load": "busybox.tar", 596 "command": "/bin/sleep", 597 "args": []string{"10"}, 598 }, 599 LogConfig: &structs.LogConfig{ 600 MaxFiles: 10, 601 MaxFileSizeMB: 10, 602 }, 603 Resources: basicResources, 604 } 605 606 _, handle, cleanup := dockerSetup(t, task) 607 defer cleanup() 608 609 go func() { 610 time.Sleep(100 * time.Millisecond) 611 err := handle.Kill() 612 if err != nil { 613 t.Fatalf("err: %v", err) 614 } 615 }() 616 617 select { 618 case res := <-handle.WaitCh(): 619 if res.Successful() { 620 t.Fatalf("should err: %v", res) 621 } 622 case <-time.After(time.Duration(tu.TestMultiplier()*10) * time.Second): 623 t.Fatalf("timeout") 624 } 625 } 626 627 func TestDockerDriver_StartN(t *testing.T) { 628 if !tu.IsTravis() { 629 t.Parallel() 630 } 631 if !testutil.DockerIsConnected(t) { 632 t.Skip("Docker not connected") 633 } 634 635 task1, _, _ := dockerTask(t) 636 task2, _, _ := dockerTask(t) 637 task3, _, _ := dockerTask(t) 638 taskList := []*structs.Task{task1, task2, task3} 639 640 handles := make([]DriverHandle, len(taskList)) 641 642 t.Logf("Starting %d tasks", len(taskList)) 643 644 // Let's spin up a bunch of things 645 for idx, task := range taskList { 646 ctx := testDockerDriverContexts(t, task) 647 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 648 defer ctx.AllocDir.Destroy() 649 d := NewDockerDriver(ctx.DriverCtx) 650 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 651 652 _, err := d.Prestart(ctx.ExecCtx, task) 653 if err != nil { 654 t.Fatalf("error in prestart #%d: %v", idx+1, err) 655 } 656 resp, err := d.Start(ctx.ExecCtx, task) 657 if err != nil { 658 t.Errorf("Failed starting task #%d: %s", idx+1, err) 659 continue 660 } 661 handles[idx] = resp.Handle 662 } 663 664 t.Log("All tasks are started. Terminating...") 665 666 for idx, handle := range handles { 667 if handle == nil { 668 t.Errorf("Bad handle for task #%d", idx+1) 669 continue 670 } 671 672 err := handle.Kill() 673 if err != nil { 674 t.Errorf("Failed stopping task #%d: %s", idx+1, err) 675 } 676 } 677 678 t.Log("Test complete!") 679 } 680 681 func TestDockerDriver_StartNVersions(t *testing.T) { 682 if !tu.IsTravis() { 683 t.Parallel() 684 } 685 if !testutil.DockerIsConnected(t) { 686 t.Skip("Docker not connected") 687 } 688 689 task1, _, _ := dockerTask(t) 690 task1.Config["image"] = "busybox" 691 task1.Config["load"] = "busybox.tar" 692 693 task2, _, _ := dockerTask(t) 694 task2.Config["image"] = "busybox:musl" 695 task2.Config["load"] = "busybox_musl.tar" 696 697 task3, _, _ := dockerTask(t) 698 task3.Config["image"] = "busybox:glibc" 699 task3.Config["load"] = "busybox_glibc.tar" 700 701 taskList := []*structs.Task{task1, task2, task3} 702 703 handles := make([]DriverHandle, len(taskList)) 704 705 t.Logf("Starting %d tasks", len(taskList)) 706 707 // Let's spin up a bunch of things 708 for idx, task := range taskList { 709 ctx := testDockerDriverContexts(t, task) 710 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 711 defer ctx.AllocDir.Destroy() 712 d := NewDockerDriver(ctx.DriverCtx) 713 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 714 copyImage(t, ctx.ExecCtx.TaskDir, "busybox_musl.tar") 715 copyImage(t, ctx.ExecCtx.TaskDir, "busybox_glibc.tar") 716 717 _, err := d.Prestart(ctx.ExecCtx, task) 718 if err != nil { 719 t.Fatalf("error in prestart #%d: %v", idx+1, err) 720 } 721 resp, err := d.Start(ctx.ExecCtx, task) 722 if err != nil { 723 t.Errorf("Failed starting task #%d: %s", idx+1, err) 724 continue 725 } 726 handles[idx] = resp.Handle 727 } 728 729 t.Log("All tasks are started. Terminating...") 730 731 for idx, handle := range handles { 732 if handle == nil { 733 t.Errorf("Bad handle for task #%d", idx+1) 734 continue 735 } 736 737 err := handle.Kill() 738 if err != nil { 739 t.Errorf("Failed stopping task #%d: %s", idx+1, err) 740 } 741 } 742 743 t.Log("Test complete!") 744 } 745 746 func waitForExist(t *testing.T, client *docker.Client, handle *DockerHandle) { 747 handle.logger.Printf("[DEBUG] docker.test: waiting for container %s to exist...", handle.ContainerID()) 748 tu.WaitForResult(func() (bool, error) { 749 container, err := client.InspectContainer(handle.ContainerID()) 750 if err != nil { 751 if _, ok := err.(*docker.NoSuchContainer); !ok { 752 return false, err 753 } 754 } 755 756 return container != nil, nil 757 }, func(err error) { 758 t.Fatalf("err: %v", err) 759 }) 760 handle.logger.Printf("[DEBUG] docker.test: ...container %s exists!", handle.ContainerID()) 761 } 762 763 func TestDockerDriver_NetworkMode_Host(t *testing.T) { 764 if !tu.IsTravis() { 765 t.Parallel() 766 } 767 if !testutil.DockerIsConnected(t) { 768 t.Skip("Docker not connected") 769 } 770 expected := "host" 771 772 task := &structs.Task{ 773 Name: "nc-demo", 774 Driver: "docker", 775 Config: map[string]interface{}{ 776 "image": "busybox", 777 "load": "busybox.tar", 778 "command": "/bin/nc", 779 "args": []string{"-l", "127.0.0.1", "-p", "0"}, 780 "network_mode": expected, 781 }, 782 Resources: &structs.Resources{ 783 MemoryMB: 256, 784 CPU: 512, 785 }, 786 LogConfig: &structs.LogConfig{ 787 MaxFiles: 10, 788 MaxFileSizeMB: 10, 789 }, 790 } 791 792 client, handle, cleanup := dockerSetup(t, task) 793 defer cleanup() 794 795 waitForExist(t, client, handle) 796 797 container, err := client.InspectContainer(handle.ContainerID()) 798 if err != nil { 799 t.Fatalf("err: %v", err) 800 } 801 802 actual := container.HostConfig.NetworkMode 803 if actual != expected { 804 t.Fatalf("Got network mode %q; want %q", expected, actual) 805 } 806 } 807 808 func TestDockerDriver_NetworkAliases_Bridge(t *testing.T) { 809 if !tu.IsTravis() { 810 t.Parallel() 811 } 812 if !testutil.DockerIsConnected(t) { 813 t.Skip("Docker not connected") 814 } 815 816 // Because go-dockerclient doesn't provide api for query network aliases, just check that 817 // a container can be created with a 'network_aliases' property 818 819 // Create network, network-scoped alias is supported only for containers in user defined networks 820 client := newTestDockerClient(t) 821 networkOpts := docker.CreateNetworkOptions{Name: "foobar", Driver: "bridge"} 822 network, err := client.CreateNetwork(networkOpts) 823 if err != nil { 824 t.Fatalf("err: %v", err) 825 } 826 defer client.RemoveNetwork(network.ID) 827 828 expected := []string{"foobar"} 829 task := &structs.Task{ 830 Name: "nc-demo", 831 Driver: "docker", 832 Config: map[string]interface{}{ 833 "image": "busybox", 834 "load": "busybox.tar", 835 "command": "/bin/nc", 836 "args": []string{"-l", "127.0.0.1", "-p", "0"}, 837 "network_mode": network.Name, 838 "network_aliases": expected, 839 }, 840 Resources: &structs.Resources{ 841 MemoryMB: 256, 842 CPU: 512, 843 }, 844 LogConfig: &structs.LogConfig{ 845 MaxFiles: 10, 846 MaxFileSizeMB: 10, 847 }, 848 } 849 850 client, handle, cleanup := dockerSetupWithClient(t, task, client) 851 defer cleanup() 852 853 waitForExist(t, client, handle) 854 855 _, err = client.InspectContainer(handle.ContainerID()) 856 if err != nil { 857 t.Fatalf("err: %v", err) 858 } 859 } 860 861 func TestDockerDriver_Sysctl_Ulimit(t *testing.T) { 862 task, _, _ := dockerTask(t) 863 expectedUlimits := map[string]string{ 864 "nproc": "4242", 865 "nofile": "2048:4096", 866 } 867 task.Config["sysctl"] = []map[string]string{ 868 { 869 "net.core.somaxconn": "16384", 870 }, 871 } 872 task.Config["ulimit"] = []map[string]string{ 873 expectedUlimits, 874 } 875 876 client, handle, cleanup := dockerSetup(t, task) 877 defer cleanup() 878 879 waitForExist(t, client, handle) 880 881 container, err := client.InspectContainer(handle.ContainerID()) 882 assert.Nil(t, err, "unexpected error: %v", err) 883 884 want := "16384" 885 got := container.HostConfig.Sysctls["net.core.somaxconn"] 886 assert.Equal(t, want, got, "Wrong net.core.somaxconn config for docker job. Expect: %s, got: %s", want, got) 887 888 expectedUlimitLen := 2 889 actualUlimitLen := len(container.HostConfig.Ulimits) 890 assert.Equal(t, want, got, "Wrong number of ulimit configs for docker job. Expect: %d, got: %d", expectedUlimitLen, actualUlimitLen) 891 892 for _, got := range container.HostConfig.Ulimits { 893 if expectedStr, ok := expectedUlimits[got.Name]; !ok { 894 t.Errorf("%s config unexpected for docker job.", got.Name) 895 } else { 896 if !strings.Contains(expectedStr, ":") { 897 expectedStr = expectedStr + ":" + expectedStr 898 } 899 900 splitted := strings.SplitN(expectedStr, ":", 2) 901 soft, _ := strconv.Atoi(splitted[0]) 902 hard, _ := strconv.Atoi(splitted[1]) 903 assert.Equal(t, int64(soft), got.Soft, "Wrong soft %s ulimit for docker job. Expect: %d, got: %d", got.Name, soft, got.Soft) 904 assert.Equal(t, int64(hard), got.Hard, "Wrong hard %s ulimit for docker job. Expect: %d, got: %d", got.Name, hard, got.Hard) 905 906 } 907 } 908 } 909 910 func TestDockerDriver_Sysctl_Ulimit_Errors(t *testing.T) { 911 brokenConfigs := []interface{}{ 912 map[string]interface{}{ 913 "nofile": "", 914 }, 915 map[string]interface{}{ 916 "nofile": "abc:1234", 917 }, 918 map[string]interface{}{ 919 "nofile": "1234:abc", 920 }, 921 } 922 923 test_cases := []struct { 924 ulimitConfig interface{} 925 err error 926 }{ 927 {[]interface{}{brokenConfigs[0]}, fmt.Errorf("Malformed ulimit specification nofile: \"\", cannot be empty")}, 928 {[]interface{}{brokenConfigs[1]}, fmt.Errorf("Malformed soft ulimit nofile: abc:1234")}, 929 {[]interface{}{brokenConfigs[2]}, fmt.Errorf("Malformed hard ulimit nofile: 1234:abc")}, 930 } 931 932 for _, tc := range test_cases { 933 task, _, _ := dockerTask(t) 934 task.Config["ulimit"] = tc.ulimitConfig 935 936 ctx := testDockerDriverContexts(t, task) 937 driver := NewDockerDriver(ctx.DriverCtx) 938 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 939 defer ctx.AllocDir.Destroy() 940 941 _, err := driver.Prestart(ctx.ExecCtx, task) 942 assert.NotNil(t, err, "Expected non nil error") 943 assert.Equal(t, err.Error(), tc.err.Error(), "unexpected error in prestart, got %v, expected %v", err, tc.err) 944 } 945 } 946 947 func TestDockerDriver_Labels(t *testing.T) { 948 if !tu.IsTravis() { 949 t.Parallel() 950 } 951 if !testutil.DockerIsConnected(t) { 952 t.Skip("Docker not connected") 953 } 954 955 task, _, _ := dockerTask(t) 956 task.Config["labels"] = []map[string]string{ 957 { 958 "label1": "value1", 959 "label2": "value2", 960 }, 961 } 962 963 client, handle, cleanup := dockerSetup(t, task) 964 defer cleanup() 965 966 waitForExist(t, client, handle) 967 968 container, err := client.InspectContainer(handle.ContainerID()) 969 if err != nil { 970 t.Fatalf("err: %v", err) 971 } 972 973 if want, got := 2, len(container.Config.Labels); want != got { 974 t.Errorf("Wrong labels count for docker job. Expect: %d, got: %d", want, got) 975 } 976 977 if want, got := "value1", container.Config.Labels["label1"]; want != got { 978 t.Errorf("Wrong label value docker job. Expect: %s, got: %s", want, got) 979 } 980 } 981 982 func TestDockerDriver_ForcePull_IsInvalidConfig(t *testing.T) { 983 if !tu.IsTravis() { 984 t.Parallel() 985 } 986 if !testutil.DockerIsConnected(t) { 987 t.Skip("Docker not connected") 988 } 989 990 task, _, _ := dockerTask(t) 991 task.Config["force_pull"] = "nothing" 992 993 ctx := testDockerDriverContexts(t, task) 994 defer ctx.AllocDir.Destroy() 995 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 996 driver := NewDockerDriver(ctx.DriverCtx) 997 998 if _, err := driver.Prestart(ctx.ExecCtx, task); err == nil { 999 t.Fatalf("error expected in prestart") 1000 } 1001 } 1002 1003 func TestDockerDriver_ForcePull(t *testing.T) { 1004 if !tu.IsTravis() { 1005 t.Parallel() 1006 } 1007 if !testutil.DockerIsConnected(t) { 1008 t.Skip("Docker not connected") 1009 } 1010 1011 task, _, _ := dockerTask(t) 1012 task.Config["force_pull"] = "true" 1013 1014 client, handle, cleanup := dockerSetup(t, task) 1015 defer cleanup() 1016 1017 waitForExist(t, client, handle) 1018 1019 _, err := client.InspectContainer(handle.ContainerID()) 1020 if err != nil { 1021 t.Fatalf("err: %v", err) 1022 } 1023 } 1024 1025 func TestDockerDriver_SecurityOpt(t *testing.T) { 1026 if !tu.IsTravis() { 1027 t.Parallel() 1028 } 1029 if !testutil.DockerIsConnected(t) { 1030 t.Skip("Docker not connected") 1031 } 1032 1033 task, _, _ := dockerTask(t) 1034 task.Config["security_opt"] = []string{"seccomp=unconfined"} 1035 1036 client, handle, cleanup := dockerSetup(t, task) 1037 defer cleanup() 1038 1039 waitForExist(t, client, handle) 1040 1041 container, err := client.InspectContainer(handle.ContainerID()) 1042 if err != nil { 1043 t.Fatalf("err: %v", err) 1044 } 1045 1046 if !reflect.DeepEqual(task.Config["security_opt"], container.HostConfig.SecurityOpt) { 1047 t.Errorf("Security Opts don't match.\nExpected:\n%s\nGot:\n%s\n", task.Config["security_opt"], container.HostConfig.SecurityOpt) 1048 } 1049 } 1050 1051 func TestDockerDriver_DNS(t *testing.T) { 1052 if !tu.IsTravis() { 1053 t.Parallel() 1054 } 1055 if !testutil.DockerIsConnected(t) { 1056 t.Skip("Docker not connected") 1057 } 1058 1059 task, _, _ := dockerTask(t) 1060 task.Config["dns_servers"] = []string{"8.8.8.8", "8.8.4.4"} 1061 task.Config["dns_search_domains"] = []string{"example.com", "example.org", "example.net"} 1062 task.Config["dns_options"] = []string{"ndots:1"} 1063 1064 client, handle, cleanup := dockerSetup(t, task) 1065 defer cleanup() 1066 1067 waitForExist(t, client, handle) 1068 1069 container, err := client.InspectContainer(handle.ContainerID()) 1070 if err != nil { 1071 t.Fatalf("err: %v", err) 1072 } 1073 1074 if !reflect.DeepEqual(task.Config["dns_servers"], container.HostConfig.DNS) { 1075 t.Errorf("DNS Servers don't match.\nExpected:\n%s\nGot:\n%s\n", task.Config["dns_servers"], container.HostConfig.DNS) 1076 } 1077 1078 if !reflect.DeepEqual(task.Config["dns_search_domains"], container.HostConfig.DNSSearch) { 1079 t.Errorf("DNS Search Domains don't match.\nExpected:\n%s\nGot:\n%s\n", task.Config["dns_search_domains"], container.HostConfig.DNSSearch) 1080 } 1081 1082 if !reflect.DeepEqual(task.Config["dns_options"], container.HostConfig.DNSOptions) { 1083 t.Errorf("DNS Options don't match.\nExpected:\n%s\nGot:\n%s\n", task.Config["dns_options"], container.HostConfig.DNSOptions) 1084 } 1085 } 1086 1087 func TestDockerDriver_MACAddress(t *testing.T) { 1088 if !tu.IsTravis() { 1089 t.Parallel() 1090 } 1091 if !testutil.DockerIsConnected(t) { 1092 t.Skip("Docker not connected") 1093 } 1094 1095 task, _, _ := dockerTask(t) 1096 task.Config["mac_address"] = "00:16:3e:00:00:00" 1097 1098 client, handle, cleanup := dockerSetup(t, task) 1099 defer cleanup() 1100 1101 waitForExist(t, client, handle) 1102 1103 container, err := client.InspectContainer(handle.ContainerID()) 1104 if err != nil { 1105 t.Fatalf("err: %v", err) 1106 } 1107 1108 if container.NetworkSettings.MacAddress != task.Config["mac_address"] { 1109 t.Errorf("expected mac_address=%q but found %q", task.Config["mac_address"], container.NetworkSettings.MacAddress) 1110 } 1111 } 1112 1113 func TestDockerWorkDir(t *testing.T) { 1114 if !tu.IsTravis() { 1115 t.Parallel() 1116 } 1117 if !testutil.DockerIsConnected(t) { 1118 t.Skip("Docker not connected") 1119 } 1120 1121 task, _, _ := dockerTask(t) 1122 task.Config["work_dir"] = "/some/path" 1123 1124 client, handle, cleanup := dockerSetup(t, task) 1125 defer cleanup() 1126 1127 container, err := client.InspectContainer(handle.ContainerID()) 1128 if err != nil { 1129 t.Fatalf("err: %v", err) 1130 } 1131 1132 if want, got := "/some/path", container.Config.WorkingDir; want != got { 1133 t.Errorf("Wrong working directory for docker job. Expect: %s, got: %s", want, got) 1134 } 1135 } 1136 1137 func inSlice(needle string, haystack []string) bool { 1138 for _, h := range haystack { 1139 if h == needle { 1140 return true 1141 } 1142 } 1143 return false 1144 } 1145 1146 func TestDockerDriver_PortsNoMap(t *testing.T) { 1147 if !tu.IsTravis() { 1148 t.Parallel() 1149 } 1150 if !testutil.DockerIsConnected(t) { 1151 t.Skip("Docker not connected") 1152 } 1153 1154 task, res, dyn := dockerTask(t) 1155 1156 client, handle, cleanup := dockerSetup(t, task) 1157 defer cleanup() 1158 1159 waitForExist(t, client, handle) 1160 1161 container, err := client.InspectContainer(handle.ContainerID()) 1162 if err != nil { 1163 t.Fatalf("err: %v", err) 1164 } 1165 1166 // Verify that the correct ports are EXPOSED 1167 expectedExposedPorts := map[docker.Port]struct{}{ 1168 docker.Port(fmt.Sprintf("%d/tcp", res)): {}, 1169 docker.Port(fmt.Sprintf("%d/udp", res)): {}, 1170 docker.Port(fmt.Sprintf("%d/tcp", dyn)): {}, 1171 docker.Port(fmt.Sprintf("%d/udp", dyn)): {}, 1172 } 1173 1174 if !reflect.DeepEqual(container.Config.ExposedPorts, expectedExposedPorts) { 1175 t.Errorf("Exposed ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedExposedPorts, container.Config.ExposedPorts) 1176 } 1177 1178 // Verify that the correct ports are FORWARDED 1179 expectedPortBindings := map[docker.Port][]docker.PortBinding{ 1180 docker.Port(fmt.Sprintf("%d/tcp", res)): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}}, 1181 docker.Port(fmt.Sprintf("%d/udp", res)): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}}, 1182 docker.Port(fmt.Sprintf("%d/tcp", dyn)): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}}, 1183 docker.Port(fmt.Sprintf("%d/udp", dyn)): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}}, 1184 } 1185 1186 if !reflect.DeepEqual(container.HostConfig.PortBindings, expectedPortBindings) { 1187 t.Errorf("Forwarded ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedPortBindings, container.HostConfig.PortBindings) 1188 } 1189 1190 expectedEnvironment := map[string]string{ 1191 "NOMAD_ADDR_main": fmt.Sprintf("127.0.0.1:%d", res), 1192 "NOMAD_ADDR_REDIS": fmt.Sprintf("127.0.0.1:%d", dyn), 1193 } 1194 1195 for key, val := range expectedEnvironment { 1196 search := fmt.Sprintf("%s=%s", key, val) 1197 if !inSlice(search, container.Config.Env) { 1198 t.Errorf("Expected to find %s in container environment: %+v", search, container.Config.Env) 1199 } 1200 } 1201 } 1202 1203 func TestDockerDriver_PortsMapping(t *testing.T) { 1204 if !tu.IsTravis() { 1205 t.Parallel() 1206 } 1207 if !testutil.DockerIsConnected(t) { 1208 t.Skip("Docker not connected") 1209 } 1210 1211 task, res, dyn := dockerTask(t) 1212 task.Config["port_map"] = []map[string]string{ 1213 { 1214 "main": "8080", 1215 "REDIS": "6379", 1216 }, 1217 } 1218 1219 client, handle, cleanup := dockerSetup(t, task) 1220 defer cleanup() 1221 1222 waitForExist(t, client, handle) 1223 1224 container, err := client.InspectContainer(handle.ContainerID()) 1225 if err != nil { 1226 t.Fatalf("err: %v", err) 1227 } 1228 1229 // Verify that the correct ports are EXPOSED 1230 expectedExposedPorts := map[docker.Port]struct{}{ 1231 docker.Port("8080/tcp"): {}, 1232 docker.Port("8080/udp"): {}, 1233 docker.Port("6379/tcp"): {}, 1234 docker.Port("6379/udp"): {}, 1235 } 1236 1237 if !reflect.DeepEqual(container.Config.ExposedPorts, expectedExposedPorts) { 1238 t.Errorf("Exposed ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedExposedPorts, container.Config.ExposedPorts) 1239 } 1240 1241 // Verify that the correct ports are FORWARDED 1242 expectedPortBindings := map[docker.Port][]docker.PortBinding{ 1243 docker.Port("8080/tcp"): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}}, 1244 docker.Port("8080/udp"): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", res)}}, 1245 docker.Port("6379/tcp"): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}}, 1246 docker.Port("6379/udp"): {{HostIP: "127.0.0.1", HostPort: fmt.Sprintf("%d", dyn)}}, 1247 } 1248 1249 if !reflect.DeepEqual(container.HostConfig.PortBindings, expectedPortBindings) { 1250 t.Errorf("Forwarded ports don't match.\nExpected:\n%s\nGot:\n%s\n", expectedPortBindings, container.HostConfig.PortBindings) 1251 } 1252 1253 expectedEnvironment := map[string]string{ 1254 "NOMAD_PORT_main": "8080", 1255 "NOMAD_PORT_REDIS": "6379", 1256 "NOMAD_HOST_PORT_main": strconv.Itoa(res), 1257 } 1258 1259 sort.Strings(container.Config.Env) 1260 for key, val := range expectedEnvironment { 1261 search := fmt.Sprintf("%s=%s", key, val) 1262 if !inSlice(search, container.Config.Env) { 1263 t.Errorf("Expected to find %s in container environment:\n%s\n\n", search, strings.Join(container.Config.Env, "\n")) 1264 } 1265 } 1266 } 1267 1268 func TestDockerDriver_User(t *testing.T) { 1269 if !tu.IsTravis() { 1270 t.Parallel() 1271 } 1272 if !testutil.DockerIsConnected(t) { 1273 t.Skip("Docker not connected") 1274 } 1275 1276 task := &structs.Task{ 1277 Name: "redis-demo", 1278 User: "alice", 1279 Driver: "docker", 1280 Config: map[string]interface{}{ 1281 "image": "busybox", 1282 "load": "busybox.tar", 1283 "command": "/bin/sleep", 1284 "args": []string{"10000"}, 1285 }, 1286 Resources: &structs.Resources{ 1287 MemoryMB: 256, 1288 CPU: 512, 1289 }, 1290 LogConfig: &structs.LogConfig{ 1291 MaxFiles: 10, 1292 MaxFileSizeMB: 10, 1293 }, 1294 } 1295 1296 ctx := testDockerDriverContexts(t, task) 1297 //ctx.DriverCtx.config.Options = map[string]string{"docker.cleanup.image": "false"} 1298 driver := NewDockerDriver(ctx.DriverCtx) 1299 defer ctx.AllocDir.Destroy() 1300 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 1301 1302 _, err := driver.Prestart(ctx.ExecCtx, task) 1303 if err != nil { 1304 t.Fatalf("error in prestart: %v", err) 1305 } 1306 1307 // It should fail because the user "alice" does not exist on the given 1308 // image. 1309 resp, err := driver.Start(ctx.ExecCtx, task) 1310 if err == nil { 1311 resp.Handle.Kill() 1312 t.Fatalf("Should've failed") 1313 } 1314 1315 if !strings.Contains(err.Error(), "alice") { 1316 t.Fatalf("Expected failure string not found, found %q instead", err.Error()) 1317 } 1318 } 1319 1320 func TestDockerDriver_CleanupContainer(t *testing.T) { 1321 if !tu.IsTravis() { 1322 t.Parallel() 1323 } 1324 if !testutil.DockerIsConnected(t) { 1325 t.Skip("Docker not connected") 1326 } 1327 1328 task := &structs.Task{ 1329 Name: "redis-demo", 1330 Driver: "docker", 1331 Config: map[string]interface{}{ 1332 "image": "busybox", 1333 "load": "busybox.tar", 1334 "command": "/bin/echo", 1335 "args": []string{"hello"}, 1336 }, 1337 Resources: &structs.Resources{ 1338 MemoryMB: 256, 1339 CPU: 512, 1340 }, 1341 LogConfig: &structs.LogConfig{ 1342 MaxFiles: 10, 1343 MaxFileSizeMB: 10, 1344 }, 1345 } 1346 1347 _, handle, cleanup := dockerSetup(t, task) 1348 defer cleanup() 1349 1350 // Update should be a no-op 1351 err := handle.Update(task) 1352 if err != nil { 1353 t.Fatalf("err: %v", err) 1354 } 1355 1356 select { 1357 case res := <-handle.WaitCh(): 1358 if !res.Successful() { 1359 t.Fatalf("err: %v", res) 1360 } 1361 1362 time.Sleep(3 * time.Second) 1363 1364 // Ensure that the container isn't present 1365 _, err := client.InspectContainer(handle.containerID) 1366 if err == nil { 1367 t.Fatalf("expected to not get container") 1368 } 1369 1370 case <-time.After(time.Duration(tu.TestMultiplier()*5) * time.Second): 1371 t.Fatalf("timeout") 1372 } 1373 } 1374 1375 func TestDockerDriver_Stats(t *testing.T) { 1376 if !tu.IsTravis() { 1377 t.Parallel() 1378 } 1379 if !testutil.DockerIsConnected(t) { 1380 t.Skip("Docker not connected") 1381 } 1382 1383 task := &structs.Task{ 1384 Name: "sleep", 1385 Driver: "docker", 1386 Config: map[string]interface{}{ 1387 "image": "busybox", 1388 "load": "busybox.tar", 1389 "command": "/bin/sleep", 1390 "args": []string{"100"}, 1391 }, 1392 LogConfig: &structs.LogConfig{ 1393 MaxFiles: 10, 1394 MaxFileSizeMB: 10, 1395 }, 1396 Resources: basicResources, 1397 } 1398 1399 _, handle, cleanup := dockerSetup(t, task) 1400 defer cleanup() 1401 1402 waitForExist(t, client, handle) 1403 1404 go func() { 1405 time.Sleep(3 * time.Second) 1406 ru, err := handle.Stats() 1407 if err != nil { 1408 t.Fatalf("err: %v", err) 1409 } 1410 if ru.ResourceUsage == nil { 1411 handle.Kill() 1412 t.Fatalf("expected resource usage") 1413 } 1414 err = handle.Kill() 1415 if err != nil { 1416 t.Fatalf("err: %v", err) 1417 } 1418 }() 1419 1420 select { 1421 case res := <-handle.WaitCh(): 1422 if res.Successful() { 1423 t.Fatalf("should err: %v", res) 1424 } 1425 case <-time.After(time.Duration(tu.TestMultiplier()*10) * time.Second): 1426 t.Fatalf("timeout") 1427 } 1428 } 1429 1430 func setupDockerVolumes(t *testing.T, cfg *config.Config, hostpath string) (*structs.Task, Driver, *ExecContext, string, func()) { 1431 if !testutil.DockerIsConnected(t) { 1432 t.Skip("Docker not connected") 1433 } 1434 1435 randfn := fmt.Sprintf("test-%d", rand.Int()) 1436 hostfile := filepath.Join(hostpath, randfn) 1437 containerPath := "/mnt/vol" 1438 containerFile := filepath.Join(containerPath, randfn) 1439 1440 task := &structs.Task{ 1441 Name: "ls", 1442 Env: map[string]string{"VOL_PATH": containerPath}, 1443 Driver: "docker", 1444 Config: map[string]interface{}{ 1445 "image": "busybox", 1446 "load": "busybox.tar", 1447 "command": "touch", 1448 "args": []string{containerFile}, 1449 "volumes": []string{fmt.Sprintf("%s:${VOL_PATH}", hostpath)}, 1450 }, 1451 LogConfig: &structs.LogConfig{ 1452 MaxFiles: 10, 1453 MaxFileSizeMB: 10, 1454 }, 1455 Resources: basicResources, 1456 } 1457 1458 // Build alloc and task directory structure 1459 allocDir := allocdir.NewAllocDir(testLogger(), filepath.Join(cfg.AllocDir, uuid.Generate())) 1460 if err := allocDir.Build(); err != nil { 1461 t.Fatalf("failed to build alloc dir: %v", err) 1462 } 1463 taskDir := allocDir.NewTaskDir(task.Name) 1464 if err := taskDir.Build(false, nil, cstructs.FSIsolationImage); err != nil { 1465 allocDir.Destroy() 1466 t.Fatalf("failed to build task dir: %v", err) 1467 } 1468 1469 alloc := mock.Alloc() 1470 envBuilder := env.NewBuilder(cfg.Node, alloc, task, cfg.Region) 1471 execCtx := NewExecContext(taskDir, envBuilder.Build()) 1472 cleanup := func() { 1473 allocDir.Destroy() 1474 if filepath.IsAbs(hostpath) { 1475 os.RemoveAll(hostpath) 1476 } 1477 } 1478 1479 logger := testLogger() 1480 emitter := func(m string, args ...interface{}) { 1481 logger.Printf("[EVENT] "+m, args...) 1482 } 1483 driverCtx := NewDriverContext(task.Name, alloc.ID, cfg, cfg.Node, testLogger(), emitter) 1484 driver := NewDockerDriver(driverCtx) 1485 copyImage(t, taskDir, "busybox.tar") 1486 1487 return task, driver, execCtx, hostfile, cleanup 1488 } 1489 1490 func TestDockerDriver_VolumesDisabled(t *testing.T) { 1491 if !tu.IsTravis() { 1492 t.Parallel() 1493 } 1494 if !testutil.DockerIsConnected(t) { 1495 t.Skip("Docker not connected") 1496 } 1497 1498 cfg := testConfig(t) 1499 cfg.Options = map[string]string{ 1500 dockerVolumesConfigOption: "false", 1501 "docker.cleanup.image": "false", 1502 } 1503 1504 { 1505 tmpvol, err := ioutil.TempDir("", "nomadtest_docker_volumesdisabled") 1506 if err != nil { 1507 t.Fatalf("error creating temporary dir: %v", err) 1508 } 1509 1510 task, driver, execCtx, _, cleanup := setupDockerVolumes(t, cfg, tmpvol) 1511 defer cleanup() 1512 1513 _, err = driver.Prestart(execCtx, task) 1514 if err != nil { 1515 t.Fatalf("error in prestart: %v", err) 1516 } 1517 if _, err := driver.Start(execCtx, task); err == nil { 1518 t.Fatalf("Started driver successfully when volumes should have been disabled.") 1519 } 1520 } 1521 1522 // Relative paths should still be allowed 1523 { 1524 task, driver, execCtx, fn, cleanup := setupDockerVolumes(t, cfg, ".") 1525 defer cleanup() 1526 1527 _, err := driver.Prestart(execCtx, task) 1528 if err != nil { 1529 t.Fatalf("error in prestart: %v", err) 1530 } 1531 resp, err := driver.Start(execCtx, task) 1532 if err != nil { 1533 t.Fatalf("err: %v", err) 1534 } 1535 defer resp.Handle.Kill() 1536 1537 select { 1538 case res := <-resp.Handle.WaitCh(): 1539 if !res.Successful() { 1540 t.Fatalf("unexpected err: %v", res) 1541 } 1542 case <-time.After(time.Duration(tu.TestMultiplier()*10) * time.Second): 1543 t.Fatalf("timeout") 1544 } 1545 1546 if _, err := ioutil.ReadFile(filepath.Join(execCtx.TaskDir.Dir, fn)); err != nil { 1547 t.Fatalf("unexpected error reading %s: %v", fn, err) 1548 } 1549 } 1550 1551 // Volume Drivers should be rejected (error) 1552 { 1553 task, driver, execCtx, _, cleanup := setupDockerVolumes(t, cfg, "fake_flocker_vol") 1554 defer cleanup() 1555 task.Config["volume_driver"] = "flocker" 1556 1557 if _, err := driver.Prestart(execCtx, task); err != nil { 1558 t.Fatalf("error in prestart: %v", err) 1559 } 1560 if _, err := driver.Start(execCtx, task); err == nil { 1561 t.Fatalf("Started driver successfully when volume drivers should have been disabled.") 1562 } 1563 } 1564 1565 } 1566 1567 func TestDockerDriver_VolumesEnabled(t *testing.T) { 1568 if !tu.IsTravis() { 1569 t.Parallel() 1570 } 1571 if !testutil.DockerIsConnected(t) { 1572 t.Skip("Docker not connected") 1573 } 1574 1575 cfg := testConfig(t) 1576 1577 tmpvol, err := ioutil.TempDir("", "nomadtest_docker_volumesenabled") 1578 if err != nil { 1579 t.Fatalf("error creating temporary dir: %v", err) 1580 } 1581 1582 // Evaluate symlinks so it works on MacOS 1583 tmpvol, err = filepath.EvalSymlinks(tmpvol) 1584 if err != nil { 1585 t.Fatalf("error evaluating symlinks: %v", err) 1586 } 1587 1588 task, driver, execCtx, hostpath, cleanup := setupDockerVolumes(t, cfg, tmpvol) 1589 defer cleanup() 1590 1591 _, err = driver.Prestart(execCtx, task) 1592 if err != nil { 1593 t.Fatalf("error in prestart: %v", err) 1594 } 1595 resp, err := driver.Start(execCtx, task) 1596 if err != nil { 1597 t.Fatalf("Failed to start docker driver: %v", err) 1598 } 1599 defer resp.Handle.Kill() 1600 1601 select { 1602 case res := <-resp.Handle.WaitCh(): 1603 if !res.Successful() { 1604 t.Fatalf("unexpected err: %v", res) 1605 } 1606 case <-time.After(time.Duration(tu.TestMultiplier()*10) * time.Second): 1607 t.Fatalf("timeout") 1608 } 1609 1610 if _, err := ioutil.ReadFile(hostpath); err != nil { 1611 t.Fatalf("unexpected error reading %s: %v", hostpath, err) 1612 } 1613 } 1614 1615 func TestDockerDriver_Mounts(t *testing.T) { 1616 if !tu.IsTravis() { 1617 t.Parallel() 1618 } 1619 if !testutil.DockerIsConnected(t) { 1620 t.Skip("Docker not connected") 1621 } 1622 1623 goodMount := map[string]interface{}{ 1624 "target": "/nomad", 1625 "volume_options": []interface{}{ 1626 map[string]interface{}{ 1627 "labels": []interface{}{ 1628 map[string]string{"foo": "bar"}, 1629 }, 1630 "driver_config": []interface{}{ 1631 map[string]interface{}{ 1632 "name": "local", 1633 "options": []interface{}{ 1634 map[string]interface{}{ 1635 "foo": "bar", 1636 }, 1637 }, 1638 }, 1639 }, 1640 }, 1641 }, 1642 "readonly": true, 1643 "source": "test", 1644 } 1645 1646 cases := []struct { 1647 Name string 1648 Mounts []interface{} 1649 Error string 1650 }{ 1651 { 1652 Name: "good-one", 1653 Error: "", 1654 Mounts: []interface{}{goodMount}, 1655 }, 1656 { 1657 Name: "good-many", 1658 Error: "", 1659 Mounts: []interface{}{goodMount, goodMount, goodMount}, 1660 }, 1661 { 1662 Name: "multiple volume options", 1663 Error: "Only one volume_options stanza allowed", 1664 Mounts: []interface{}{ 1665 map[string]interface{}{ 1666 "target": "/nomad", 1667 "volume_options": []interface{}{ 1668 map[string]interface{}{ 1669 "driver_config": []interface{}{ 1670 map[string]interface{}{ 1671 "name": "local", 1672 }, 1673 }, 1674 }, 1675 map[string]interface{}{ 1676 "driver_config": []interface{}{ 1677 map[string]interface{}{ 1678 "name": "local", 1679 }, 1680 }, 1681 }, 1682 }, 1683 }, 1684 }, 1685 }, 1686 { 1687 Name: "multiple driver configs", 1688 Error: "volume driver config may only be specified once", 1689 Mounts: []interface{}{ 1690 map[string]interface{}{ 1691 "target": "/nomad", 1692 "volume_options": []interface{}{ 1693 map[string]interface{}{ 1694 "driver_config": []interface{}{ 1695 map[string]interface{}{ 1696 "name": "local", 1697 }, 1698 map[string]interface{}{ 1699 "name": "local", 1700 }, 1701 }, 1702 }, 1703 }, 1704 }, 1705 }, 1706 }, 1707 { 1708 Name: "multiple volume labels", 1709 Error: "labels may only be", 1710 Mounts: []interface{}{ 1711 map[string]interface{}{ 1712 "target": "/nomad", 1713 "volume_options": []interface{}{ 1714 map[string]interface{}{ 1715 "labels": []interface{}{ 1716 map[string]string{"foo": "bar"}, 1717 map[string]string{"baz": "bam"}, 1718 }, 1719 }, 1720 }, 1721 }, 1722 }, 1723 }, 1724 { 1725 Name: "multiple driver options", 1726 Error: "driver options may only", 1727 Mounts: []interface{}{ 1728 map[string]interface{}{ 1729 "target": "/nomad", 1730 "volume_options": []interface{}{ 1731 map[string]interface{}{ 1732 "driver_config": []interface{}{ 1733 map[string]interface{}{ 1734 "name": "local", 1735 "options": []interface{}{ 1736 map[string]interface{}{ 1737 "foo": "bar", 1738 }, 1739 map[string]interface{}{ 1740 "bam": "bar", 1741 }, 1742 }, 1743 }, 1744 }, 1745 }, 1746 }, 1747 }, 1748 }, 1749 }, 1750 } 1751 1752 task := &structs.Task{ 1753 Name: "redis-demo", 1754 Driver: "docker", 1755 Config: map[string]interface{}{ 1756 "image": "busybox", 1757 "load": "busybox.tar", 1758 "command": "/bin/sleep", 1759 "args": []string{"10000"}, 1760 }, 1761 Resources: &structs.Resources{ 1762 MemoryMB: 256, 1763 CPU: 512, 1764 }, 1765 LogConfig: &structs.LogConfig{ 1766 MaxFiles: 10, 1767 MaxFileSizeMB: 10, 1768 }, 1769 } 1770 1771 for _, c := range cases { 1772 t.Run(c.Name, func(t *testing.T) { 1773 // Build the task 1774 task.Config["mounts"] = c.Mounts 1775 1776 ctx := testDockerDriverContexts(t, task) 1777 driver := NewDockerDriver(ctx.DriverCtx) 1778 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 1779 defer ctx.AllocDir.Destroy() 1780 1781 _, err := driver.Prestart(ctx.ExecCtx, task) 1782 if err == nil && c.Error != "" { 1783 t.Fatalf("expected error: %v", c.Error) 1784 } else if err != nil { 1785 if c.Error == "" { 1786 t.Fatalf("unexpected error in prestart: %v", err) 1787 } else if !strings.Contains(err.Error(), c.Error) { 1788 t.Fatalf("expected error %q; got %v", c.Error, err) 1789 } 1790 } 1791 }) 1792 } 1793 } 1794 1795 // TestDockerDriver_Cleanup ensures Cleanup removes only downloaded images. 1796 func TestDockerDriver_Cleanup(t *testing.T) { 1797 if !tu.IsTravis() { 1798 t.Parallel() 1799 } 1800 if !testutil.DockerIsConnected(t) { 1801 t.Skip("Docker not connected") 1802 } 1803 1804 imageName := "hello-world:latest" 1805 task := &structs.Task{ 1806 Name: "cleanup_test", 1807 Driver: "docker", 1808 Config: map[string]interface{}{ 1809 "image": imageName, 1810 }, 1811 } 1812 tctx := testDockerDriverContexts(t, task) 1813 defer tctx.AllocDir.Destroy() 1814 1815 // Run Prestart 1816 driver := NewDockerDriver(tctx.DriverCtx).(*DockerDriver) 1817 resp, err := driver.Prestart(tctx.ExecCtx, task) 1818 if err != nil { 1819 t.Fatalf("error in prestart: %v", err) 1820 } 1821 res := resp.CreatedResources 1822 if len(res.Resources) == 0 || len(res.Resources[dockerImageResKey]) == 0 { 1823 t.Fatalf("no created resources: %#v", res) 1824 } 1825 1826 // Cleanup 1827 rescopy := res.Copy() 1828 if err := driver.Cleanup(tctx.ExecCtx, rescopy); err != nil { 1829 t.Fatalf("Cleanup failed: %v", err) 1830 } 1831 1832 // Make sure rescopy is updated 1833 if len(rescopy.Resources) > 0 { 1834 t.Errorf("Cleanup should have cleared resource map: %#v", rescopy.Resources) 1835 } 1836 1837 // Ensure image was removed 1838 tu.WaitForResult(func() (bool, error) { 1839 if _, err := client.InspectImage(driver.driverConfig.ImageName); err == nil { 1840 return false, fmt.Errorf("image exists but should have been removed. Does another %v container exist?", imageName) 1841 } 1842 1843 return true, nil 1844 }, func(err error) { 1845 t.Fatalf("err: %v", err) 1846 }) 1847 1848 // The image doesn't exist which shouldn't be an error when calling 1849 // Cleanup, so call it again to make sure. 1850 if err := driver.Cleanup(tctx.ExecCtx, res.Copy()); err != nil { 1851 t.Fatalf("Cleanup failed: %v", err) 1852 } 1853 } 1854 1855 func copyImage(t *testing.T, taskDir *allocdir.TaskDir, image string) { 1856 dst := filepath.Join(taskDir.LocalDir, image) 1857 copyFile(filepath.Join("./test-resources/docker", image), dst, t) 1858 } 1859 1860 func TestDockerDriver_AuthConfiguration(t *testing.T) { 1861 if !tu.IsTravis() { 1862 t.Parallel() 1863 } 1864 if !testutil.DockerIsConnected(t) { 1865 t.Skip("Docker not connected") 1866 } 1867 1868 path := "./test-resources/docker/auth.json" 1869 cases := []struct { 1870 Repo string 1871 AuthConfig *docker.AuthConfiguration 1872 }{ 1873 { 1874 Repo: "lolwhat.com/what:1337", 1875 AuthConfig: nil, 1876 }, 1877 { 1878 Repo: "redis:3.2", 1879 AuthConfig: &docker.AuthConfiguration{ 1880 Username: "test", 1881 Password: "1234", 1882 Email: "", 1883 ServerAddress: "https://index.docker.io/v1/", 1884 }, 1885 }, 1886 { 1887 Repo: "quay.io/redis:3.2", 1888 AuthConfig: &docker.AuthConfiguration{ 1889 Username: "test", 1890 Password: "5678", 1891 Email: "", 1892 ServerAddress: "quay.io", 1893 }, 1894 }, 1895 { 1896 Repo: "other.io/redis:3.2", 1897 AuthConfig: &docker.AuthConfiguration{ 1898 Username: "test", 1899 Password: "abcd", 1900 Email: "", 1901 ServerAddress: "https://other.io/v1/", 1902 }, 1903 }, 1904 } 1905 1906 for i, c := range cases { 1907 act, err := authFromDockerConfig(path)(c.Repo) 1908 if err != nil { 1909 t.Fatalf("Test %d failed: %v", i+1, err) 1910 } 1911 1912 if !reflect.DeepEqual(act, c.AuthConfig) { 1913 t.Fatalf("Test %d failed: Unexpected auth config: got %+v; want %+v", i+1, act, c.AuthConfig) 1914 } 1915 } 1916 } 1917 1918 func TestDockerDriver_OOMKilled(t *testing.T) { 1919 if !tu.IsTravis() { 1920 t.Parallel() 1921 } 1922 if !testutil.DockerIsConnected(t) { 1923 t.Skip("Docker not connected") 1924 } 1925 1926 task := &structs.Task{ 1927 Name: "oom-killed", 1928 Driver: "docker", 1929 Config: map[string]interface{}{ 1930 "image": "busybox", 1931 "load": "busybox.tar", 1932 "command": "sh", 1933 // Incrementally creates a bigger and bigger variable. 1934 "args": []string{"-c", "x=a; while true; do eval x='$x$x'; done"}, 1935 }, 1936 LogConfig: &structs.LogConfig{ 1937 MaxFiles: 10, 1938 MaxFileSizeMB: 10, 1939 }, 1940 Resources: &structs.Resources{ 1941 CPU: 250, 1942 MemoryMB: 10, 1943 DiskMB: 20, 1944 Networks: []*structs.NetworkResource{}, 1945 }, 1946 } 1947 1948 _, handle, cleanup := dockerSetup(t, task) 1949 defer cleanup() 1950 1951 select { 1952 case res := <-handle.WaitCh(): 1953 if res.Successful() { 1954 t.Fatalf("expected error, but container exited successful") 1955 } 1956 1957 if res.Err.Error() != "OOM Killed" { 1958 t.Fatalf("not killed by OOM killer: %s", res.Err) 1959 } 1960 1961 t.Logf("Successfully killed by OOM killer") 1962 1963 case <-time.After(time.Duration(tu.TestMultiplier()*5) * time.Second): 1964 t.Fatalf("timeout") 1965 } 1966 } 1967 1968 func TestDockerDriver_Devices_IsInvalidConfig(t *testing.T) { 1969 if !tu.IsTravis() { 1970 t.Parallel() 1971 } 1972 if !testutil.DockerIsConnected(t) { 1973 t.Skip("Docker not connected") 1974 } 1975 1976 brokenConfigs := []interface{}{ 1977 map[string]interface{}{ 1978 "host_path": "", 1979 }, 1980 map[string]interface{}{ 1981 "host_path": "/dev/sda1", 1982 "cgroup_permissions": "rxb", 1983 }, 1984 } 1985 1986 test_cases := []struct { 1987 deviceConfig interface{} 1988 err error 1989 }{ 1990 {[]interface{}{brokenConfigs[0]}, fmt.Errorf("host path must be set in configuration for devices")}, 1991 {[]interface{}{brokenConfigs[1]}, fmt.Errorf("invalid cgroup permission string: \"rxb\"")}, 1992 } 1993 1994 for _, tc := range test_cases { 1995 task, _, _ := dockerTask(t) 1996 task.Config["devices"] = tc.deviceConfig 1997 1998 ctx := testDockerDriverContexts(t, task) 1999 driver := NewDockerDriver(ctx.DriverCtx) 2000 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 2001 defer ctx.AllocDir.Destroy() 2002 2003 if _, err := driver.Prestart(ctx.ExecCtx, task); err == nil || err.Error() != tc.err.Error() { 2004 t.Fatalf("error expected in prestart, got %v, expected %v", err, tc.err) 2005 } 2006 } 2007 } 2008 2009 func TestDockerDriver_Device_Success(t *testing.T) { 2010 if !tu.IsTravis() { 2011 t.Parallel() 2012 } 2013 if !testutil.DockerIsConnected(t) { 2014 t.Skip("Docker not connected") 2015 } 2016 2017 if runtime.GOOS != "linux" { 2018 t.Skip("test device mounts only on linux") 2019 } 2020 2021 hostPath := "/dev/random" 2022 containerPath := "/dev/myrandom" 2023 perms := "rwm" 2024 2025 expectedDevice := docker.Device{ 2026 PathOnHost: hostPath, 2027 PathInContainer: containerPath, 2028 CgroupPermissions: perms, 2029 } 2030 config := map[string]interface{}{ 2031 "host_path": hostPath, 2032 "container_path": containerPath, 2033 } 2034 2035 task, _, _ := dockerTask(t) 2036 task.Config["devices"] = []interface{}{config} 2037 2038 client, handle, cleanup := dockerSetup(t, task) 2039 defer cleanup() 2040 2041 waitForExist(t, client, handle) 2042 2043 container, err := client.InspectContainer(handle.ContainerID()) 2044 if err != nil { 2045 t.Fatalf("err: %v", err) 2046 } 2047 2048 assert.NotEmpty(t, container.HostConfig.Devices, "Expected one device") 2049 assert.Equal(t, expectedDevice, container.HostConfig.Devices[0], "Incorrect device ") 2050 2051 } 2052 2053 func TestDockerDriver_Kill(t *testing.T) { 2054 assert := assert.New(t) 2055 if !tu.IsTravis() { 2056 t.Parallel() 2057 } 2058 if !testutil.DockerIsConnected(t) { 2059 t.Skip("Docker not connected") 2060 } 2061 2062 // Tasks started with a signal that is not supported should not error 2063 task := &structs.Task{ 2064 Name: "nc-demo", 2065 Driver: "docker", 2066 KillSignal: "SIGKILL", 2067 Config: map[string]interface{}{ 2068 "load": "busybox.tar", 2069 "image": "busybox", 2070 "command": "/bin/nc", 2071 "args": []string{"-l", "127.0.0.1", "-p", "0"}, 2072 }, 2073 LogConfig: &structs.LogConfig{ 2074 MaxFiles: 10, 2075 MaxFileSizeMB: 10, 2076 }, 2077 Resources: basicResources, 2078 } 2079 2080 ctx := testDockerDriverContexts(t, task) 2081 defer ctx.AllocDir.Destroy() 2082 d := NewDockerDriver(ctx.DriverCtx) 2083 copyImage(t, ctx.ExecCtx.TaskDir, "busybox.tar") 2084 2085 _, err := d.Prestart(ctx.ExecCtx, task) 2086 if err != nil { 2087 t.Fatalf("error in prestart: %v", err) 2088 } 2089 2090 resp, err := d.Start(ctx.ExecCtx, task) 2091 assert.Nil(err) 2092 assert.NotNil(resp.Handle) 2093 2094 handle := resp.Handle.(*DockerHandle) 2095 waitForExist(t, client, handle) 2096 err = handle.Kill() 2097 assert.Nil(err) 2098 }