github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/docker.go (about) 1 package driver 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "log" 7 "net" 8 "os" 9 "os/exec" 10 "path/filepath" 11 "regexp" 12 "strconv" 13 "strings" 14 "sync" 15 "time" 16 17 docker "github.com/fsouza/go-dockerclient" 18 19 "github.com/hashicorp/go-multierror" 20 "github.com/hashicorp/go-plugin" 21 "github.com/hashicorp/nomad/client/allocdir" 22 "github.com/hashicorp/nomad/client/config" 23 "github.com/hashicorp/nomad/client/driver/executor" 24 cstructs "github.com/hashicorp/nomad/client/driver/structs" 25 "github.com/hashicorp/nomad/helper/discover" 26 "github.com/hashicorp/nomad/helper/fields" 27 "github.com/hashicorp/nomad/nomad/structs" 28 "github.com/mitchellh/mapstructure" 29 ) 30 31 var ( 32 // We store the client globally to cache the connection to the docker daemon. 33 createClient sync.Once 34 client *docker.Client 35 ) 36 37 const ( 38 // NoSuchContainerError is returned by the docker daemon if the container 39 // does not exist. 40 NoSuchContainerError = "No such container" 41 42 // The key populated in Node Attributes to indicate presence of the Docker 43 // driver 44 dockerDriverAttr = "driver.docker" 45 46 // dockerTimeout is the length of time a request can be outstanding before 47 // it is timed out. 48 dockerTimeout = 1 * time.Minute 49 ) 50 51 type DockerDriver struct { 52 DriverContext 53 } 54 55 type DockerDriverAuth struct { 56 Username string `mapstructure:"username"` // username for the registry 57 Password string `mapstructure:"password"` // password to access the registry 58 Email string `mapstructure:"email"` // email address of the user who is allowed to access the registry 59 ServerAddress string `mapstructure:"server_address"` // server address of the registry 60 } 61 62 type DockerDriverConfig struct { 63 ImageName string `mapstructure:"image"` // Container's Image Name 64 LoadImages []string `mapstructure:"load"` // LoadImage is array of paths to image archive files 65 Command string `mapstructure:"command"` // The Command/Entrypoint to run when the container starts up 66 Args []string `mapstructure:"args"` // The arguments to the Command/Entrypoint 67 IpcMode string `mapstructure:"ipc_mode"` // The IPC mode of the container - host and none 68 NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, net and none 69 PidMode string `mapstructure:"pid_mode"` // The PID mode of the container - host and none 70 UTSMode string `mapstructure:"uts_mode"` // The UTS mode of the container - host and none 71 PortMapRaw []map[string]int `mapstructure:"port_map"` // 72 PortMap map[string]int `mapstructure:"-"` // A map of host port labels and the ports exposed on the container 73 Privileged bool `mapstructure:"privileged"` // Flag to run the container in privileged mode 74 DNSServers []string `mapstructure:"dns_servers"` // DNS Server for containers 75 DNSSearchDomains []string `mapstructure:"dns_search_domains"` // DNS Search domains for containers 76 Hostname string `mapstructure:"hostname"` // Hostname for containers 77 LabelsRaw []map[string]string `mapstructure:"labels"` // 78 Labels map[string]string `mapstructure:"-"` // Labels to set when the container starts up 79 Auth []DockerDriverAuth `mapstructure:"auth"` // Authentication credentials for a private Docker registry 80 SSL bool `mapstructure:"ssl"` // Flag indicating repository is served via https 81 TTY bool `mapstructure:"tty"` // Allocate a Pseudo-TTY 82 Interactive bool `mapstructure:"interactive"` // Keep STDIN open even if not attached 83 } 84 85 func (c *DockerDriverConfig) Init() error { 86 if strings.Contains(c.ImageName, "https://") { 87 c.SSL = true 88 c.ImageName = strings.Replace(c.ImageName, "https://", "", 1) 89 } 90 91 return nil 92 } 93 94 func (c *DockerDriverConfig) Validate() error { 95 if c.ImageName == "" { 96 return fmt.Errorf("Docker Driver needs an image name") 97 } 98 99 c.PortMap = mapMergeStrInt(c.PortMapRaw...) 100 c.Labels = mapMergeStrStr(c.LabelsRaw...) 101 102 return nil 103 } 104 105 type dockerPID struct { 106 Version string 107 ImageID string 108 ContainerID string 109 KillTimeout time.Duration 110 MaxKillTimeout time.Duration 111 PluginConfig *PluginReattachConfig 112 } 113 114 type DockerHandle struct { 115 pluginClient *plugin.Client 116 executor executor.Executor 117 client *docker.Client 118 logger *log.Logger 119 cleanupImage bool 120 imageID string 121 containerID string 122 version string 123 killTimeout time.Duration 124 maxKillTimeout time.Duration 125 waitCh chan *cstructs.WaitResult 126 doneCh chan struct{} 127 } 128 129 func NewDockerDriver(ctx *DriverContext) Driver { 130 return &DockerDriver{DriverContext: *ctx} 131 } 132 133 // Validate is used to validate the driver configuration 134 func (d *DockerDriver) Validate(config map[string]interface{}) error { 135 fd := &fields.FieldData{ 136 Raw: config, 137 Schema: map[string]*fields.FieldSchema{ 138 "image": &fields.FieldSchema{ 139 Type: fields.TypeString, 140 Required: true, 141 }, 142 "load": &fields.FieldSchema{ 143 Type: fields.TypeArray, 144 }, 145 "command": &fields.FieldSchema{ 146 Type: fields.TypeString, 147 }, 148 "args": &fields.FieldSchema{ 149 Type: fields.TypeArray, 150 }, 151 "ipc_mode": &fields.FieldSchema{ 152 Type: fields.TypeString, 153 }, 154 "network_mode": &fields.FieldSchema{ 155 Type: fields.TypeString, 156 }, 157 "pid_mode": &fields.FieldSchema{ 158 Type: fields.TypeString, 159 }, 160 "uts_mode": &fields.FieldSchema{ 161 Type: fields.TypeString, 162 }, 163 "port_map": &fields.FieldSchema{ 164 Type: fields.TypeArray, 165 }, 166 "privileged": &fields.FieldSchema{ 167 Type: fields.TypeBool, 168 }, 169 "dns_servers": &fields.FieldSchema{ 170 Type: fields.TypeArray, 171 }, 172 "dns_search_domains": &fields.FieldSchema{ 173 Type: fields.TypeArray, 174 }, 175 "hostname": &fields.FieldSchema{ 176 Type: fields.TypeString, 177 }, 178 "labels": &fields.FieldSchema{ 179 Type: fields.TypeArray, 180 }, 181 "auth": &fields.FieldSchema{ 182 Type: fields.TypeArray, 183 }, 184 "ssl": &fields.FieldSchema{ 185 Type: fields.TypeBool, 186 }, 187 "tty": &fields.FieldSchema{ 188 Type: fields.TypeBool, 189 }, 190 "interactive": &fields.FieldSchema{ 191 Type: fields.TypeBool, 192 }, 193 }, 194 } 195 196 if err := fd.Validate(); err != nil { 197 return err 198 } 199 200 return nil 201 } 202 203 // dockerClient creates *docker.Client. In test / dev mode we can use ENV vars 204 // to connect to the docker daemon. In production mode we will read 205 // docker.endpoint from the config file. 206 func (d *DockerDriver) dockerClient() (*docker.Client, error) { 207 if client != nil { 208 return client, nil 209 } 210 211 var err error 212 createClient.Do(func() { 213 // Default to using whatever is configured in docker.endpoint. If this is 214 // not specified we'll fall back on NewClientFromEnv which reads config from 215 // the DOCKER_* environment variables DOCKER_HOST, DOCKER_TLS_VERIFY, and 216 // DOCKER_CERT_PATH. This allows us to lock down the config in production 217 // but also accept the standard ENV configs for dev and test. 218 dockerEndpoint := d.config.Read("docker.endpoint") 219 if dockerEndpoint != "" { 220 cert := d.config.Read("docker.tls.cert") 221 key := d.config.Read("docker.tls.key") 222 ca := d.config.Read("docker.tls.ca") 223 224 if cert+key+ca != "" { 225 d.logger.Printf("[DEBUG] driver.docker: using TLS client connection to %s", dockerEndpoint) 226 client, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca) 227 } else { 228 d.logger.Printf("[DEBUG] driver.docker: using standard client connection to %s", dockerEndpoint) 229 client, err = docker.NewClient(dockerEndpoint) 230 } 231 client.HTTPClient.Timeout = dockerTimeout 232 return 233 } 234 235 d.logger.Println("[DEBUG] driver.docker: using client connection initialized from environment") 236 client, err = docker.NewClientFromEnv() 237 client.HTTPClient.Timeout = dockerTimeout 238 }) 239 return client, err 240 } 241 242 func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { 243 // Get the current status so that we can log any debug messages only if the 244 // state changes 245 _, currentlyEnabled := node.Attributes[dockerDriverAttr] 246 247 // Initialize docker API client 248 client, err := d.dockerClient() 249 if err != nil { 250 delete(node.Attributes, dockerDriverAttr) 251 if currentlyEnabled { 252 d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s", err) 253 } 254 return false, nil 255 } 256 257 privileged := d.config.ReadBoolDefault("docker.privileged.enabled", false) 258 if privileged { 259 node.Attributes["docker.privileged.enabled"] = "1" 260 } 261 262 // This is the first operation taken on the client so we'll try to 263 // establish a connection to the Docker daemon. If this fails it means 264 // Docker isn't available so we'll simply disable the docker driver. 265 env, err := client.Version() 266 if err != nil { 267 if currentlyEnabled { 268 d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon at %s: %s", client.Endpoint(), err) 269 } 270 delete(node.Attributes, dockerDriverAttr) 271 return false, nil 272 } 273 274 node.Attributes[dockerDriverAttr] = "1" 275 node.Attributes["driver.docker.version"] = env.Get("Version") 276 return true, nil 277 } 278 279 func (d *DockerDriver) containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, error) { 280 shared := alloc.SharedDir 281 local, ok := alloc.TaskDirs[task.Name] 282 if !ok { 283 return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name) 284 } 285 286 return []string{ 287 // "z" and "Z" option is to allocate directory with SELinux label. 288 fmt.Sprintf("%s:/%s:rw,z", shared, allocdir.SharedAllocName), 289 // capital "Z" will label with Multi-Category Security (MCS) labels 290 fmt.Sprintf("%s:/%s:rw,Z", local, allocdir.TaskLocal), 291 }, nil 292 } 293 294 // createContainer initializes a struct needed to call docker.client.CreateContainer() 295 func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task, 296 driverConfig *DockerDriverConfig, syslogAddr string) (docker.CreateContainerOptions, error) { 297 var c docker.CreateContainerOptions 298 if task.Resources == nil { 299 // Guard against missing resources. We should never have been able to 300 // schedule a job without specifying this. 301 d.logger.Println("[ERR] driver.docker: task.Resources is empty") 302 return c, fmt.Errorf("task.Resources is empty") 303 } 304 305 binds, err := d.containerBinds(ctx.AllocDir, task) 306 if err != nil { 307 return c, err 308 } 309 310 // Set environment variables. 311 d.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)) 312 d.taskEnv.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)) 313 314 config := &docker.Config{ 315 Image: driverConfig.ImageName, 316 Hostname: driverConfig.Hostname, 317 User: task.User, 318 Tty: driverConfig.TTY, 319 OpenStdin: driverConfig.Interactive, 320 } 321 322 hostConfig := &docker.HostConfig{ 323 // Convert MB to bytes. This is an absolute value. 324 Memory: int64(task.Resources.MemoryMB) * 1024 * 1024, 325 MemorySwap: -1, 326 // Convert Mhz to shares. This is a relative value. 327 CPUShares: int64(task.Resources.CPU), 328 329 // Binds are used to mount a host volume into the container. We mount a 330 // local directory for storage and a shared alloc directory that can be 331 // used to share data between different tasks in the same task group. 332 Binds: binds, 333 LogConfig: docker.LogConfig{ 334 Type: "syslog", 335 Config: map[string]string{ 336 "syslog-address": syslogAddr, 337 }, 338 }, 339 } 340 341 d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"]) 342 d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"]) 343 d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"]) 344 345 // set privileged mode 346 hostPrivileged := d.config.ReadBoolDefault("docker.privileged.enabled", false) 347 if driverConfig.Privileged && !hostPrivileged { 348 return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`) 349 } 350 hostConfig.Privileged = hostPrivileged 351 352 // set DNS servers 353 for _, ip := range driverConfig.DNSServers { 354 if net.ParseIP(ip) != nil { 355 hostConfig.DNS = append(hostConfig.DNS, ip) 356 } else { 357 d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip) 358 } 359 } 360 361 // set DNS search domains 362 for _, domain := range driverConfig.DNSSearchDomains { 363 hostConfig.DNSSearch = append(hostConfig.DNSSearch, domain) 364 } 365 366 if driverConfig.IpcMode != "" { 367 if !hostPrivileged { 368 return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting ipc mode not allowed`) 369 } 370 d.logger.Printf("[DEBUG] driver.docker: setting ipc mode to %s", driverConfig.IpcMode) 371 } 372 hostConfig.IpcMode = driverConfig.IpcMode 373 374 if driverConfig.PidMode != "" { 375 if !hostPrivileged { 376 return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting pid mode not allowed`) 377 } 378 d.logger.Printf("[DEBUG] driver.docker: setting pid mode to %s", driverConfig.PidMode) 379 } 380 hostConfig.PidMode = driverConfig.PidMode 381 382 if driverConfig.UTSMode != "" { 383 if !hostPrivileged { 384 return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting UTS mode not allowed`) 385 } 386 d.logger.Printf("[DEBUG] driver.docker: setting UTS mode to %s", driverConfig.UTSMode) 387 } 388 hostConfig.UTSMode = driverConfig.UTSMode 389 390 hostConfig.NetworkMode = driverConfig.NetworkMode 391 if hostConfig.NetworkMode == "" { 392 // docker default 393 d.logger.Println("[DEBUG] driver.docker: networking mode not specified; defaulting to bridge") 394 hostConfig.NetworkMode = "bridge" 395 } 396 397 // Setup port mapping and exposed ports 398 if len(task.Resources.Networks) == 0 { 399 d.logger.Println("[DEBUG] driver.docker: No network interfaces are available") 400 if len(driverConfig.PortMap) > 0 { 401 return c, fmt.Errorf("Trying to map ports but no network interface is available") 402 } 403 } else { 404 // TODO add support for more than one network 405 network := task.Resources.Networks[0] 406 publishedPorts := map[docker.Port][]docker.PortBinding{} 407 exposedPorts := map[docker.Port]struct{}{} 408 409 for _, port := range network.ReservedPorts { 410 // By default we will map the allocated port 1:1 to the container 411 containerPortInt := port.Value 412 413 // If the user has mapped a port using port_map we'll change it here 414 if mapped, ok := driverConfig.PortMap[port.Label]; ok { 415 containerPortInt = mapped 416 } 417 418 hostPortStr := strconv.Itoa(port.Value) 419 containerPort := docker.Port(strconv.Itoa(containerPortInt)) 420 421 publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} 422 publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} 423 d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value) 424 425 exposedPorts[containerPort+"/tcp"] = struct{}{} 426 exposedPorts[containerPort+"/udp"] = struct{}{} 427 d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value) 428 } 429 430 for _, port := range network.DynamicPorts { 431 // By default we will map the allocated port 1:1 to the container 432 containerPortInt := port.Value 433 434 // If the user has mapped a port using port_map we'll change it here 435 if mapped, ok := driverConfig.PortMap[port.Label]; ok { 436 containerPortInt = mapped 437 } 438 439 hostPortStr := strconv.Itoa(port.Value) 440 containerPort := docker.Port(strconv.Itoa(containerPortInt)) 441 442 publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} 443 publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}} 444 d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt) 445 446 exposedPorts[containerPort+"/tcp"] = struct{}{} 447 exposedPorts[containerPort+"/udp"] = struct{}{} 448 d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort) 449 } 450 451 d.taskEnv.SetPortMap(driverConfig.PortMap) 452 453 hostConfig.PortBindings = publishedPorts 454 config.ExposedPorts = exposedPorts 455 } 456 457 d.taskEnv.Build() 458 parsedArgs := d.taskEnv.ParseAndReplace(driverConfig.Args) 459 460 // If the user specified a custom command to run as their entrypoint, we'll 461 // inject it here. 462 if driverConfig.Command != "" { 463 // Validate command 464 if err := validateCommand(driverConfig.Command, "args"); err != nil { 465 return c, err 466 } 467 468 cmd := []string{driverConfig.Command} 469 if len(driverConfig.Args) != 0 { 470 cmd = append(cmd, parsedArgs...) 471 } 472 d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " ")) 473 config.Cmd = cmd 474 } else if len(driverConfig.Args) != 0 { 475 d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified") 476 } 477 478 if len(driverConfig.Labels) > 0 { 479 config.Labels = driverConfig.Labels 480 d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels) 481 } 482 483 config.Env = d.taskEnv.EnvList() 484 485 containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID) 486 d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName) 487 488 return docker.CreateContainerOptions{ 489 Name: containerName, 490 Config: config, 491 HostConfig: hostConfig, 492 }, nil 493 } 494 495 var ( 496 // imageNotFoundMatcher is a regex expression that matches the image not 497 // found error Docker returns. 498 imageNotFoundMatcher = regexp.MustCompile(`Error: image .+ not found`) 499 ) 500 501 // recoverablePullError wraps the error gotten when trying to pull and image if 502 // the error is recoverable. 503 func (d *DockerDriver) recoverablePullError(err error, image string) error { 504 recoverable := true 505 if imageNotFoundMatcher.MatchString(err.Error()) { 506 recoverable = false 507 } 508 return cstructs.NewRecoverableError(fmt.Errorf("Failed to pull `%s`: %s", image, err), recoverable) 509 } 510 511 func (d *DockerDriver) Periodic() (bool, time.Duration) { 512 return true, 15 * time.Second 513 } 514 515 // createImage creates a docker image either by pulling it from a registry or by 516 // loading it from the file system 517 func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir string) error { 518 image := driverConfig.ImageName 519 repo, tag := docker.ParseRepositoryTag(image) 520 if tag == "" { 521 tag = "latest" 522 } 523 524 var dockerImage *docker.Image 525 var err error 526 // We're going to check whether the image is already downloaded. If the tag 527 // is "latest" we have to check for a new version every time so we don't 528 // bother to check and cache the id here. We'll download first, then cache. 529 if tag != "latest" { 530 dockerImage, err = client.InspectImage(image) 531 } 532 533 // Download the image 534 if dockerImage == nil { 535 if len(driverConfig.LoadImages) > 0 { 536 return d.loadImage(driverConfig, client, taskDir) 537 } 538 539 return d.pullImage(driverConfig, client, repo, tag) 540 } 541 return err 542 } 543 544 // pullImage creates an image by pulling it from a docker registry 545 func (d *DockerDriver) pullImage(driverConfig *DockerDriverConfig, client *docker.Client, repo string, tag string) error { 546 pullOptions := docker.PullImageOptions{ 547 Repository: repo, 548 Tag: tag, 549 } 550 551 authOptions := docker.AuthConfiguration{} 552 if len(driverConfig.Auth) != 0 { 553 authOptions = docker.AuthConfiguration{ 554 Username: driverConfig.Auth[0].Username, 555 Password: driverConfig.Auth[0].Password, 556 Email: driverConfig.Auth[0].Email, 557 ServerAddress: driverConfig.Auth[0].ServerAddress, 558 } 559 } 560 561 if authConfigFile := d.config.Read("docker.auth.config"); authConfigFile != "" { 562 if f, err := os.Open(authConfigFile); err == nil { 563 defer f.Close() 564 var authConfigurations *docker.AuthConfigurations 565 if authConfigurations, err = docker.NewAuthConfigurations(f); err != nil { 566 return fmt.Errorf("Failed to create docker auth object: %v", err) 567 } 568 569 authConfigurationKey := "" 570 if driverConfig.SSL { 571 authConfigurationKey += "https://" 572 } 573 574 authConfigurationKey += strings.Split(driverConfig.ImageName, "/")[0] 575 if authConfiguration, ok := authConfigurations.Configs[authConfigurationKey]; ok { 576 authOptions = authConfiguration 577 } 578 } else { 579 return fmt.Errorf("Failed to open auth config file: %v, error: %v", authConfigFile, err) 580 } 581 } 582 583 err := client.PullImage(pullOptions, authOptions) 584 if err != nil { 585 d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err) 586 return d.recoverablePullError(err, driverConfig.ImageName) 587 } 588 d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag) 589 return nil 590 } 591 592 // loadImage creates an image by loading it from the file system 593 func (d *DockerDriver) loadImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir string) error { 594 var errors multierror.Error 595 for _, image := range driverConfig.LoadImages { 596 archive := filepath.Join(taskDir, allocdir.TaskLocal, image) 597 d.logger.Printf("[DEBUG] driver.docker: loading image from: %v", archive) 598 f, err := os.Open(archive) 599 if err != nil { 600 errors.Errors = append(errors.Errors, fmt.Errorf("unable to open image archive: %v", err)) 601 continue 602 } 603 if err := client.LoadImage(docker.LoadImageOptions{InputStream: f}); err != nil { 604 errors.Errors = append(errors.Errors, err) 605 } 606 f.Close() 607 } 608 return errors.ErrorOrNil() 609 } 610 611 func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { 612 var driverConfig DockerDriverConfig 613 if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil { 614 return nil, err 615 } 616 617 if err := driverConfig.Init(); err != nil { 618 return nil, err 619 } 620 621 if err := driverConfig.Validate(); err != nil { 622 return nil, err 623 } 624 625 cleanupImage := d.config.ReadBoolDefault("docker.cleanup.image", true) 626 627 taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName] 628 if !ok { 629 return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName) 630 } 631 632 // Initialize docker API client 633 client, err := d.dockerClient() 634 if err != nil { 635 return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) 636 } 637 638 if err := d.createImage(&driverConfig, client, taskDir); err != nil { 639 return nil, fmt.Errorf("failed to create image: %v", err) 640 } 641 642 image := driverConfig.ImageName 643 // Now that we have the image we can get the image id 644 dockerImage, err := client.InspectImage(image) 645 if err != nil { 646 d.logger.Printf("[ERR] driver.docker: failed getting image id for %s: %s", image, err) 647 return nil, fmt.Errorf("Failed to determine image id for `%s`: %s", image, err) 648 } 649 d.logger.Printf("[DEBUG] driver.docker: identified image %s as %s", image, dockerImage.ID) 650 651 bin, err := discover.NomadExecutable() 652 if err != nil { 653 return nil, fmt.Errorf("unable to find the nomad binary: %v", err) 654 } 655 pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name)) 656 pluginConfig := &plugin.ClientConfig{ 657 Cmd: exec.Command(bin, "executor", pluginLogFile), 658 } 659 660 exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 661 if err != nil { 662 return nil, err 663 } 664 executorCtx := &executor.ExecutorContext{ 665 TaskEnv: d.taskEnv, 666 Task: task, 667 Driver: "docker", 668 AllocDir: ctx.AllocDir, 669 AllocID: ctx.AllocID, 670 PortLowerBound: d.config.ClientMinPort, 671 PortUpperBound: d.config.ClientMaxPort, 672 } 673 ss, err := exec.LaunchSyslogServer(executorCtx) 674 if err != nil { 675 return nil, fmt.Errorf("failed to start syslog collector: %v", err) 676 } 677 678 config, err := d.createContainer(ctx, task, &driverConfig, ss.Addr) 679 if err != nil { 680 d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %s: %s", image, err) 681 pluginClient.Kill() 682 return nil, fmt.Errorf("Failed to create container configuration for image %s: %s", image, err) 683 } 684 // Create a container 685 container, err := client.CreateContainer(config) 686 if err != nil { 687 // If the container already exists because of a previous failure we'll 688 // try to purge it and re-create it. 689 if strings.Contains(err.Error(), "container already exists") { 690 // Get the ID of the existing container so we can delete it 691 containers, err := client.ListContainers(docker.ListContainersOptions{ 692 // The image might be in use by a stopped container, so check everything 693 All: true, 694 Filters: map[string][]string{ 695 "name": []string{config.Name}, 696 }, 697 }) 698 if err != nil { 699 d.logger.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name) 700 pluginClient.Kill() 701 return nil, fmt.Errorf("Failed to query list of containers: %s", err) 702 } 703 704 // Couldn't find any matching containers 705 if len(containers) == 0 { 706 d.logger.Printf("[ERR] driver.docker: failed to get id for container %s: %#v", config.Name, containers) 707 pluginClient.Kill() 708 return nil, fmt.Errorf("Failed to get id for container %s", config.Name) 709 } 710 711 // Delete matching containers 712 d.logger.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create", config.Name) 713 for _, container := range containers { 714 err = client.RemoveContainer(docker.RemoveContainerOptions{ 715 ID: container.ID, 716 }) 717 if err != nil { 718 d.logger.Printf("[ERR] driver.docker: failed to purge container %s", container.ID) 719 pluginClient.Kill() 720 return nil, fmt.Errorf("Failed to purge container %s: %s", container.ID, err) 721 } 722 d.logger.Printf("[INFO] driver.docker: purged container %s", container.ID) 723 } 724 725 container, err = client.CreateContainer(config) 726 if err != nil { 727 d.logger.Printf("[ERR] driver.docker: failed to re-create container %s; aborting", config.Name) 728 pluginClient.Kill() 729 return nil, fmt.Errorf("Failed to re-create container %s; aborting", config.Name) 730 } 731 } else { 732 // We failed to create the container for some other reason. 733 d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s", image, err) 734 pluginClient.Kill() 735 return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err) 736 } 737 } 738 d.logger.Printf("[INFO] driver.docker: created container %s", container.ID) 739 740 // Start the container 741 err = client.StartContainer(container.ID, container.HostConfig) 742 if err != nil { 743 d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err) 744 pluginClient.Kill() 745 return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err) 746 } 747 d.logger.Printf("[INFO] driver.docker: started container %s", container.ID) 748 749 // Return a driver handle 750 maxKill := d.DriverContext.config.MaxKillTimeout 751 h := &DockerHandle{ 752 client: client, 753 executor: exec, 754 pluginClient: pluginClient, 755 cleanupImage: cleanupImage, 756 logger: d.logger, 757 imageID: dockerImage.ID, 758 containerID: container.ID, 759 version: d.config.Version, 760 killTimeout: GetKillTimeout(task.KillTimeout, maxKill), 761 maxKillTimeout: maxKill, 762 doneCh: make(chan struct{}), 763 waitCh: make(chan *cstructs.WaitResult, 1), 764 } 765 if err := exec.SyncServices(consulContext(d.config, container.ID)); err != nil { 766 d.logger.Printf("[ERR] driver.docker: error registering services with consul for task: %q: %v", task.Name, err) 767 } 768 go h.run() 769 return h, nil 770 } 771 772 func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { 773 cleanupImage := d.config.ReadBoolDefault("docker.cleanup.image", true) 774 775 // Split the handle 776 pidBytes := []byte(strings.TrimPrefix(handleID, "DOCKER:")) 777 pid := &dockerPID{} 778 if err := json.Unmarshal(pidBytes, pid); err != nil { 779 return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) 780 } 781 d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", pid.ContainerID) 782 d.logger.Printf("[DEBUG] driver.docker: re-attached to handle: %s", handleID) 783 pluginConfig := &plugin.ClientConfig{ 784 Reattach: pid.PluginConfig.PluginConfig(), 785 } 786 787 client, err := d.dockerClient() 788 if err != nil { 789 return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) 790 } 791 792 // Look for a running container with this ID 793 containers, err := client.ListContainers(docker.ListContainersOptions{ 794 Filters: map[string][]string{ 795 "id": []string{pid.ContainerID}, 796 }, 797 }) 798 if err != nil { 799 return nil, fmt.Errorf("Failed to query for container %s: %v", pid.ContainerID, err) 800 } 801 802 found := false 803 for _, container := range containers { 804 if container.ID == pid.ContainerID { 805 found = true 806 } 807 } 808 if !found { 809 return nil, fmt.Errorf("Failed to find container %s: %v", pid.ContainerID, err) 810 } 811 exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config) 812 if err != nil { 813 d.logger.Printf("[INFO] driver.docker: couldn't re-attach to the plugin process: %v", err) 814 if e := client.StopContainer(pid.ContainerID, uint(pid.KillTimeout*time.Second)); e != nil { 815 d.logger.Printf("[DEBUG] driver.docker: couldn't stop container: %v", e) 816 } 817 return nil, err 818 } 819 820 ver, _ := exec.Version() 821 d.logger.Printf("[DEBUG] driver.docker: version of executor: %v", ver.Version) 822 823 // Return a driver handle 824 h := &DockerHandle{ 825 client: client, 826 executor: exec, 827 pluginClient: pluginClient, 828 cleanupImage: cleanupImage, 829 logger: d.logger, 830 imageID: pid.ImageID, 831 containerID: pid.ContainerID, 832 version: pid.Version, 833 killTimeout: pid.KillTimeout, 834 maxKillTimeout: pid.MaxKillTimeout, 835 doneCh: make(chan struct{}), 836 waitCh: make(chan *cstructs.WaitResult, 1), 837 } 838 if err := exec.SyncServices(consulContext(d.config, pid.ContainerID)); err != nil { 839 h.logger.Printf("[ERR] driver.docker: error registering services with consul: %v", err) 840 } 841 842 go h.run() 843 return h, nil 844 } 845 846 func (h *DockerHandle) ID() string { 847 // Return a handle to the PID 848 pid := dockerPID{ 849 Version: h.version, 850 ImageID: h.imageID, 851 ContainerID: h.containerID, 852 KillTimeout: h.killTimeout, 853 MaxKillTimeout: h.maxKillTimeout, 854 PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), 855 } 856 data, err := json.Marshal(pid) 857 if err != nil { 858 h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s", err) 859 } 860 return fmt.Sprintf("DOCKER:%s", string(data)) 861 } 862 863 func (h *DockerHandle) ContainerID() string { 864 return h.containerID 865 } 866 867 func (h *DockerHandle) WaitCh() chan *cstructs.WaitResult { 868 return h.waitCh 869 } 870 871 func (h *DockerHandle) Update(task *structs.Task) error { 872 // Store the updated kill timeout. 873 h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout) 874 if err := h.executor.UpdateTask(task); err != nil { 875 h.logger.Printf("[DEBUG] driver.docker: failed to update log config: %v", err) 876 } 877 878 // Update is not possible 879 return nil 880 } 881 882 // Kill is used to terminate the task. This uses `docker stop -t killTimeout` 883 func (h *DockerHandle) Kill() error { 884 // Stop the container 885 err := h.client.StopContainer(h.containerID, uint(h.killTimeout.Seconds())) 886 if err != nil { 887 h.executor.Exit() 888 h.pluginClient.Kill() 889 890 // Container has already been removed. 891 if strings.Contains(err.Error(), NoSuchContainerError) { 892 h.logger.Printf("[DEBUG] driver.docker: attempted to stop non-existent container %s", h.containerID) 893 return nil 894 } 895 h.logger.Printf("[ERR] driver.docker: failed to stop container %s: %v", h.containerID, err) 896 return fmt.Errorf("Failed to stop container %s: %s", h.containerID, err) 897 } 898 h.logger.Printf("[INFO] driver.docker: stopped container %s", h.containerID) 899 return nil 900 } 901 902 func (h *DockerHandle) run() { 903 // Wait for it... 904 exitCode, err := h.client.WaitContainer(h.containerID) 905 if err != nil { 906 h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated", h.containerID) 907 } 908 909 if exitCode != 0 { 910 err = fmt.Errorf("Docker container exited with non-zero exit code: %d", exitCode) 911 } 912 913 close(h.doneCh) 914 h.waitCh <- cstructs.NewWaitResult(exitCode, 0, err) 915 close(h.waitCh) 916 917 // Remove services 918 if err := h.executor.DeregisterServices(); err != nil { 919 h.logger.Printf("[ERR] driver.docker: error deregistering services: %v", err) 920 } 921 922 // Shutdown the syslog collector 923 if err := h.executor.Exit(); err != nil { 924 h.logger.Printf("[ERR] driver.docker: failed to kill the syslog collector: %v", err) 925 } 926 h.pluginClient.Kill() 927 928 // Stop the container just incase the docker daemon's wait returned 929 // incorrectly 930 if err := h.client.StopContainer(h.containerID, 0); err != nil { 931 _, noSuchContainer := err.(*docker.NoSuchContainer) 932 _, containerNotRunning := err.(*docker.ContainerNotRunning) 933 if !containerNotRunning && !noSuchContainer { 934 h.logger.Printf("[ERR] driver.docker: error stopping container: %v", err) 935 } 936 } 937 938 // Remove the container 939 if err := h.client.RemoveContainer(docker.RemoveContainerOptions{ID: h.containerID, Force: true}); err != nil { 940 h.logger.Printf("[ERR] driver.docker: error removing container: %v", err) 941 } 942 943 // Cleanup the image 944 if h.cleanupImage { 945 if err := h.client.RemoveImage(h.imageID); err != nil { 946 h.logger.Printf("[DEBUG] driver.docker: error removing image: %v", err) 947 } 948 } 949 }