github.com/taylorchu/nomad@v0.5.3-rc1.0.20170407200202-db11e7dd7b55/client/driver/docker.go (about) 1 package driver 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "log" 7 "net" 8 "os" 9 "path/filepath" 10 "runtime" 11 "strconv" 12 "strings" 13 "sync" 14 "syscall" 15 "time" 16 17 docker "github.com/fsouza/go-dockerclient" 18 19 "github.com/docker/docker/cli/config/configfile" 20 "github.com/docker/docker/reference" 21 "github.com/docker/docker/registry" 22 23 "github.com/hashicorp/go-multierror" 24 "github.com/hashicorp/go-plugin" 25 "github.com/hashicorp/nomad/client/allocdir" 26 "github.com/hashicorp/nomad/client/config" 27 "github.com/hashicorp/nomad/client/driver/env" 28 "github.com/hashicorp/nomad/client/driver/executor" 29 dstructs "github.com/hashicorp/nomad/client/driver/structs" 30 cstructs "github.com/hashicorp/nomad/client/structs" 31 "github.com/hashicorp/nomad/helper" 32 "github.com/hashicorp/nomad/helper/fields" 33 shelpers "github.com/hashicorp/nomad/helper/stats" 34 "github.com/hashicorp/nomad/nomad/structs" 35 "github.com/mitchellh/mapstructure" 36 ) 37 38 var ( 39 // We store the clients globally to cache the connection to the docker daemon. 40 createClients sync.Once 41 42 // client is a docker client with a timeout of 1 minute. This is for doing 43 // all operations with the docker daemon besides which are not long running 44 // such as creating, killing containers, etc. 45 client *docker.Client 46 47 // waitClient is a docker client with no timeouts. This is used for long 48 // running operations such as waiting on containers and collect stats 49 waitClient *docker.Client 50 51 // The statistics the Docker driver exposes 52 DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"} 53 DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"} 54 55 // recoverableErrTimeouts returns a recoverable error if the error was due 56 // to timeouts 57 recoverableErrTimeouts = func(err error) error { 58 r := false 59 if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") || 60 strings.Contains(err.Error(), "EOF") { 61 r = true 62 } 63 return structs.NewRecoverableError(err, r) 64 } 65 ) 66 67 const ( 68 // NoSuchContainerError is returned by the docker daemon if the container 69 // does not exist. 70 NoSuchContainerError = "No such container" 71 72 // The key populated in Node Attributes to indicate presence of the Docker 73 // driver 74 dockerDriverAttr = "driver.docker" 75 76 // dockerSELinuxLabelConfigOption is the key for configuring the 77 // SELinux label for binds. 78 dockerSELinuxLabelConfigOption = "docker.volumes.selinuxlabel" 79 80 // dockerVolumesConfigOption is the key for enabling the use of custom 81 // bind volumes to arbitrary host paths. 82 dockerVolumesConfigOption = "docker.volumes.enabled" 83 dockerVolumesConfigDefault = true 84 85 // dockerPrivilegedConfigOption is the key for running containers in 86 // Docker's privileged mode. 87 dockerPrivilegedConfigOption = "docker.privileged.enabled" 88 89 // dockerCleanupImageConfigOption is the key for whether or not to 90 // cleanup images after the task exits. 91 dockerCleanupImageConfigOption = "docker.cleanup.image" 92 dockerCleanupImageConfigDefault = true 93 94 // dockerPullTimeoutConfigOption is the key for setting an images pull 95 // timeout 96 dockerImageRemoveDelayConfigOption = "docker.cleanup.image.delay" 97 dockerImageRemoveDelayConfigDefault = 3 * time.Minute 98 99 // dockerTimeout is the length of time a request can be outstanding before 100 // it is timed out. 101 dockerTimeout = 5 * time.Minute 102 103 // dockerImageResKey is the CreatedResources key for docker images 104 dockerImageResKey = "image" 105 ) 106 107 type DockerDriver struct { 108 DriverContext 109 110 driverConfig *DockerDriverConfig 111 imageID string 112 113 // A tri-state boolean to know if the fingerprinting has happened and 114 // whether it has been successful 115 fingerprintSuccess *bool 116 } 117 118 type DockerDriverAuth struct { 119 Username string `mapstructure:"username"` // username for the registry 120 Password string `mapstructure:"password"` // password to access the registry 121 Email string `mapstructure:"email"` // email address of the user who is allowed to access the registry 122 ServerAddress string `mapstructure:"server_address"` // server address of the registry 123 } 124 125 type DockerLoggingOpts struct { 126 Type string `mapstructure:"type"` 127 ConfigRaw []map[string]string `mapstructure:"config"` 128 Config map[string]string `mapstructure:"-"` 129 } 130 131 type DockerDriverConfig struct { 132 ImageName string `mapstructure:"image"` // Container's Image Name 133 LoadImage string `mapstructure:"load"` // LoadImage is a path to an image archive file 134 Command string `mapstructure:"command"` // The Command to run when the container starts up 135 Args []string `mapstructure:"args"` // The arguments to the Command 136 IpcMode string `mapstructure:"ipc_mode"` // The IPC mode of the container - host and none 137 NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, nat and none 138 NetworkAliases []string `mapstructure:"network_aliases"` // The network-scoped alias for the container 139 IPv4Address string `mapstructure:"ipv4_address"` // The container ipv4 address 140 IPv6Address string `mapstructure:"ipv6_address"` // the container ipv6 address 141 PidMode string `mapstructure:"pid_mode"` // The PID mode of the container - host and none 142 UTSMode string `mapstructure:"uts_mode"` // The UTS mode of the container - host and none 143 UsernsMode string `mapstructure:"userns_mode"` // The User namespace mode of the container - host and none 144 PortMapRaw []map[string]int `mapstructure:"port_map"` // 145 PortMap map[string]int `mapstructure:"-"` // A map of host port labels and the ports exposed on the container 146 Privileged bool `mapstructure:"privileged"` // Flag to run the container in privileged mode 147 DNSServers []string `mapstructure:"dns_servers"` // DNS Server for containers 148 DNSSearchDomains []string `mapstructure:"dns_search_domains"` // DNS Search domains for containers 149 Hostname string `mapstructure:"hostname"` // Hostname for containers 150 LabelsRaw []map[string]string `mapstructure:"labels"` // 151 Labels map[string]string `mapstructure:"-"` // Labels to set when the container starts up 152 Auth []DockerDriverAuth `mapstructure:"auth"` // Authentication credentials for a private Docker registry 153 TTY bool `mapstructure:"tty"` // Allocate a Pseudo-TTY 154 Interactive bool `mapstructure:"interactive"` // Keep STDIN open even if not attached 155 ShmSize int64 `mapstructure:"shm_size"` // Size of /dev/shm of the container in bytes 156 WorkDir string `mapstructure:"work_dir"` // Working directory inside the container 157 Logging []DockerLoggingOpts `mapstructure:"logging"` // Logging options for syslog server 158 Volumes []string `mapstructure:"volumes"` // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container 159 VolumeDriver string `mapstructure:"volume_driver"` // Docker volume driver used for the container's volumes 160 ForcePull bool `mapstructure:"force_pull"` // Always force pull before running image, useful if your tags are mutable 161 } 162 163 // Validate validates a docker driver config 164 func (c *DockerDriverConfig) Validate() error { 165 if c.ImageName == "" { 166 return fmt.Errorf("Docker Driver needs an image name") 167 } 168 169 c.PortMap = mapMergeStrInt(c.PortMapRaw...) 170 c.Labels = mapMergeStrStr(c.LabelsRaw...) 171 if len(c.Logging) > 0 { 172 c.Logging[0].Config = mapMergeStrStr(c.Logging[0].ConfigRaw...) 173 } 174 return nil 175 } 176 177 // NewDockerDriverConfig returns a docker driver config by parsing the HCL 178 // config 179 func NewDockerDriverConfig(task *structs.Task, env *env.TaskEnvironment) (*DockerDriverConfig, error) { 180 var dconf DockerDriverConfig 181 182 if err := mapstructure.WeakDecode(task.Config, &dconf); err != nil { 183 return nil, err 184 } 185 186 // Interpolate everthing that is a string 187 dconf.ImageName = env.ReplaceEnv(dconf.ImageName) 188 dconf.Command = env.ReplaceEnv(dconf.Command) 189 dconf.IpcMode = env.ReplaceEnv(dconf.IpcMode) 190 dconf.NetworkMode = env.ReplaceEnv(dconf.NetworkMode) 191 dconf.NetworkAliases = env.ParseAndReplace(dconf.NetworkAliases) 192 dconf.IPv4Address = env.ReplaceEnv(dconf.IPv4Address) 193 dconf.IPv6Address = env.ReplaceEnv(dconf.IPv6Address) 194 dconf.PidMode = env.ReplaceEnv(dconf.PidMode) 195 dconf.UTSMode = env.ReplaceEnv(dconf.UTSMode) 196 dconf.Hostname = env.ReplaceEnv(dconf.Hostname) 197 dconf.WorkDir = env.ReplaceEnv(dconf.WorkDir) 198 dconf.LoadImage = env.ReplaceEnv(dconf.LoadImage) 199 dconf.Volumes = env.ParseAndReplace(dconf.Volumes) 200 dconf.VolumeDriver = env.ReplaceEnv(dconf.VolumeDriver) 201 dconf.DNSServers = env.ParseAndReplace(dconf.DNSServers) 202 dconf.DNSSearchDomains = env.ParseAndReplace(dconf.DNSSearchDomains) 203 204 for _, m := range dconf.LabelsRaw { 205 for k, v := range m { 206 delete(m, k) 207 m[env.ReplaceEnv(k)] = env.ReplaceEnv(v) 208 } 209 } 210 211 for i, a := range dconf.Auth { 212 dconf.Auth[i].Username = env.ReplaceEnv(a.Username) 213 dconf.Auth[i].Password = env.ReplaceEnv(a.Password) 214 dconf.Auth[i].Email = env.ReplaceEnv(a.Email) 215 dconf.Auth[i].ServerAddress = env.ReplaceEnv(a.ServerAddress) 216 } 217 218 for i, l := range dconf.Logging { 219 dconf.Logging[i].Type = env.ReplaceEnv(l.Type) 220 for _, c := range l.ConfigRaw { 221 for k, v := range c { 222 delete(c, k) 223 c[env.ReplaceEnv(k)] = env.ReplaceEnv(v) 224 } 225 } 226 } 227 228 for _, m := range dconf.PortMapRaw { 229 for k, v := range m { 230 delete(m, k) 231 m[env.ReplaceEnv(k)] = v 232 } 233 } 234 235 // Remove any http 236 if strings.Contains(dconf.ImageName, "https://") { 237 dconf.ImageName = strings.Replace(dconf.ImageName, "https://", "", 1) 238 } 239 240 if err := dconf.Validate(); err != nil { 241 return nil, err 242 } 243 return &dconf, nil 244 } 245 246 type dockerPID struct { 247 Version string 248 Image string 249 ImageID string 250 ContainerID string 251 KillTimeout time.Duration 252 MaxKillTimeout time.Duration 253 PluginConfig *PluginReattachConfig 254 } 255 256 type DockerHandle struct { 257 pluginClient *plugin.Client 258 executor executor.Executor 259 client *docker.Client 260 waitClient *docker.Client 261 logger *log.Logger 262 Image string 263 ImageID string 264 containerID string 265 version string 266 clkSpeed float64 267 killTimeout time.Duration 268 maxKillTimeout time.Duration 269 resourceUsageLock sync.RWMutex 270 resourceUsage *cstructs.TaskResourceUsage 271 waitCh chan *dstructs.WaitResult 272 doneCh chan bool 273 } 274 275 func NewDockerDriver(ctx *DriverContext) Driver { 276 return &DockerDriver{DriverContext: *ctx} 277 } 278 279 func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) { 280 // Initialize docker API clients 281 client, _, err := d.dockerClients() 282 if err != nil { 283 if d.fingerprintSuccess == nil || *d.fingerprintSuccess { 284 d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s", err) 285 } 286 delete(node.Attributes, dockerDriverAttr) 287 d.fingerprintSuccess = helper.BoolToPtr(false) 288 return false, nil 289 } 290 291 // This is the first operation taken on the client so we'll try to 292 // establish a connection to the Docker daemon. If this fails it means 293 // Docker isn't available so we'll simply disable the docker driver. 294 env, err := client.Version() 295 if err != nil { 296 delete(node.Attributes, dockerDriverAttr) 297 if d.fingerprintSuccess == nil || *d.fingerprintSuccess { 298 d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon at %s: %s", client.Endpoint(), err) 299 } 300 d.fingerprintSuccess = helper.BoolToPtr(false) 301 return false, nil 302 } 303 304 node.Attributes[dockerDriverAttr] = "1" 305 node.Attributes["driver.docker.version"] = env.Get("Version") 306 307 privileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false) 308 if privileged { 309 node.Attributes[dockerPrivilegedConfigOption] = "1" 310 } 311 312 // Advertise if this node supports Docker volumes 313 if d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault) { 314 node.Attributes["driver."+dockerVolumesConfigOption] = "1" 315 } 316 317 d.fingerprintSuccess = helper.BoolToPtr(true) 318 return true, nil 319 } 320 321 // Validate is used to validate the driver configuration 322 func (d *DockerDriver) Validate(config map[string]interface{}) error { 323 fd := &fields.FieldData{ 324 Raw: config, 325 Schema: map[string]*fields.FieldSchema{ 326 "image": &fields.FieldSchema{ 327 Type: fields.TypeString, 328 Required: true, 329 }, 330 "load": &fields.FieldSchema{ 331 Type: fields.TypeString, 332 }, 333 "command": &fields.FieldSchema{ 334 Type: fields.TypeString, 335 }, 336 "args": &fields.FieldSchema{ 337 Type: fields.TypeArray, 338 }, 339 "ipc_mode": &fields.FieldSchema{ 340 Type: fields.TypeString, 341 }, 342 "network_mode": &fields.FieldSchema{ 343 Type: fields.TypeString, 344 }, 345 "network_aliases": &fields.FieldSchema{ 346 Type: fields.TypeArray, 347 }, 348 "ipv4_address": &fields.FieldSchema{ 349 Type: fields.TypeString, 350 }, 351 "ipv6_address": &fields.FieldSchema{ 352 Type: fields.TypeString, 353 }, 354 "pid_mode": &fields.FieldSchema{ 355 Type: fields.TypeString, 356 }, 357 "uts_mode": &fields.FieldSchema{ 358 Type: fields.TypeString, 359 }, 360 "userns_mode": &fields.FieldSchema{ 361 Type: fields.TypeString, 362 }, 363 "port_map": &fields.FieldSchema{ 364 Type: fields.TypeArray, 365 }, 366 "privileged": &fields.FieldSchema{ 367 Type: fields.TypeBool, 368 }, 369 "dns_servers": &fields.FieldSchema{ 370 Type: fields.TypeArray, 371 }, 372 "dns_search_domains": &fields.FieldSchema{ 373 Type: fields.TypeArray, 374 }, 375 "hostname": &fields.FieldSchema{ 376 Type: fields.TypeString, 377 }, 378 "labels": &fields.FieldSchema{ 379 Type: fields.TypeArray, 380 }, 381 "auth": &fields.FieldSchema{ 382 Type: fields.TypeArray, 383 }, 384 // COMPAT: Remove in 0.6.0. SSL is no longer needed 385 "ssl": &fields.FieldSchema{ 386 Type: fields.TypeBool, 387 }, 388 "tty": &fields.FieldSchema{ 389 Type: fields.TypeBool, 390 }, 391 "interactive": &fields.FieldSchema{ 392 Type: fields.TypeBool, 393 }, 394 "shm_size": &fields.FieldSchema{ 395 Type: fields.TypeInt, 396 }, 397 "work_dir": &fields.FieldSchema{ 398 Type: fields.TypeString, 399 }, 400 "logging": &fields.FieldSchema{ 401 Type: fields.TypeArray, 402 }, 403 "volumes": &fields.FieldSchema{ 404 Type: fields.TypeArray, 405 }, 406 "volume_driver": &fields.FieldSchema{ 407 Type: fields.TypeString, 408 }, 409 "force_pull": &fields.FieldSchema{ 410 Type: fields.TypeBool, 411 }, 412 }, 413 } 414 415 if err := fd.Validate(); err != nil { 416 return err 417 } 418 419 return nil 420 } 421 422 func (d *DockerDriver) Abilities() DriverAbilities { 423 return DriverAbilities{ 424 SendSignals: true, 425 } 426 } 427 428 func (d *DockerDriver) FSIsolation() cstructs.FSIsolation { 429 return cstructs.FSIsolationImage 430 } 431 432 // getDockerCoordinator returns the docker coordinator and the caller ID to use when 433 // interacting with the coordinator 434 func (d *DockerDriver) getDockerCoordinator(client *docker.Client) (*dockerCoordinator, string) { 435 config := &dockerCoordinatorConfig{ 436 client: client, 437 cleanup: d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault), 438 logger: d.logger, 439 removeDelay: d.config.ReadDurationDefault(dockerImageRemoveDelayConfigOption, dockerImageRemoveDelayConfigDefault), 440 } 441 442 return GetDockerCoordinator(config), fmt.Sprintf("%s-%s", d.DriverContext.allocID, d.DriverContext.taskName) 443 } 444 445 func (d *DockerDriver) Prestart(ctx *ExecContext, task *structs.Task) (*CreatedResources, error) { 446 driverConfig, err := NewDockerDriverConfig(task, d.taskEnv) 447 if err != nil { 448 return nil, err 449 } 450 451 // Set state needed by Start() 452 d.driverConfig = driverConfig 453 454 // Initialize docker API clients 455 client, _, err := d.dockerClients() 456 if err != nil { 457 return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) 458 } 459 460 // Ensure the image is available 461 id, err := d.createImage(driverConfig, client, ctx.TaskDir) 462 if err != nil { 463 return nil, err 464 } 465 466 res := NewCreatedResources() 467 res.Add(dockerImageResKey, id) 468 d.imageID = id 469 return res, nil 470 } 471 472 func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) { 473 474 pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") 475 executorConfig := &dstructs.ExecutorConfig{ 476 LogFile: pluginLogFile, 477 LogLevel: d.config.LogLevel, 478 } 479 480 exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) 481 if err != nil { 482 return nil, err 483 } 484 executorCtx := &executor.ExecutorContext{ 485 TaskEnv: d.taskEnv, 486 Task: task, 487 Driver: "docker", 488 AllocID: d.DriverContext.allocID, 489 LogDir: ctx.TaskDir.LogDir, 490 TaskDir: ctx.TaskDir.Dir, 491 PortLowerBound: d.config.ClientMinPort, 492 PortUpperBound: d.config.ClientMaxPort, 493 } 494 if err := exec.SetContext(executorCtx); err != nil { 495 pluginClient.Kill() 496 return nil, fmt.Errorf("failed to set executor context: %v", err) 497 } 498 499 // Only launch syslog server if we're going to use it! 500 syslogAddr := "" 501 if runtime.GOOS == "darwin" && len(d.driverConfig.Logging) == 0 { 502 d.logger.Printf("[DEBUG] driver.docker: disabling syslog driver as Docker for Mac workaround") 503 } else if len(d.driverConfig.Logging) == 0 || d.driverConfig.Logging[0].Type == "syslog" { 504 ss, err := exec.LaunchSyslogServer() 505 if err != nil { 506 pluginClient.Kill() 507 return nil, fmt.Errorf("failed to start syslog collector: %v", err) 508 } 509 syslogAddr = ss.Addr 510 } 511 512 config, err := d.createContainerConfig(ctx, task, d.driverConfig, syslogAddr) 513 if err != nil { 514 d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err) 515 pluginClient.Kill() 516 return nil, fmt.Errorf("Failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err) 517 } 518 519 container, err := d.createContainer(config) 520 if err != nil { 521 wrapped := fmt.Sprintf("Failed to create container: %v", err) 522 d.logger.Printf("[ERR] driver.docker: %s", wrapped) 523 pluginClient.Kill() 524 return nil, structs.WrapRecoverable(wrapped, err) 525 } 526 527 d.logger.Printf("[INFO] driver.docker: created container %s", container.ID) 528 529 // We don't need to start the container if the container is already running 530 // since we don't create containers which are already present on the host 531 // and are running 532 if !container.State.Running { 533 // Start the container 534 if err := d.startContainer(container); err != nil { 535 d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err) 536 pluginClient.Kill() 537 return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err) 538 } 539 d.logger.Printf("[INFO] driver.docker: started container %s", container.ID) 540 } else { 541 d.logger.Printf("[DEBUG] driver.docker: re-attaching to container %s with status %q", 542 container.ID, container.State.String()) 543 } 544 545 // Return a driver handle 546 maxKill := d.DriverContext.config.MaxKillTimeout 547 h := &DockerHandle{ 548 client: client, 549 waitClient: waitClient, 550 executor: exec, 551 pluginClient: pluginClient, 552 logger: d.logger, 553 Image: d.driverConfig.ImageName, 554 ImageID: d.imageID, 555 containerID: container.ID, 556 version: d.config.Version, 557 killTimeout: GetKillTimeout(task.KillTimeout, maxKill), 558 maxKillTimeout: maxKill, 559 doneCh: make(chan bool), 560 waitCh: make(chan *dstructs.WaitResult, 1), 561 } 562 if err := exec.SyncServices(consulContext(d.config, container.ID)); err != nil { 563 d.logger.Printf("[ERR] driver.docker: error registering services with consul for task: %q: %v", task.Name, err) 564 } 565 go h.collectStats() 566 go h.run() 567 return h, nil 568 } 569 570 func (d *DockerDriver) Cleanup(_ *ExecContext, res *CreatedResources) error { 571 retry := false 572 var merr multierror.Error 573 for key, resources := range res.Resources { 574 switch key { 575 case dockerImageResKey: 576 for _, value := range resources { 577 err := d.cleanupImage(value) 578 if err != nil { 579 if structs.IsRecoverable(err) { 580 retry = true 581 } 582 merr.Errors = append(merr.Errors, err) 583 continue 584 } 585 586 // Remove cleaned image from resources 587 res.Remove(dockerImageResKey, value) 588 } 589 default: 590 d.logger.Printf("[ERR] driver.docker: unknown resource to cleanup: %q", key) 591 } 592 } 593 return structs.NewRecoverableError(merr.ErrorOrNil(), retry) 594 } 595 596 // cleanupImage removes a Docker image. No error is returned if the image 597 // doesn't exist or is still in use. Requires the global client to already be 598 // initialized. 599 func (d *DockerDriver) cleanupImage(imageID string) error { 600 if !d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault) { 601 // Config says not to cleanup 602 return nil 603 } 604 605 coordinator, callerID := d.getDockerCoordinator(client) 606 coordinator.RemoveImage(imageID, callerID) 607 608 return nil 609 } 610 611 // dockerClients creates two *docker.Client, one for long running operations and 612 // the other for shorter operations. In test / dev mode we can use ENV vars to 613 // connect to the docker daemon. In production mode we will read docker.endpoint 614 // from the config file. 615 func (d *DockerDriver) dockerClients() (*docker.Client, *docker.Client, error) { 616 if client != nil && waitClient != nil { 617 return client, waitClient, nil 618 } 619 620 var err error 621 var merr multierror.Error 622 createClients.Do(func() { 623 // Default to using whatever is configured in docker.endpoint. If this is 624 // not specified we'll fall back on NewClientFromEnv which reads config from 625 // the DOCKER_* environment variables DOCKER_HOST, DOCKER_TLS_VERIFY, and 626 // DOCKER_CERT_PATH. This allows us to lock down the config in production 627 // but also accept the standard ENV configs for dev and test. 628 dockerEndpoint := d.config.Read("docker.endpoint") 629 if dockerEndpoint != "" { 630 cert := d.config.Read("docker.tls.cert") 631 key := d.config.Read("docker.tls.key") 632 ca := d.config.Read("docker.tls.ca") 633 634 if cert+key+ca != "" { 635 d.logger.Printf("[DEBUG] driver.docker: using TLS client connection to %s", dockerEndpoint) 636 client, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca) 637 if err != nil { 638 merr.Errors = append(merr.Errors, err) 639 } 640 waitClient, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca) 641 if err != nil { 642 merr.Errors = append(merr.Errors, err) 643 } 644 } else { 645 d.logger.Printf("[DEBUG] driver.docker: using standard client connection to %s", dockerEndpoint) 646 client, err = docker.NewClient(dockerEndpoint) 647 if err != nil { 648 merr.Errors = append(merr.Errors, err) 649 } 650 waitClient, err = docker.NewClient(dockerEndpoint) 651 if err != nil { 652 merr.Errors = append(merr.Errors, err) 653 } 654 } 655 client.SetTimeout(dockerTimeout) 656 return 657 } 658 659 d.logger.Println("[DEBUG] driver.docker: using client connection initialized from environment") 660 client, err = docker.NewClientFromEnv() 661 if err != nil { 662 merr.Errors = append(merr.Errors, err) 663 } 664 client.SetTimeout(dockerTimeout) 665 666 waitClient, err = docker.NewClientFromEnv() 667 if err != nil { 668 merr.Errors = append(merr.Errors, err) 669 } 670 }) 671 return client, waitClient, merr.ErrorOrNil() 672 } 673 674 func (d *DockerDriver) containerBinds(driverConfig *DockerDriverConfig, taskDir *allocdir.TaskDir, 675 task *structs.Task) ([]string, error) { 676 677 allocDirBind := fmt.Sprintf("%s:%s", taskDir.SharedAllocDir, allocdir.SharedAllocContainerPath) 678 taskLocalBind := fmt.Sprintf("%s:%s", taskDir.LocalDir, allocdir.TaskLocalContainerPath) 679 secretDirBind := fmt.Sprintf("%s:%s", taskDir.SecretsDir, allocdir.TaskSecretsContainerPath) 680 binds := []string{allocDirBind, taskLocalBind, secretDirBind} 681 682 volumesEnabled := d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault) 683 684 if !volumesEnabled && driverConfig.VolumeDriver != "" { 685 return nil, fmt.Errorf("%s is false; cannot use volume driver %q", dockerVolumesConfigOption, driverConfig.VolumeDriver) 686 } 687 688 for _, userbind := range driverConfig.Volumes { 689 parts := strings.Split(userbind, ":") 690 if len(parts) < 2 { 691 return nil, fmt.Errorf("invalid docker volume: %q", userbind) 692 } 693 694 // Resolve dotted path segments 695 parts[0] = filepath.Clean(parts[0]) 696 697 // Absolute paths aren't always supported 698 if filepath.IsAbs(parts[0]) { 699 if !volumesEnabled { 700 // Disallow mounting arbitrary absolute paths 701 return nil, fmt.Errorf("%s is false; cannot mount host paths: %+q", dockerVolumesConfigOption, userbind) 702 } 703 binds = append(binds, userbind) 704 continue 705 } 706 707 // Relative paths are always allowed as they mount within a container 708 // When a VolumeDriver is set, we assume we receive a binding in the format volume-name:container-dest 709 // Otherwise, we assume we receive a relative path binding in the format relative/to/task:/also/in/container 710 if driverConfig.VolumeDriver == "" { 711 // Expand path relative to alloc dir 712 parts[0] = filepath.Join(taskDir.Dir, parts[0]) 713 } 714 715 binds = append(binds, strings.Join(parts, ":")) 716 } 717 718 if selinuxLabel := d.config.Read(dockerSELinuxLabelConfigOption); selinuxLabel != "" { 719 // Apply SELinux Label to each volume 720 for i := range binds { 721 binds[i] = fmt.Sprintf("%s:%s", binds[i], selinuxLabel) 722 } 723 } 724 725 return binds, nil 726 } 727 728 // createContainerConfig initializes a struct needed to call docker.client.CreateContainer() 729 func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Task, 730 driverConfig *DockerDriverConfig, syslogAddr string) (docker.CreateContainerOptions, error) { 731 var c docker.CreateContainerOptions 732 if task.Resources == nil { 733 // Guard against missing resources. We should never have been able to 734 // schedule a job without specifying this. 735 d.logger.Println("[ERR] driver.docker: task.Resources is empty") 736 return c, fmt.Errorf("task.Resources is empty") 737 } 738 739 binds, err := d.containerBinds(driverConfig, ctx.TaskDir, task) 740 if err != nil { 741 return c, err 742 } 743 744 config := &docker.Config{ 745 Image: d.imageID, 746 Hostname: driverConfig.Hostname, 747 User: task.User, 748 Tty: driverConfig.TTY, 749 OpenStdin: driverConfig.Interactive, 750 } 751 752 if driverConfig.WorkDir != "" { 753 config.WorkingDir = driverConfig.WorkDir 754 } 755 756 memLimit := int64(task.Resources.MemoryMB) * 1024 * 1024 757 758 if len(driverConfig.Logging) == 0 { 759 if runtime.GOOS != "darwin" { 760 d.logger.Printf("[DEBUG] driver.docker: Setting default logging options to syslog and %s", syslogAddr) 761 driverConfig.Logging = []DockerLoggingOpts{ 762 {Type: "syslog", Config: map[string]string{"syslog-address": syslogAddr}}, 763 } 764 } else { 765 d.logger.Printf("[DEBUG] driver.docker: deferring logging to docker on Docker for Mac") 766 } 767 } 768 769 hostConfig := &docker.HostConfig{ 770 // Convert MB to bytes. This is an absolute value. 771 Memory: memLimit, 772 // Convert Mhz to shares. This is a relative value. 773 CPUShares: int64(task.Resources.CPU), 774 775 // Binds are used to mount a host volume into the container. We mount a 776 // local directory for storage and a shared alloc directory that can be 777 // used to share data between different tasks in the same task group. 778 Binds: binds, 779 780 VolumeDriver: driverConfig.VolumeDriver, 781 } 782 783 // Windows does not support MemorySwap #2193 784 if runtime.GOOS != "windows" { 785 hostConfig.MemorySwap = memLimit // MemorySwap is memory + swap. 786 } 787 788 if len(driverConfig.Logging) != 0 { 789 d.logger.Printf("[DEBUG] driver.docker: Using config for logging: %+v", driverConfig.Logging[0]) 790 hostConfig.LogConfig = docker.LogConfig{ 791 Type: driverConfig.Logging[0].Type, 792 Config: driverConfig.Logging[0].Config, 793 } 794 } 795 796 d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name) 797 d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name) 798 d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name) 799 800 // set privileged mode 801 hostPrivileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false) 802 if driverConfig.Privileged && !hostPrivileged { 803 return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`) 804 } 805 hostConfig.Privileged = driverConfig.Privileged 806 807 // set SHM size 808 if driverConfig.ShmSize != 0 { 809 hostConfig.ShmSize = driverConfig.ShmSize 810 } 811 812 // set DNS servers 813 for _, ip := range driverConfig.DNSServers { 814 if net.ParseIP(ip) != nil { 815 hostConfig.DNS = append(hostConfig.DNS, ip) 816 } else { 817 d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip) 818 } 819 } 820 821 // set DNS search domains 822 for _, domain := range driverConfig.DNSSearchDomains { 823 hostConfig.DNSSearch = append(hostConfig.DNSSearch, domain) 824 } 825 826 hostConfig.IpcMode = driverConfig.IpcMode 827 hostConfig.PidMode = driverConfig.PidMode 828 hostConfig.UTSMode = driverConfig.UTSMode 829 hostConfig.UsernsMode = driverConfig.UsernsMode 830 831 hostConfig.NetworkMode = driverConfig.NetworkMode 832 if hostConfig.NetworkMode == "" { 833 // docker default 834 d.logger.Printf("[DEBUG] driver.docker: networking mode not specified; defaulting to %s", defaultNetworkMode) 835 hostConfig.NetworkMode = defaultNetworkMode 836 } 837 838 // Setup port mapping and exposed ports 839 if len(task.Resources.Networks) == 0 { 840 d.logger.Println("[DEBUG] driver.docker: No network interfaces are available") 841 if len(driverConfig.PortMap) > 0 { 842 return c, fmt.Errorf("Trying to map ports but no network interface is available") 843 } 844 } else { 845 // TODO add support for more than one network 846 network := task.Resources.Networks[0] 847 publishedPorts := map[docker.Port][]docker.PortBinding{} 848 exposedPorts := map[docker.Port]struct{}{} 849 850 for _, port := range network.ReservedPorts { 851 // By default we will map the allocated port 1:1 to the container 852 containerPortInt := port.Value 853 854 // If the user has mapped a port using port_map we'll change it here 855 if mapped, ok := driverConfig.PortMap[port.Label]; ok { 856 containerPortInt = mapped 857 } 858 859 hostPortStr := strconv.Itoa(port.Value) 860 containerPort := docker.Port(strconv.Itoa(containerPortInt)) 861 862 publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr) 863 publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr) 864 d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value) 865 866 exposedPorts[containerPort+"/tcp"] = struct{}{} 867 exposedPorts[containerPort+"/udp"] = struct{}{} 868 d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value) 869 } 870 871 for _, port := range network.DynamicPorts { 872 // By default we will map the allocated port 1:1 to the container 873 containerPortInt := port.Value 874 875 // If the user has mapped a port using port_map we'll change it here 876 if mapped, ok := driverConfig.PortMap[port.Label]; ok { 877 containerPortInt = mapped 878 } 879 880 hostPortStr := strconv.Itoa(port.Value) 881 containerPort := docker.Port(strconv.Itoa(containerPortInt)) 882 883 publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr) 884 publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr) 885 d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt) 886 887 exposedPorts[containerPort+"/tcp"] = struct{}{} 888 exposedPorts[containerPort+"/udp"] = struct{}{} 889 d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort) 890 } 891 892 d.taskEnv.SetPortMap(driverConfig.PortMap) 893 894 hostConfig.PortBindings = publishedPorts 895 config.ExposedPorts = exposedPorts 896 } 897 898 d.taskEnv.Build() 899 parsedArgs := d.taskEnv.ParseAndReplace(driverConfig.Args) 900 901 // If the user specified a custom command to run, we'll inject it here. 902 if driverConfig.Command != "" { 903 // Validate command 904 if err := validateCommand(driverConfig.Command, "args"); err != nil { 905 return c, err 906 } 907 908 cmd := []string{driverConfig.Command} 909 if len(driverConfig.Args) != 0 { 910 cmd = append(cmd, parsedArgs...) 911 } 912 d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " ")) 913 config.Cmd = cmd 914 } else if len(driverConfig.Args) != 0 { 915 config.Cmd = parsedArgs 916 } 917 918 if len(driverConfig.Labels) > 0 { 919 config.Labels = driverConfig.Labels 920 d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels) 921 } 922 923 config.Env = d.taskEnv.EnvList() 924 925 containerName := fmt.Sprintf("%s-%s", task.Name, d.DriverContext.allocID) 926 d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName) 927 928 var networkingConfig *docker.NetworkingConfig 929 if len(driverConfig.NetworkAliases) > 0 || driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" { 930 networkingConfig = &docker.NetworkingConfig{ 931 EndpointsConfig: map[string]*docker.EndpointConfig{ 932 hostConfig.NetworkMode: &docker.EndpointConfig{}, 933 }, 934 } 935 } 936 937 if len(driverConfig.NetworkAliases) > 0 { 938 networkingConfig.EndpointsConfig[hostConfig.NetworkMode].Aliases = driverConfig.NetworkAliases 939 d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with network aliases: %v", 940 hostConfig.NetworkMode, strings.Join(driverConfig.NetworkAliases, ", ")) 941 } 942 943 if driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" { 944 networkingConfig.EndpointsConfig[hostConfig.NetworkMode].IPAMConfig = &docker.EndpointIPAMConfig{ 945 IPv4Address: driverConfig.IPv4Address, 946 IPv6Address: driverConfig.IPv6Address, 947 } 948 d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with ipv4: %q and ipv6: %q", 949 hostConfig.NetworkMode, driverConfig.IPv4Address, driverConfig.IPv6Address) 950 } 951 952 return docker.CreateContainerOptions{ 953 Name: containerName, 954 Config: config, 955 HostConfig: hostConfig, 956 NetworkingConfig: networkingConfig, 957 }, nil 958 } 959 960 func (d *DockerDriver) Periodic() (bool, time.Duration) { 961 return true, 15 * time.Second 962 } 963 964 // createImage creates a docker image either by pulling it from a registry or by 965 // loading it from the file system 966 func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir *allocdir.TaskDir) (string, error) { 967 image := driverConfig.ImageName 968 repo, tag := docker.ParseRepositoryTag(image) 969 if tag == "" { 970 tag = "latest" 971 } 972 973 coordinator, callerID := d.getDockerCoordinator(client) 974 975 // We're going to check whether the image is already downloaded. If the tag 976 // is "latest", or ForcePull is set, we have to check for a new version every time so we don't 977 // bother to check and cache the id here. We'll download first, then cache. 978 if driverConfig.ForcePull { 979 d.logger.Printf("[DEBUG] driver.docker: force pull image '%s:%s' instead of inspecting local", repo, tag) 980 } else if tag != "latest" { 981 if dockerImage, _ := client.InspectImage(image); dockerImage != nil { 982 // Image exists so just increment its reference count 983 coordinator.IncrementImageReference(dockerImage.ID, image, callerID) 984 return dockerImage.ID, nil 985 } 986 } 987 988 // Load the image if specified 989 if driverConfig.LoadImage != "" { 990 return d.loadImage(driverConfig, client, taskDir) 991 } 992 993 // Download the image 994 return d.pullImage(driverConfig, client, repo, tag) 995 } 996 997 // pullImage creates an image by pulling it from a docker registry 998 func (d *DockerDriver) pullImage(driverConfig *DockerDriverConfig, client *docker.Client, repo, tag string) (id string, err error) { 999 var authOptions *docker.AuthConfiguration 1000 if len(driverConfig.Auth) != 0 { 1001 authOptions = &docker.AuthConfiguration{ 1002 Username: driverConfig.Auth[0].Username, 1003 Password: driverConfig.Auth[0].Password, 1004 Email: driverConfig.Auth[0].Email, 1005 ServerAddress: driverConfig.Auth[0].ServerAddress, 1006 } 1007 } else if authConfigFile := d.config.Read("docker.auth.config"); authConfigFile != "" { 1008 var err error 1009 authOptions, err = authOptionFrom(authConfigFile, repo) 1010 if err != nil { 1011 d.logger.Printf("[INFO] driver.docker: failed to find docker auth for repo %q: %v", repo, err) 1012 return "", fmt.Errorf("Failed to find docker auth for repo %q: %v", repo, err) 1013 } 1014 1015 if authOptions.Email == "" && authOptions.Password == "" && 1016 authOptions.ServerAddress == "" && authOptions.Username == "" { 1017 d.logger.Printf("[DEBUG] driver.docker: did not find docker auth for repo %q", repo) 1018 } 1019 } 1020 1021 d.emitEvent("Downloading image %s:%s", repo, tag) 1022 coordinator, callerID := d.getDockerCoordinator(client) 1023 return coordinator.PullImage(driverConfig.ImageName, authOptions, callerID) 1024 } 1025 1026 // loadImage creates an image by loading it from the file system 1027 func (d *DockerDriver) loadImage(driverConfig *DockerDriverConfig, client *docker.Client, 1028 taskDir *allocdir.TaskDir) (id string, err error) { 1029 1030 archive := filepath.Join(taskDir.LocalDir, driverConfig.LoadImage) 1031 d.logger.Printf("[DEBUG] driver.docker: loading image from: %v", archive) 1032 1033 f, err := os.Open(archive) 1034 if err != nil { 1035 return "", fmt.Errorf("unable to open image archive: %v", err) 1036 } 1037 1038 if err := client.LoadImage(docker.LoadImageOptions{InputStream: f}); err != nil { 1039 return "", err 1040 } 1041 f.Close() 1042 1043 dockerImage, err := client.InspectImage(driverConfig.ImageName) 1044 if err != nil { 1045 return "", recoverableErrTimeouts(err) 1046 } 1047 1048 coordinator, callerID := d.getDockerCoordinator(client) 1049 coordinator.IncrementImageReference(dockerImage.ID, driverConfig.ImageName, callerID) 1050 return dockerImage.ID, nil 1051 } 1052 1053 // createContainer creates the container given the passed configuration. It 1054 // attempts to handle any transient Docker errors. 1055 func (d *DockerDriver) createContainer(config docker.CreateContainerOptions) (*docker.Container, error) { 1056 // Create a container 1057 attempted := 0 1058 CREATE: 1059 container, createErr := client.CreateContainer(config) 1060 if createErr == nil { 1061 return container, nil 1062 } 1063 1064 d.logger.Printf("[DEBUG] driver.docker: failed to create container %q from image %q (ID: %q) (attempt %d): %v", 1065 config.Name, d.driverConfig.ImageName, d.imageID, attempted+1, createErr) 1066 if strings.Contains(strings.ToLower(createErr.Error()), "container already exists") { 1067 containers, err := client.ListContainers(docker.ListContainersOptions{ 1068 All: true, 1069 }) 1070 if err != nil { 1071 d.logger.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name) 1072 return nil, recoverableErrTimeouts(fmt.Errorf("Failed to query list of containers: %s", err)) 1073 } 1074 1075 // Delete matching containers 1076 // Adding a / infront of the container name since Docker returns the 1077 // container names with a / pre-pended to the Nomad generated container names 1078 containerName := "/" + config.Name 1079 d.logger.Printf("[DEBUG] driver.docker: searching for container name %q to purge", containerName) 1080 for _, container := range containers { 1081 d.logger.Printf("[DEBUG] driver.docker: listed container %+v", container) 1082 found := false 1083 for _, name := range container.Names { 1084 if name == containerName { 1085 found = true 1086 break 1087 } 1088 } 1089 1090 if !found { 1091 continue 1092 } 1093 1094 // Inspect the container and if the container isn't dead then return 1095 // the container 1096 container, err := client.InspectContainer(container.ID) 1097 if err != nil { 1098 return nil, recoverableErrTimeouts(fmt.Errorf("Failed to inspect container %s: %s", container.ID, err)) 1099 } 1100 if container != nil && (container.State.Running || container.State.FinishedAt.IsZero()) { 1101 return container, nil 1102 } 1103 1104 err = client.RemoveContainer(docker.RemoveContainerOptions{ 1105 ID: container.ID, 1106 Force: true, 1107 }) 1108 if err != nil { 1109 d.logger.Printf("[ERR] driver.docker: failed to purge container %s", container.ID) 1110 return nil, recoverableErrTimeouts(fmt.Errorf("Failed to purge container %s: %s", container.ID, err)) 1111 } else if err == nil { 1112 d.logger.Printf("[INFO] driver.docker: purged container %s", container.ID) 1113 } 1114 } 1115 1116 if attempted < 5 { 1117 attempted++ 1118 time.Sleep(1 * time.Second) 1119 goto CREATE 1120 } 1121 } else if strings.Contains(strings.ToLower(createErr.Error()), "no such image") { 1122 // There is still a very small chance this is possible even with the 1123 // coordinator so retry. 1124 return nil, structs.NewRecoverableError(createErr, true) 1125 } 1126 1127 return nil, recoverableErrTimeouts(createErr) 1128 } 1129 1130 // startContainer starts the passed container. It attempts to handle any 1131 // transient Docker errors. 1132 func (d *DockerDriver) startContainer(c *docker.Container) error { 1133 // Start a container 1134 attempted := 0 1135 START: 1136 startErr := client.StartContainer(c.ID, c.HostConfig) 1137 if startErr == nil { 1138 return nil 1139 } 1140 1141 d.logger.Printf("[DEBUG] driver.docker: failed to start container %q (attempt %d): %v", c.ID, attempted+1, startErr) 1142 1143 // If it is a 500 error it is likely we can retry and be successful 1144 if strings.Contains(startErr.Error(), "API error (500)") { 1145 if attempted < 5 { 1146 attempted++ 1147 time.Sleep(1 * time.Second) 1148 goto START 1149 } 1150 } 1151 1152 return recoverableErrTimeouts(startErr) 1153 } 1154 1155 func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { 1156 // Split the handle 1157 pidBytes := []byte(strings.TrimPrefix(handleID, "DOCKER:")) 1158 pid := &dockerPID{} 1159 if err := json.Unmarshal(pidBytes, pid); err != nil { 1160 return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) 1161 } 1162 d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", pid.ContainerID) 1163 d.logger.Printf("[DEBUG] driver.docker: re-attached to handle: %s", handleID) 1164 pluginConfig := &plugin.ClientConfig{ 1165 Reattach: pid.PluginConfig.PluginConfig(), 1166 } 1167 1168 client, waitClient, err := d.dockerClients() 1169 if err != nil { 1170 return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) 1171 } 1172 1173 // Look for a running container with this ID 1174 containers, err := client.ListContainers(docker.ListContainersOptions{ 1175 Filters: map[string][]string{ 1176 "id": []string{pid.ContainerID}, 1177 }, 1178 }) 1179 if err != nil { 1180 return nil, fmt.Errorf("Failed to query for container %s: %v", pid.ContainerID, err) 1181 } 1182 1183 found := false 1184 for _, container := range containers { 1185 if container.ID == pid.ContainerID { 1186 found = true 1187 } 1188 } 1189 if !found { 1190 return nil, fmt.Errorf("Failed to find container %s", pid.ContainerID) 1191 } 1192 exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) 1193 if err != nil { 1194 d.logger.Printf("[INFO] driver.docker: couldn't re-attach to the plugin process: %v", err) 1195 d.logger.Printf("[DEBUG] driver.docker: stopping container %q", pid.ContainerID) 1196 if e := client.StopContainer(pid.ContainerID, uint(pid.KillTimeout.Seconds())); e != nil { 1197 d.logger.Printf("[DEBUG] driver.docker: couldn't stop container: %v", e) 1198 } 1199 return nil, err 1200 } 1201 1202 ver, _ := exec.Version() 1203 d.logger.Printf("[DEBUG] driver.docker: version of executor: %v", ver.Version) 1204 1205 // Increment the reference count since we successfully attached to this 1206 // container 1207 coordinator, callerID := d.getDockerCoordinator(client) 1208 coordinator.IncrementImageReference(pid.ImageID, pid.Image, callerID) 1209 1210 // Return a driver handle 1211 h := &DockerHandle{ 1212 client: client, 1213 waitClient: waitClient, 1214 executor: exec, 1215 pluginClient: pluginClient, 1216 logger: d.logger, 1217 Image: pid.Image, 1218 ImageID: pid.ImageID, 1219 containerID: pid.ContainerID, 1220 version: pid.Version, 1221 killTimeout: pid.KillTimeout, 1222 maxKillTimeout: pid.MaxKillTimeout, 1223 doneCh: make(chan bool), 1224 waitCh: make(chan *dstructs.WaitResult, 1), 1225 } 1226 if err := exec.SyncServices(consulContext(d.config, pid.ContainerID)); err != nil { 1227 h.logger.Printf("[ERR] driver.docker: error registering services with consul: %v", err) 1228 } 1229 1230 go h.collectStats() 1231 go h.run() 1232 return h, nil 1233 } 1234 1235 func (h *DockerHandle) ID() string { 1236 // Return a handle to the PID 1237 pid := dockerPID{ 1238 Version: h.version, 1239 ContainerID: h.containerID, 1240 Image: h.Image, 1241 ImageID: h.ImageID, 1242 KillTimeout: h.killTimeout, 1243 MaxKillTimeout: h.maxKillTimeout, 1244 PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), 1245 } 1246 data, err := json.Marshal(pid) 1247 if err != nil { 1248 h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s", err) 1249 } 1250 return fmt.Sprintf("DOCKER:%s", string(data)) 1251 } 1252 1253 func (h *DockerHandle) ContainerID() string { 1254 return h.containerID 1255 } 1256 1257 func (h *DockerHandle) WaitCh() chan *dstructs.WaitResult { 1258 return h.waitCh 1259 } 1260 1261 func (h *DockerHandle) Update(task *structs.Task) error { 1262 // Store the updated kill timeout. 1263 h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout) 1264 if err := h.executor.UpdateTask(task); err != nil { 1265 h.logger.Printf("[DEBUG] driver.docker: failed to update log config: %v", err) 1266 } 1267 1268 // Update is not possible 1269 return nil 1270 } 1271 1272 func (h *DockerHandle) Signal(s os.Signal) error { 1273 // Convert types 1274 sysSig, ok := s.(syscall.Signal) 1275 if !ok { 1276 return fmt.Errorf("Failed to determine signal number") 1277 } 1278 1279 dockerSignal := docker.Signal(sysSig) 1280 opts := docker.KillContainerOptions{ 1281 ID: h.containerID, 1282 Signal: dockerSignal, 1283 } 1284 return h.client.KillContainer(opts) 1285 1286 } 1287 1288 // Kill is used to terminate the task. This uses `docker stop -t killTimeout` 1289 func (h *DockerHandle) Kill() error { 1290 // Stop the container 1291 err := h.client.StopContainer(h.containerID, uint(h.killTimeout.Seconds())) 1292 if err != nil { 1293 h.executor.Exit() 1294 h.pluginClient.Kill() 1295 1296 // Container has already been removed. 1297 if strings.Contains(err.Error(), NoSuchContainerError) { 1298 h.logger.Printf("[DEBUG] driver.docker: attempted to stop non-existent container %s", h.containerID) 1299 return nil 1300 } 1301 h.logger.Printf("[ERR] driver.docker: failed to stop container %s: %v", h.containerID, err) 1302 return fmt.Errorf("Failed to stop container %s: %s", h.containerID, err) 1303 } 1304 h.logger.Printf("[INFO] driver.docker: stopped container %s", h.containerID) 1305 return nil 1306 } 1307 1308 func (h *DockerHandle) Stats() (*cstructs.TaskResourceUsage, error) { 1309 h.resourceUsageLock.RLock() 1310 defer h.resourceUsageLock.RUnlock() 1311 var err error 1312 if h.resourceUsage == nil { 1313 err = fmt.Errorf("stats collection hasn't started yet") 1314 } 1315 return h.resourceUsage, err 1316 } 1317 1318 func (h *DockerHandle) run() { 1319 // Wait for it... 1320 exitCode, werr := h.waitClient.WaitContainer(h.containerID) 1321 if werr != nil { 1322 h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated", h.containerID) 1323 } 1324 1325 if exitCode != 0 { 1326 werr = fmt.Errorf("Docker container exited with non-zero exit code: %d", exitCode) 1327 } 1328 1329 close(h.doneCh) 1330 1331 // Remove services 1332 if err := h.executor.DeregisterServices(); err != nil { 1333 h.logger.Printf("[ERR] driver.docker: error deregistering services: %v", err) 1334 } 1335 1336 // Shutdown the syslog collector 1337 if err := h.executor.Exit(); err != nil { 1338 h.logger.Printf("[ERR] driver.docker: failed to kill the syslog collector: %v", err) 1339 } 1340 h.pluginClient.Kill() 1341 1342 // Stop the container just incase the docker daemon's wait returned 1343 // incorrectly 1344 if err := h.client.StopContainer(h.containerID, 0); err != nil { 1345 _, noSuchContainer := err.(*docker.NoSuchContainer) 1346 _, containerNotRunning := err.(*docker.ContainerNotRunning) 1347 if !containerNotRunning && !noSuchContainer { 1348 h.logger.Printf("[ERR] driver.docker: error stopping container: %v", err) 1349 } 1350 } 1351 1352 // Remove the container 1353 if err := h.client.RemoveContainer(docker.RemoveContainerOptions{ID: h.containerID, RemoveVolumes: true, Force: true}); err != nil { 1354 h.logger.Printf("[ERR] driver.docker: error removing container: %v", err) 1355 } 1356 1357 // Send the results 1358 h.waitCh <- dstructs.NewWaitResult(exitCode, 0, werr) 1359 close(h.waitCh) 1360 } 1361 1362 // collectStats starts collecting resource usage stats of a docker container 1363 func (h *DockerHandle) collectStats() { 1364 statsCh := make(chan *docker.Stats) 1365 statsOpts := docker.StatsOptions{ID: h.containerID, Done: h.doneCh, Stats: statsCh, Stream: true} 1366 go func() { 1367 //TODO handle Stats error 1368 if err := h.waitClient.Stats(statsOpts); err != nil { 1369 h.logger.Printf("[DEBUG] driver.docker: error collecting stats from container %s: %v", h.containerID, err) 1370 } 1371 }() 1372 numCores := runtime.NumCPU() 1373 for { 1374 select { 1375 case s := <-statsCh: 1376 if s != nil { 1377 ms := &cstructs.MemoryStats{ 1378 RSS: s.MemoryStats.Stats.Rss, 1379 Cache: s.MemoryStats.Stats.Cache, 1380 Swap: s.MemoryStats.Stats.Swap, 1381 MaxUsage: s.MemoryStats.MaxUsage, 1382 Measured: DockerMeasuredMemStats, 1383 } 1384 1385 cs := &cstructs.CpuStats{ 1386 ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods, 1387 ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime, 1388 Measured: DockerMeasuredCpuStats, 1389 } 1390 1391 // Calculate percentage 1392 cores := len(s.CPUStats.CPUUsage.PercpuUsage) 1393 cs.Percent = calculatePercent( 1394 s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, 1395 s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, cores) 1396 cs.SystemMode = calculatePercent( 1397 s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode, 1398 s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores) 1399 cs.UserMode = calculatePercent( 1400 s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode, 1401 s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores) 1402 cs.TotalTicks = (cs.Percent / 100) * shelpers.TotalTicksAvailable() / float64(numCores) 1403 1404 h.resourceUsageLock.Lock() 1405 h.resourceUsage = &cstructs.TaskResourceUsage{ 1406 ResourceUsage: &cstructs.ResourceUsage{ 1407 MemoryStats: ms, 1408 CpuStats: cs, 1409 }, 1410 Timestamp: s.Read.UTC().UnixNano(), 1411 } 1412 h.resourceUsageLock.Unlock() 1413 } 1414 case <-h.doneCh: 1415 return 1416 } 1417 } 1418 } 1419 1420 func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 { 1421 numerator := newSample - oldSample 1422 denom := newTotal - oldTotal 1423 if numerator <= 0 || denom <= 0 { 1424 return 0.0 1425 } 1426 1427 return (float64(numerator) / float64(denom)) * float64(cores) * 100.0 1428 } 1429 1430 // authOptionFrom takes the Docker auth config file and the repo being pulled 1431 // and returns an AuthConfiguration or an error if the file/repo could not be 1432 // parsed or looked up. 1433 func authOptionFrom(file, repo string) (*docker.AuthConfiguration, error) { 1434 name, err := reference.ParseNamed(repo) 1435 if err != nil { 1436 return nil, fmt.Errorf("Failed to parse named repo %q: %v", repo, err) 1437 } 1438 1439 repoInfo, err := registry.ParseRepositoryInfo(name) 1440 if err != nil { 1441 return nil, fmt.Errorf("Failed to parse repository: %v", err) 1442 } 1443 1444 f, err := os.Open(file) 1445 if err != nil { 1446 return nil, fmt.Errorf("Failed to open auth config file: %v, error: %v", file, err) 1447 } 1448 defer f.Close() 1449 1450 cfile := new(configfile.ConfigFile) 1451 if err := cfile.LoadFromReader(f); err != nil { 1452 return nil, fmt.Errorf("Failed to parse auth config file: %v", err) 1453 } 1454 1455 dockerAuthConfig := registry.ResolveAuthConfig(cfile.AuthConfigs, repoInfo.Index) 1456 1457 // Convert to Api version 1458 apiAuthConfig := &docker.AuthConfiguration{ 1459 Username: dockerAuthConfig.Username, 1460 Password: dockerAuthConfig.Password, 1461 Email: dockerAuthConfig.Email, 1462 ServerAddress: dockerAuthConfig.ServerAddress, 1463 } 1464 1465 return apiAuthConfig, nil 1466 }