github.com/anuvu/nomad@v0.8.7-atom1/client/driver/docker.go (about) 1 package driver 2 3 import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "log" 8 "net" 9 "os" 10 "os/exec" 11 "path/filepath" 12 "runtime" 13 "strconv" 14 "strings" 15 "sync" 16 "syscall" 17 "time" 18 19 "github.com/armon/circbuf" 20 docker "github.com/fsouza/go-dockerclient" 21 22 "github.com/docker/docker/cli/config/configfile" 23 "github.com/docker/docker/reference" 24 "github.com/docker/docker/registry" 25 26 metrics "github.com/armon/go-metrics" 27 multierror "github.com/hashicorp/go-multierror" 28 plugin "github.com/hashicorp/go-plugin" 29 "github.com/hashicorp/nomad/client/allocdir" 30 "github.com/hashicorp/nomad/client/driver/env" 31 "github.com/hashicorp/nomad/client/driver/executor" 32 dstructs "github.com/hashicorp/nomad/client/driver/structs" 33 cstructs "github.com/hashicorp/nomad/client/structs" 34 "github.com/hashicorp/nomad/helper" 35 "github.com/hashicorp/nomad/helper/fields" 36 shelpers "github.com/hashicorp/nomad/helper/stats" 37 "github.com/hashicorp/nomad/nomad/structs" 38 "github.com/mitchellh/mapstructure" 39 ) 40 41 var ( 42 // createClientsLock is a lock that protects reading/writing global client 43 // variables 44 createClientsLock sync.Mutex 45 46 // client is a docker client with a timeout of 5 minutes. This is for doing 47 // all operations with the docker daemon besides which are not long running 48 // such as creating, killing containers, etc. 49 client *docker.Client 50 51 // waitClient is a docker client with no timeouts. This is used for long 52 // running operations such as waiting on containers and collect stats 53 waitClient *docker.Client 54 55 // healthCheckClient is a docker client with a timeout of 1 minute. This is 56 // necessary to have a shorter timeout than other API or fingerprint calls 57 healthCheckClient *docker.Client 58 59 // The statistics the Docker driver exposes 60 DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"} 61 DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"} 62 63 // recoverableErrTimeouts returns a recoverable error if the error was due 64 // to timeouts 65 recoverableErrTimeouts = func(err error) error { 66 r := false 67 if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") || 68 strings.Contains(err.Error(), "EOF") { 69 r = true 70 } 71 return structs.NewRecoverableError(err, r) 72 } 73 ) 74 75 const ( 76 // NoSuchContainerError is returned by the docker daemon if the container 77 // does not exist. 78 NoSuchContainerError = "No such container" 79 80 // The key populated in Node Attributes to indicate presence of the Docker 81 // driver 82 dockerDriverAttr = "driver.docker" 83 84 // dockerSELinuxLabelConfigOption is the key for configuring the 85 // SELinux label for binds. 86 dockerSELinuxLabelConfigOption = "docker.volumes.selinuxlabel" 87 88 // dockerVolumesConfigOption is the key for enabling the use of custom 89 // bind volumes to arbitrary host paths. 90 dockerVolumesConfigOption = "docker.volumes.enabled" 91 dockerVolumesConfigDefault = true 92 93 // dockerPrivilegedConfigOption is the key for running containers in 94 // Docker's privileged mode. 95 dockerPrivilegedConfigOption = "docker.privileged.enabled" 96 97 // dockerCleanupImageConfigOption is the key for whether or not to 98 // cleanup images after the task exits. 99 dockerCleanupImageConfigOption = "docker.cleanup.image" 100 dockerCleanupImageConfigDefault = true 101 102 // dockerPullTimeoutConfigOption is the key for setting an images pull 103 // timeout 104 dockerImageRemoveDelayConfigOption = "docker.cleanup.image.delay" 105 dockerImageRemoveDelayConfigDefault = 3 * time.Minute 106 107 // dockerCapsWhitelistConfigOption is the key for setting the list of 108 // allowed Linux capabilities 109 dockerCapsWhitelistConfigOption = "docker.caps.whitelist" 110 dockerCapsWhitelistConfigDefault = dockerBasicCaps 111 112 // dockerTimeout is the length of time a request can be outstanding before 113 // it is timed out. 114 dockerTimeout = 5 * time.Minute 115 116 // dockerHealthCheckTimeout is the length of time a request for a health 117 // check client can be outstanding before it is timed out. 118 dockerHealthCheckTimeout = 1 * time.Minute 119 120 // dockerImageResKey is the CreatedResources key for docker images 121 dockerImageResKey = "image" 122 123 // dockerAuthHelperPrefix is the prefix to attach to the credential helper 124 // and should be found in the $PATH. Example: ${prefix-}${helper-name} 125 dockerAuthHelperPrefix = "docker-credential-" 126 127 // dockerBasicCaps is comma-separated list of Linux capabilities that are 128 // allowed by docker by default, as documented in 129 // https://docs.docker.com/engine/reference/run/#block-io-bandwidth-blkio-constraint 130 dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," + 131 "SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE" 132 133 // This is cpu.cfs_period_us: the length of a period. 134 // The default values is 100 milliseconds (ms) represented in microseconds (us). 135 // Below is the documentation: 136 // https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt 137 // https://docs.docker.com/engine/api/v1.35/# 138 defaultCFSPeriodUS = 100000 139 140 // dockerCleanupContainerConfigOption is the key for whether or not to 141 // remove containers after the task exits. 142 dockerCleanupContainerConfigOption = "docker.cleanup.container" 143 dockerCleanupContainerConfigDefault = true 144 ) 145 146 type DockerDriver struct { 147 DriverContext 148 149 driverConfig *DockerDriverConfig 150 imageID string 151 152 // A tri-state boolean to know if the fingerprinting has happened and 153 // whether it has been successful 154 fingerprintSuccess *bool 155 } 156 157 type DockerDriverAuth struct { 158 Username string `mapstructure:"username"` // username for the registry 159 Password string `mapstructure:"password"` // password to access the registry 160 Email string `mapstructure:"email"` // email address of the user who is allowed to access the registry 161 ServerAddress string `mapstructure:"server_address"` // server address of the registry 162 } 163 164 type DockerLoggingOpts struct { 165 Type string `mapstructure:"type"` 166 ConfigRaw []map[string]string `mapstructure:"config"` 167 Config map[string]string `mapstructure:"-"` 168 } 169 170 type DockerMount struct { 171 Type string `mapstructure:"type"` 172 Target string `mapstructure:"target"` 173 Source string `mapstructure:"source"` 174 ReadOnly bool `mapstructure:"readonly"` 175 BindOptions []*DockerBindOptions `mapstructure:"bind_options"` 176 VolumeOptions []*DockerVolumeOptions `mapstructure:"volume_options"` 177 } 178 179 type DockerDevice struct { 180 HostPath string `mapstructure:"host_path"` 181 ContainerPath string `mapstructure:"container_path"` 182 CgroupPermissions string `mapstructure:"cgroup_permissions"` 183 } 184 185 type DockerBindOptions struct { 186 Propagation string `mapstructure:"propagation"` 187 } 188 189 type DockerVolumeOptions struct { 190 NoCopy bool `mapstructure:"no_copy"` 191 Labels []map[string]string `mapstructure:"labels"` 192 DriverConfig []DockerVolumeDriverConfig `mapstructure:"driver_config"` 193 } 194 195 // VolumeDriverConfig holds a map of volume driver specific options 196 type DockerVolumeDriverConfig struct { 197 Name string `mapstructure:"name"` 198 Options []map[string]string `mapstructure:"options"` 199 } 200 201 // DockerDriverConfig defines the user specified config block in a jobspec 202 type DockerDriverConfig struct { 203 ImageName string `mapstructure:"image"` // Container's Image Name 204 LoadImage string `mapstructure:"load"` // LoadImage is a path to an image archive file 205 Command string `mapstructure:"command"` // The Command to run when the container starts up 206 Args []string `mapstructure:"args"` // The arguments to the Command 207 Entrypoint []string `mapstructure:"entrypoint"` // Override the containers entrypoint 208 IpcMode string `mapstructure:"ipc_mode"` // The IPC mode of the container - host and none 209 NetworkMode string `mapstructure:"network_mode"` // The network mode of the container - host, nat and none 210 NetworkAliases []string `mapstructure:"network_aliases"` // The network-scoped alias for the container 211 IPv4Address string `mapstructure:"ipv4_address"` // The container ipv4 address 212 IPv6Address string `mapstructure:"ipv6_address"` // the container ipv6 address 213 PidMode string `mapstructure:"pid_mode"` // The PID mode of the container - host and none 214 UTSMode string `mapstructure:"uts_mode"` // The UTS mode of the container - host and none 215 UsernsMode string `mapstructure:"userns_mode"` // The User namespace mode of the container - host and none 216 PortMapRaw []map[string]string `mapstructure:"port_map"` // 217 PortMap map[string]int `mapstructure:"-"` // A map of host port labels and the ports exposed on the container 218 Privileged bool `mapstructure:"privileged"` // Flag to run the container in privileged mode 219 SysctlRaw []map[string]string `mapstructure:"sysctl"` // 220 Sysctl map[string]string `mapstructure:"-"` // The sysctl custom configurations 221 UlimitRaw []map[string]string `mapstructure:"ulimit"` // 222 Ulimit []docker.ULimit `mapstructure:"-"` // The ulimit custom configurations 223 DNSServers []string `mapstructure:"dns_servers"` // DNS Server for containers 224 DNSSearchDomains []string `mapstructure:"dns_search_domains"` // DNS Search domains for containers 225 DNSOptions []string `mapstructure:"dns_options"` // DNS Options 226 ExtraHosts []string `mapstructure:"extra_hosts"` // Add host to /etc/hosts (host:IP) 227 Hostname string `mapstructure:"hostname"` // Hostname for containers 228 LabelsRaw []map[string]string `mapstructure:"labels"` // 229 Labels map[string]string `mapstructure:"-"` // Labels to set when the container starts up 230 Auth []DockerDriverAuth `mapstructure:"auth"` // Authentication credentials for a private Docker registry 231 AuthSoftFail bool `mapstructure:"auth_soft_fail"` // Soft-fail if auth creds are provided but fail 232 TTY bool `mapstructure:"tty"` // Allocate a Pseudo-TTY 233 Interactive bool `mapstructure:"interactive"` // Keep STDIN open even if not attached 234 ShmSize int64 `mapstructure:"shm_size"` // Size of /dev/shm of the container in bytes 235 WorkDir string `mapstructure:"work_dir"` // Working directory inside the container 236 Logging []DockerLoggingOpts `mapstructure:"logging"` // Logging options for syslog server 237 Volumes []string `mapstructure:"volumes"` // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container 238 Mounts []DockerMount `mapstructure:"mounts"` // Docker volumes to mount 239 VolumeDriver string `mapstructure:"volume_driver"` // Docker volume driver used for the container's volumes 240 ForcePull bool `mapstructure:"force_pull"` // Always force pull before running image, useful if your tags are mutable 241 MacAddress string `mapstructure:"mac_address"` // Pin mac address to container 242 SecurityOpt []string `mapstructure:"security_opt"` // Flags to pass directly to security-opt 243 Devices []DockerDevice `mapstructure:"devices"` // To allow mounting USB or other serial control devices 244 CapAdd []string `mapstructure:"cap_add"` // Flags to pass directly to cap-add 245 CapDrop []string `mapstructure:"cap_drop"` // Flags to pass directly to cap-drop 246 ReadonlyRootfs bool `mapstructure:"readonly_rootfs"` // Mount the container’s root filesystem as read only 247 AdvertiseIPv6Address bool `mapstructure:"advertise_ipv6_address"` // Flag to use the GlobalIPv6Address from the container as the detected IP 248 CPUHardLimit bool `mapstructure:"cpu_hard_limit"` // Enforce CPU hard limit. 249 PidsLimit int64 `mapstructure:"pids_limit"` // Enforce Docker Pids limit 250 } 251 252 func sliceMergeUlimit(ulimitsRaw map[string]string) ([]docker.ULimit, error) { 253 var ulimits []docker.ULimit 254 255 for name, ulimitRaw := range ulimitsRaw { 256 if len(ulimitRaw) == 0 { 257 return []docker.ULimit{}, fmt.Errorf("Malformed ulimit specification %v: %q, cannot be empty", name, ulimitRaw) 258 } 259 // hard limit is optional 260 if strings.Contains(ulimitRaw, ":") == false { 261 ulimitRaw = ulimitRaw + ":" + ulimitRaw 262 } 263 264 splitted := strings.SplitN(ulimitRaw, ":", 2) 265 if len(splitted) < 2 { 266 return []docker.ULimit{}, fmt.Errorf("Malformed ulimit specification %v: %v", name, ulimitRaw) 267 } 268 soft, err := strconv.Atoi(splitted[0]) 269 if err != nil { 270 return []docker.ULimit{}, fmt.Errorf("Malformed soft ulimit %v: %v", name, ulimitRaw) 271 } 272 hard, err := strconv.Atoi(splitted[1]) 273 if err != nil { 274 return []docker.ULimit{}, fmt.Errorf("Malformed hard ulimit %v: %v", name, ulimitRaw) 275 } 276 277 ulimit := docker.ULimit{ 278 Name: name, 279 Soft: int64(soft), 280 Hard: int64(hard), 281 } 282 ulimits = append(ulimits, ulimit) 283 } 284 return ulimits, nil 285 } 286 287 // Validate validates a docker driver config 288 func (c *DockerDriverConfig) Validate() error { 289 if c.ImageName == "" { 290 return fmt.Errorf("Docker Driver needs an image name") 291 } 292 if len(c.Devices) > 0 { 293 for _, dev := range c.Devices { 294 if dev.HostPath == "" { 295 return fmt.Errorf("host path must be set in configuration for devices") 296 } 297 if dev.CgroupPermissions != "" { 298 for _, c := range dev.CgroupPermissions { 299 ch := string(c) 300 if ch != "r" && ch != "w" && ch != "m" { 301 return fmt.Errorf("invalid cgroup permission string: %q", dev.CgroupPermissions) 302 } 303 } 304 } 305 } 306 } 307 c.Sysctl = mapMergeStrStr(c.SysctlRaw...) 308 c.Labels = mapMergeStrStr(c.LabelsRaw...) 309 if len(c.Logging) > 0 { 310 c.Logging[0].Config = mapMergeStrStr(c.Logging[0].ConfigRaw...) 311 } 312 313 mergedUlimitsRaw := mapMergeStrStr(c.UlimitRaw...) 314 ulimit, err := sliceMergeUlimit(mergedUlimitsRaw) 315 if err != nil { 316 return err 317 } 318 c.Ulimit = ulimit 319 return nil 320 } 321 322 // NewDockerDriverConfig returns a docker driver config by parsing the HCL 323 // config 324 func NewDockerDriverConfig(task *structs.Task, env *env.TaskEnv) (*DockerDriverConfig, error) { 325 var dconf DockerDriverConfig 326 327 if err := mapstructure.WeakDecode(task.Config, &dconf); err != nil { 328 return nil, err 329 } 330 331 // Interpolate everything that is a string 332 dconf.ImageName = env.ReplaceEnv(dconf.ImageName) 333 dconf.Command = env.ReplaceEnv(dconf.Command) 334 dconf.Entrypoint = env.ParseAndReplace(dconf.Entrypoint) 335 dconf.IpcMode = env.ReplaceEnv(dconf.IpcMode) 336 dconf.NetworkMode = env.ReplaceEnv(dconf.NetworkMode) 337 dconf.NetworkAliases = env.ParseAndReplace(dconf.NetworkAliases) 338 dconf.IPv4Address = env.ReplaceEnv(dconf.IPv4Address) 339 dconf.IPv6Address = env.ReplaceEnv(dconf.IPv6Address) 340 dconf.PidMode = env.ReplaceEnv(dconf.PidMode) 341 dconf.UTSMode = env.ReplaceEnv(dconf.UTSMode) 342 dconf.Hostname = env.ReplaceEnv(dconf.Hostname) 343 dconf.WorkDir = env.ReplaceEnv(dconf.WorkDir) 344 dconf.LoadImage = env.ReplaceEnv(dconf.LoadImage) 345 dconf.Volumes = env.ParseAndReplace(dconf.Volumes) 346 dconf.VolumeDriver = env.ReplaceEnv(dconf.VolumeDriver) 347 dconf.DNSServers = env.ParseAndReplace(dconf.DNSServers) 348 dconf.DNSSearchDomains = env.ParseAndReplace(dconf.DNSSearchDomains) 349 dconf.DNSOptions = env.ParseAndReplace(dconf.DNSOptions) 350 dconf.ExtraHosts = env.ParseAndReplace(dconf.ExtraHosts) 351 dconf.MacAddress = env.ReplaceEnv(dconf.MacAddress) 352 dconf.SecurityOpt = env.ParseAndReplace(dconf.SecurityOpt) 353 dconf.CapAdd = env.ParseAndReplace(dconf.CapAdd) 354 dconf.CapDrop = env.ParseAndReplace(dconf.CapDrop) 355 356 for _, m := range dconf.SysctlRaw { 357 for k, v := range m { 358 delete(m, k) 359 m[env.ReplaceEnv(k)] = env.ReplaceEnv(v) 360 } 361 } 362 363 for _, m := range dconf.UlimitRaw { 364 for k, v := range m { 365 delete(m, k) 366 m[env.ReplaceEnv(k)] = env.ReplaceEnv(v) 367 } 368 } 369 370 for _, m := range dconf.LabelsRaw { 371 for k, v := range m { 372 delete(m, k) 373 m[env.ReplaceEnv(k)] = env.ReplaceEnv(v) 374 } 375 } 376 dconf.Labels = mapMergeStrStr(dconf.LabelsRaw...) 377 378 for i, a := range dconf.Auth { 379 dconf.Auth[i].Username = env.ReplaceEnv(a.Username) 380 dconf.Auth[i].Password = env.ReplaceEnv(a.Password) 381 dconf.Auth[i].Email = env.ReplaceEnv(a.Email) 382 dconf.Auth[i].ServerAddress = env.ReplaceEnv(a.ServerAddress) 383 } 384 385 for i, l := range dconf.Logging { 386 dconf.Logging[i].Type = env.ReplaceEnv(l.Type) 387 for _, c := range l.ConfigRaw { 388 for k, v := range c { 389 delete(c, k) 390 c[env.ReplaceEnv(k)] = env.ReplaceEnv(v) 391 } 392 } 393 } 394 395 for i, m := range dconf.Mounts { 396 m.Target = env.ReplaceEnv(m.Target) 397 m.Source = env.ReplaceEnv(m.Source) 398 399 if m.Type == "" { 400 // default to `volume` type for backwards compatibility 401 m.Type = "volume" 402 } 403 404 if m.Type != "bind" && m.Type != "volume" { 405 return nil, fmt.Errorf(`Docker mount type must be "bind" or "volume"`) 406 } 407 408 if m.Type == "bind" && len(m.VolumeOptions) > 0 { 409 return nil, fmt.Errorf(`volume_options invalid on "bind" mount type`) 410 } 411 412 if m.Type == "volume" && len(m.BindOptions) > 0 { 413 return nil, fmt.Errorf(`bind_options invalid on "volume" mount type`) 414 } 415 416 if len(m.BindOptions) > 1 { 417 return nil, fmt.Errorf("only one bind_options stanza allowed") 418 } 419 420 if len(m.VolumeOptions) > 1 { 421 return nil, fmt.Errorf("only one volume_options stanza allowed") 422 } 423 424 if len(m.VolumeOptions) == 1 { 425 vo := m.VolumeOptions[0] 426 if len(vo.Labels) > 1 { 427 return nil, fmt.Errorf("labels may only be specified once in volume_options stanza") 428 } 429 430 if len(vo.Labels) == 1 { 431 for k, v := range vo.Labels[0] { 432 if k != env.ReplaceEnv(k) { 433 delete(vo.Labels[0], k) 434 } 435 vo.Labels[0][env.ReplaceEnv(k)] = env.ReplaceEnv(v) 436 } 437 } 438 439 if len(vo.DriverConfig) > 1 { 440 return nil, fmt.Errorf("volume driver config may only be specified once") 441 } 442 if len(vo.DriverConfig) == 1 { 443 vo.DriverConfig[0].Name = env.ReplaceEnv(vo.DriverConfig[0].Name) 444 if len(vo.DriverConfig[0].Options) > 1 { 445 return nil, fmt.Errorf("volume driver options may only be specified once") 446 } 447 448 if len(vo.DriverConfig[0].Options) == 1 { 449 options := vo.DriverConfig[0].Options[0] 450 for k, v := range options { 451 if k != env.ReplaceEnv(k) { 452 delete(options, k) 453 } 454 options[env.ReplaceEnv(k)] = env.ReplaceEnv(v) 455 } 456 } 457 } 458 } 459 460 // Copy altered version back into slice 461 dconf.Mounts[i] = m 462 } 463 464 if len(dconf.Logging) > 0 { 465 dconf.Logging[0].Config = mapMergeStrStr(dconf.Logging[0].ConfigRaw...) 466 } 467 468 portMap := make(map[string]int) 469 for _, m := range dconf.PortMapRaw { 470 for k, v := range m { 471 ki, vi := env.ReplaceEnv(k), env.ReplaceEnv(v) 472 p, err := strconv.Atoi(vi) 473 if err != nil { 474 return nil, fmt.Errorf("failed to parse port map value %v to %v: %v", ki, vi, err) 475 } 476 portMap[ki] = p 477 } 478 } 479 dconf.PortMap = portMap 480 481 // Remove any http 482 if strings.Contains(dconf.ImageName, "https://") { 483 dconf.ImageName = strings.Replace(dconf.ImageName, "https://", "", 1) 484 } 485 486 // If devices are configured set default cgroup permissions 487 if len(dconf.Devices) > 0 { 488 for i, dev := range dconf.Devices { 489 if dev.CgroupPermissions == "" { 490 dev.CgroupPermissions = "rwm" 491 } 492 dconf.Devices[i] = dev 493 } 494 } 495 496 if err := dconf.Validate(); err != nil { 497 return nil, err 498 } 499 return &dconf, nil 500 } 501 502 type dockerPID struct { 503 Version string 504 Image string 505 ImageID string 506 ContainerID string 507 KillTimeout time.Duration 508 MaxKillTimeout time.Duration 509 PluginConfig *PluginReattachConfig 510 } 511 512 type DockerHandle struct { 513 pluginClient *plugin.Client 514 executor executor.Executor 515 client *docker.Client 516 waitClient *docker.Client 517 logger *log.Logger 518 jobName string 519 taskGroupName string 520 taskName string 521 Image string 522 ImageID string 523 containerID string 524 version string 525 killTimeout time.Duration 526 maxKillTimeout time.Duration 527 resourceUsageLock sync.RWMutex 528 resourceUsage *cstructs.TaskResourceUsage 529 waitCh chan *dstructs.WaitResult 530 doneCh chan bool 531 removeContainerOnExit bool 532 } 533 534 func NewDockerDriver(ctx *DriverContext) Driver { 535 return &DockerDriver{DriverContext: *ctx} 536 } 537 538 func (d *DockerDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error { 539 client, _, err := d.dockerClients() 540 if err != nil { 541 if d.fingerprintSuccess == nil || *d.fingerprintSuccess { 542 d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s", err) 543 } 544 d.fingerprintSuccess = helper.BoolToPtr(false) 545 return nil 546 } 547 548 // This is the first operation taken on the client so we'll try to 549 // establish a connection to the Docker daemon. If this fails it means 550 // Docker isn't available so we'll simply disable the docker driver. 551 env, err := client.Version() 552 if err != nil { 553 if d.fingerprintSuccess == nil || *d.fingerprintSuccess { 554 d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon at %s: %s", client.Endpoint(), err) 555 } 556 d.fingerprintSuccess = helper.BoolToPtr(false) 557 resp.RemoveAttribute(dockerDriverAttr) 558 return nil 559 } 560 561 resp.AddAttribute(dockerDriverAttr, "1") 562 resp.AddAttribute("driver.docker.version", env.Get("Version")) 563 resp.Detected = true 564 565 privileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false) 566 if privileged { 567 resp.AddAttribute(dockerPrivilegedConfigOption, "1") 568 } 569 570 // Advertise if this node supports Docker volumes 571 if d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault) { 572 resp.AddAttribute("driver."+dockerVolumesConfigOption, "1") 573 } 574 575 // Detect bridge IP address - #2785 576 if nets, err := client.ListNetworks(); err != nil { 577 d.logger.Printf("[WARN] driver.docker: error discovering bridge IP: %v", err) 578 } else { 579 for _, n := range nets { 580 if n.Name != "bridge" { 581 continue 582 } 583 584 if len(n.IPAM.Config) == 0 { 585 d.logger.Printf("[WARN] driver.docker: no IPAM config for bridge network") 586 break 587 } 588 589 if n.IPAM.Config[0].Gateway != "" { 590 resp.AddAttribute("driver.docker.bridge_ip", n.IPAM.Config[0].Gateway) 591 } else if d.fingerprintSuccess == nil { 592 // Docker 17.09.0-ce dropped the Gateway IP from the bridge network 593 // See https://github.com/moby/moby/issues/32648 594 d.logger.Printf("[DEBUG] driver.docker: bridge_ip could not be discovered") 595 } 596 break 597 } 598 } 599 600 d.fingerprintSuccess = helper.BoolToPtr(true) 601 return nil 602 } 603 604 // HealthCheck implements the interface for the HealthCheck interface. This 605 // performs a health check on the docker driver, asserting whether the docker 606 // driver is responsive to a `docker ps` command. 607 func (d *DockerDriver) HealthCheck(req *cstructs.HealthCheckRequest, resp *cstructs.HealthCheckResponse) error { 608 dinfo := &structs.DriverInfo{ 609 UpdateTime: time.Now(), 610 } 611 612 healthCheckClient, err := d.dockerHealthCheckClient() 613 if err != nil { 614 d.logger.Printf("[WARN] driver.docker: failed to retrieve Docker client in the process of a docker health check: %v", err) 615 dinfo.HealthDescription = fmt.Sprintf("Failed retrieving Docker client: %v", err) 616 resp.AddDriverInfo("docker", dinfo) 617 return nil 618 } 619 620 _, err = healthCheckClient.ListContainers(docker.ListContainersOptions{All: false}) 621 if err != nil { 622 d.logger.Printf("[WARN] driver.docker: failed to list Docker containers in the process of a Docker health check: %v", err) 623 dinfo.HealthDescription = fmt.Sprintf("Failed to list Docker containers: %v", err) 624 resp.AddDriverInfo("docker", dinfo) 625 return nil 626 } 627 628 d.logger.Printf("[TRACE] driver.docker: docker driver is available and is responsive to `docker ps`") 629 dinfo.Healthy = true 630 dinfo.HealthDescription = "Driver is available and responsive" 631 resp.AddDriverInfo("docker", dinfo) 632 return nil 633 } 634 635 // GetHealthChecks implements the interface for the HealthCheck interface. This 636 // sets whether the driver is eligible for periodic health checks and the 637 // interval at which to do them. 638 func (d *DockerDriver) GetHealthCheckInterval(req *cstructs.HealthCheckIntervalRequest, resp *cstructs.HealthCheckIntervalResponse) error { 639 resp.Eligible = true 640 resp.Period = 1 * time.Minute 641 return nil 642 } 643 644 // Validate is used to validate the driver configuration 645 func (d *DockerDriver) Validate(config map[string]interface{}) error { 646 fd := &fields.FieldData{ 647 Raw: config, 648 Schema: map[string]*fields.FieldSchema{ 649 "image": { 650 Type: fields.TypeString, 651 Required: true, 652 }, 653 "load": { 654 Type: fields.TypeString, 655 }, 656 "command": { 657 Type: fields.TypeString, 658 }, 659 "args": { 660 Type: fields.TypeArray, 661 }, 662 "entrypoint": { 663 Type: fields.TypeArray, 664 }, 665 "ipc_mode": { 666 Type: fields.TypeString, 667 }, 668 "network_mode": { 669 Type: fields.TypeString, 670 }, 671 "network_aliases": { 672 Type: fields.TypeArray, 673 }, 674 "ipv4_address": { 675 Type: fields.TypeString, 676 }, 677 "ipv6_address": { 678 Type: fields.TypeString, 679 }, 680 "mac_address": { 681 Type: fields.TypeString, 682 }, 683 "pid_mode": { 684 Type: fields.TypeString, 685 }, 686 "uts_mode": { 687 Type: fields.TypeString, 688 }, 689 "userns_mode": { 690 Type: fields.TypeString, 691 }, 692 "sysctl": { 693 Type: fields.TypeArray, 694 }, 695 "ulimit": { 696 Type: fields.TypeArray, 697 }, 698 "port_map": { 699 Type: fields.TypeArray, 700 }, 701 "privileged": { 702 Type: fields.TypeBool, 703 }, 704 "dns_servers": { 705 Type: fields.TypeArray, 706 }, 707 "dns_options": { 708 Type: fields.TypeArray, 709 }, 710 "dns_search_domains": { 711 Type: fields.TypeArray, 712 }, 713 "extra_hosts": { 714 Type: fields.TypeArray, 715 }, 716 "hostname": { 717 Type: fields.TypeString, 718 }, 719 "labels": { 720 Type: fields.TypeArray, 721 }, 722 "auth": { 723 Type: fields.TypeArray, 724 }, 725 "auth_soft_fail": { 726 Type: fields.TypeBool, 727 }, 728 // COMPAT: Remove in 0.6.0. SSL is no longer needed 729 "ssl": { 730 Type: fields.TypeBool, 731 }, 732 "tty": { 733 Type: fields.TypeBool, 734 }, 735 "interactive": { 736 Type: fields.TypeBool, 737 }, 738 "shm_size": { 739 Type: fields.TypeInt, 740 }, 741 "work_dir": { 742 Type: fields.TypeString, 743 }, 744 "logging": { 745 Type: fields.TypeArray, 746 }, 747 "volumes": { 748 Type: fields.TypeArray, 749 }, 750 "volume_driver": { 751 Type: fields.TypeString, 752 }, 753 "mounts": { 754 Type: fields.TypeArray, 755 }, 756 "force_pull": { 757 Type: fields.TypeBool, 758 }, 759 "security_opt": { 760 Type: fields.TypeArray, 761 }, 762 "devices": { 763 Type: fields.TypeArray, 764 }, 765 "cap_add": { 766 Type: fields.TypeArray, 767 }, 768 "cap_drop": { 769 Type: fields.TypeArray, 770 }, 771 "readonly_rootfs": { 772 Type: fields.TypeBool, 773 }, 774 "advertise_ipv6_address": { 775 Type: fields.TypeBool, 776 }, 777 "cpu_hard_limit": { 778 Type: fields.TypeBool, 779 }, 780 "pids_limit": { 781 Type: fields.TypeInt, 782 }, 783 }, 784 } 785 786 if err := fd.Validate(); err != nil { 787 return err 788 } 789 790 return nil 791 } 792 793 func (d *DockerDriver) Abilities() DriverAbilities { 794 return DriverAbilities{ 795 SendSignals: true, 796 Exec: true, 797 } 798 } 799 800 func (d *DockerDriver) FSIsolation() cstructs.FSIsolation { 801 return cstructs.FSIsolationImage 802 } 803 804 // getDockerCoordinator returns the docker coordinator and the caller ID to use when 805 // interacting with the coordinator 806 func (d *DockerDriver) getDockerCoordinator(client *docker.Client) (*dockerCoordinator, string) { 807 config := &dockerCoordinatorConfig{ 808 client: client, 809 cleanup: d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault), 810 logger: d.logger, 811 removeDelay: d.config.ReadDurationDefault(dockerImageRemoveDelayConfigOption, dockerImageRemoveDelayConfigDefault), 812 } 813 814 return GetDockerCoordinator(config), fmt.Sprintf("%s-%s", d.DriverContext.allocID, d.DriverContext.taskName) 815 } 816 817 func (d *DockerDriver) Prestart(ctx *ExecContext, task *structs.Task) (*PrestartResponse, error) { 818 driverConfig, err := NewDockerDriverConfig(task, ctx.TaskEnv) 819 if err != nil { 820 return nil, err 821 } 822 823 // Set state needed by Start 824 d.driverConfig = driverConfig 825 826 // Initialize docker API clients 827 client, _, err := d.dockerClients() 828 if err != nil { 829 return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) 830 } 831 832 // Ensure the image is available 833 id, err := d.createImage(driverConfig, client, ctx.TaskDir) 834 if err != nil { 835 return nil, err 836 } 837 d.imageID = id 838 839 resp := NewPrestartResponse() 840 resp.CreatedResources.Add(dockerImageResKey, id) 841 842 // Return the PortMap if it's set 843 if len(driverConfig.PortMap) > 0 { 844 resp.Network = &cstructs.DriverNetwork{ 845 PortMap: driverConfig.PortMap, 846 } 847 } 848 return resp, nil 849 } 850 851 func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) { 852 pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out") 853 executorConfig := &dstructs.ExecutorConfig{ 854 LogFile: pluginLogFile, 855 LogLevel: d.config.LogLevel, 856 } 857 858 exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig) 859 if err != nil { 860 return nil, err 861 } 862 executorCtx := &executor.ExecutorContext{ 863 TaskEnv: ctx.TaskEnv, 864 Task: task, 865 Driver: "docker", 866 LogDir: ctx.TaskDir.LogDir, 867 TaskDir: ctx.TaskDir.Dir, 868 PortLowerBound: d.config.ClientMinPort, 869 PortUpperBound: d.config.ClientMaxPort, 870 } 871 if err := exec.SetContext(executorCtx); err != nil { 872 pluginClient.Kill() 873 return nil, fmt.Errorf("failed to set executor context: %v", err) 874 } 875 876 // The user hasn't specified any logging options so launch our own syslog 877 // server if possible. 878 syslogAddr := "" 879 if len(d.driverConfig.Logging) == 0 { 880 if runtime.GOOS == "darwin" { 881 d.logger.Printf("[DEBUG] driver.docker: disabling syslog driver as Docker for Mac workaround") 882 } else { 883 ss, err := exec.LaunchSyslogServer() 884 if err != nil { 885 pluginClient.Kill() 886 return nil, fmt.Errorf("failed to start syslog collector: %v", err) 887 } 888 syslogAddr = ss.Addr 889 } 890 } 891 892 config, err := d.createContainerConfig(ctx, task, d.driverConfig, syslogAddr) 893 if err != nil { 894 d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err) 895 pluginClient.Kill() 896 return nil, fmt.Errorf("Failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err) 897 } 898 899 container, err := d.createContainer(client, config) 900 if err != nil { 901 wrapped := fmt.Sprintf("Failed to create container: %v", err) 902 d.logger.Printf("[ERR] driver.docker: %s", wrapped) 903 pluginClient.Kill() 904 return nil, structs.WrapRecoverable(wrapped, err) 905 } 906 907 d.logger.Printf("[INFO] driver.docker: created container %s", container.ID) 908 909 // We don't need to start the container if the container is already running 910 // since we don't create containers which are already present on the host 911 // and are running 912 if !container.State.Running { 913 // Start the container 914 if err := d.startContainer(container); err != nil { 915 d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err) 916 pluginClient.Kill() 917 return nil, structs.NewRecoverableError(fmt.Errorf("Failed to start container %s: %s", container.ID, err), structs.IsRecoverable(err)) 918 } 919 920 // InspectContainer to get all of the container metadata as 921 // much of the metadata (eg networking) isn't populated until 922 // the container is started 923 runningContainer, err := client.InspectContainer(container.ID) 924 if err != nil { 925 err = fmt.Errorf("failed to inspect started container %s: %s", container.ID, err) 926 d.logger.Printf("[ERR] driver.docker: %v", err) 927 pluginClient.Kill() 928 return nil, structs.NewRecoverableError(err, true) 929 } 930 container = runningContainer 931 d.logger.Printf("[INFO] driver.docker: started container %s", container.ID) 932 } else { 933 d.logger.Printf("[DEBUG] driver.docker: re-attaching to container %s with status %q", 934 container.ID, container.State.String()) 935 } 936 937 // Return a driver handle 938 maxKill := d.DriverContext.config.MaxKillTimeout 939 h := &DockerHandle{ 940 client: client, 941 waitClient: waitClient, 942 executor: exec, 943 pluginClient: pluginClient, 944 logger: d.logger, 945 jobName: d.DriverContext.jobName, 946 taskGroupName: d.DriverContext.taskGroupName, 947 taskName: d.DriverContext.taskName, 948 Image: d.driverConfig.ImageName, 949 ImageID: d.imageID, 950 containerID: container.ID, 951 version: d.config.Version.VersionNumber(), 952 killTimeout: GetKillTimeout(task.KillTimeout, maxKill), 953 maxKillTimeout: maxKill, 954 doneCh: make(chan bool), 955 waitCh: make(chan *dstructs.WaitResult, 1), 956 removeContainerOnExit: d.config.ReadBoolDefault(dockerCleanupContainerConfigOption, dockerCleanupContainerConfigDefault), 957 } 958 go h.collectStats() 959 go h.run() 960 961 // Detect container address 962 ip, autoUse := d.detectIP(container) 963 964 // Create a response with the driver handle and container network metadata 965 resp := &StartResponse{ 966 Handle: h, 967 Network: &cstructs.DriverNetwork{ 968 PortMap: d.driverConfig.PortMap, 969 IP: ip, 970 AutoAdvertise: autoUse, 971 }, 972 } 973 return resp, nil 974 } 975 976 // detectIP of Docker container. Returns the first IP found as well as true if 977 // the IP should be advertised (bridge network IPs return false). Returns an 978 // empty string and false if no IP could be found. 979 func (d *DockerDriver) detectIP(c *docker.Container) (string, bool) { 980 if c.NetworkSettings == nil { 981 // This should only happen if there's been a coding error (such 982 // as not calling InspectContainer after CreateContainer). Code 983 // defensively in case the Docker API changes subtly. 984 d.logger.Printf("[ERROR] driver.docker: no network settings for container %s", c.ID) 985 return "", false 986 } 987 988 ip, ipName := "", "" 989 auto := false 990 for name, net := range c.NetworkSettings.Networks { 991 if net.IPAddress == "" { 992 // Ignore networks without an IP address 993 continue 994 } 995 996 ip = net.IPAddress 997 if d.driverConfig.AdvertiseIPv6Address { 998 ip = net.GlobalIPv6Address 999 auto = true 1000 } 1001 ipName = name 1002 1003 // Don't auto-advertise IPs for default networks (bridge on 1004 // Linux, nat on Windows) 1005 if name != "bridge" && name != "nat" { 1006 auto = true 1007 } 1008 1009 break 1010 } 1011 1012 if n := len(c.NetworkSettings.Networks); n > 1 { 1013 d.logger.Printf("[WARN] driver.docker: task %s multiple (%d) Docker networks for container %q but Nomad only supports 1: choosing %q", d.taskName, n, c.ID, ipName) 1014 } 1015 1016 return ip, auto 1017 } 1018 1019 func (d *DockerDriver) Cleanup(_ *ExecContext, res *CreatedResources) error { 1020 retry := false 1021 var merr multierror.Error 1022 for key, resources := range res.Resources { 1023 switch key { 1024 case dockerImageResKey: 1025 for _, value := range resources { 1026 err := d.cleanupImage(value) 1027 if err != nil { 1028 if structs.IsRecoverable(err) { 1029 retry = true 1030 } 1031 merr.Errors = append(merr.Errors, err) 1032 continue 1033 } 1034 1035 // Remove cleaned image from resources 1036 res.Remove(dockerImageResKey, value) 1037 } 1038 default: 1039 d.logger.Printf("[ERR] driver.docker: unknown resource to cleanup: %q", key) 1040 } 1041 } 1042 return structs.NewRecoverableError(merr.ErrorOrNil(), retry) 1043 } 1044 1045 // cleanupImage removes a Docker image. No error is returned if the image 1046 // doesn't exist or is still in use. Requires the global client to already be 1047 // initialized. 1048 func (d *DockerDriver) cleanupImage(imageID string) error { 1049 if !d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault) { 1050 // Config says not to cleanup 1051 return nil 1052 } 1053 1054 coordinator, callerID := d.getDockerCoordinator(client) 1055 coordinator.RemoveImage(imageID, callerID) 1056 1057 return nil 1058 } 1059 1060 // dockerHealthCheckClient creates a single *docker.Client with a timeout of 1061 // one minute, which will be used when performing Docker health checks. 1062 func (d *DockerDriver) dockerHealthCheckClient() (*docker.Client, error) { 1063 createClientsLock.Lock() 1064 defer createClientsLock.Unlock() 1065 1066 if healthCheckClient != nil { 1067 return healthCheckClient, nil 1068 } 1069 1070 var err error 1071 healthCheckClient, err = d.newDockerClient(dockerHealthCheckTimeout) 1072 if err != nil { 1073 return nil, err 1074 } 1075 1076 return healthCheckClient, nil 1077 } 1078 1079 // dockerClients creates two *docker.Client, one for long running operations and 1080 // the other for shorter operations. In test / dev mode we can use ENV vars to 1081 // connect to the docker daemon. In production mode we will read docker.endpoint 1082 // from the config file. 1083 func (d *DockerDriver) dockerClients() (*docker.Client, *docker.Client, error) { 1084 createClientsLock.Lock() 1085 defer createClientsLock.Unlock() 1086 1087 if client != nil && waitClient != nil { 1088 return client, waitClient, nil 1089 } 1090 1091 var err error 1092 1093 // Onlt initialize the client if it hasn't yet been done 1094 if client == nil { 1095 client, err = d.newDockerClient(dockerTimeout) 1096 if err != nil { 1097 return nil, nil, err 1098 } 1099 } 1100 1101 // Only initialize the waitClient if it hasn't yet been done 1102 if waitClient == nil { 1103 waitClient, err = d.newDockerClient(0 * time.Minute) 1104 if err != nil { 1105 return nil, nil, err 1106 } 1107 } 1108 1109 return client, waitClient, nil 1110 } 1111 1112 // newDockerClient creates a new *docker.Client with a configurable timeout 1113 func (d *DockerDriver) newDockerClient(timeout time.Duration) (*docker.Client, error) { 1114 var err error 1115 var merr multierror.Error 1116 var newClient *docker.Client 1117 1118 // Default to using whatever is configured in docker.endpoint. If this is 1119 // not specified we'll fall back on NewClientFromEnv which reads config from 1120 // the DOCKER_* environment variables DOCKER_HOST, DOCKER_TLS_VERIFY, and 1121 // DOCKER_CERT_PATH. This allows us to lock down the config in production 1122 // but also accept the standard ENV configs for dev and test. 1123 dockerEndpoint := d.config.Read("docker.endpoint") 1124 if dockerEndpoint != "" { 1125 cert := d.config.Read("docker.tls.cert") 1126 key := d.config.Read("docker.tls.key") 1127 ca := d.config.Read("docker.tls.ca") 1128 1129 if cert+key+ca != "" { 1130 d.logger.Printf("[DEBUG] driver.docker: using TLS client connection to %s", dockerEndpoint) 1131 newClient, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca) 1132 if err != nil { 1133 merr.Errors = append(merr.Errors, err) 1134 } 1135 } else { 1136 d.logger.Printf("[DEBUG] driver.docker: using standard client connection to %s", dockerEndpoint) 1137 newClient, err = docker.NewClient(dockerEndpoint) 1138 if err != nil { 1139 merr.Errors = append(merr.Errors, err) 1140 } 1141 } 1142 } else { 1143 d.logger.Println("[DEBUG] driver.docker: using client connection initialized from environment") 1144 newClient, err = docker.NewClientFromEnv() 1145 if err != nil { 1146 merr.Errors = append(merr.Errors, err) 1147 } 1148 } 1149 1150 if timeout != 0 && newClient != nil { 1151 newClient.SetTimeout(timeout) 1152 } 1153 return newClient, merr.ErrorOrNil() 1154 } 1155 1156 func (d *DockerDriver) containerBinds(driverConfig *DockerDriverConfig, ctx *ExecContext, 1157 task *structs.Task) ([]string, error) { 1158 1159 allocDirBind := fmt.Sprintf("%s:%s", ctx.TaskDir.SharedAllocDir, ctx.TaskEnv.EnvMap[env.AllocDir]) 1160 taskLocalBind := fmt.Sprintf("%s:%s", ctx.TaskDir.LocalDir, ctx.TaskEnv.EnvMap[env.TaskLocalDir]) 1161 secretDirBind := fmt.Sprintf("%s:%s", ctx.TaskDir.SecretsDir, ctx.TaskEnv.EnvMap[env.SecretsDir]) 1162 binds := []string{allocDirBind, taskLocalBind, secretDirBind} 1163 1164 volumesEnabled := d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault) 1165 1166 if !volumesEnabled && driverConfig.VolumeDriver != "" { 1167 return nil, fmt.Errorf("%s is false; cannot use volume driver %q", dockerVolumesConfigOption, driverConfig.VolumeDriver) 1168 } 1169 1170 for _, userbind := range driverConfig.Volumes { 1171 parts := strings.Split(userbind, ":") 1172 if len(parts) < 2 { 1173 return nil, fmt.Errorf("invalid docker volume: %q", userbind) 1174 } 1175 1176 // Resolve dotted path segments 1177 parts[0] = filepath.Clean(parts[0]) 1178 1179 // Absolute paths aren't always supported 1180 if filepath.IsAbs(parts[0]) { 1181 if !volumesEnabled { 1182 // Disallow mounting arbitrary absolute paths 1183 return nil, fmt.Errorf("%s is false; cannot mount host paths: %+q", dockerVolumesConfigOption, userbind) 1184 } 1185 binds = append(binds, userbind) 1186 continue 1187 } 1188 1189 // Relative paths are always allowed as they mount within a container 1190 // When a VolumeDriver is set, we assume we receive a binding in the format volume-name:container-dest 1191 // Otherwise, we assume we receive a relative path binding in the format relative/to/task:/also/in/container 1192 if driverConfig.VolumeDriver == "" { 1193 // Expand path relative to alloc dir 1194 parts[0] = filepath.Join(ctx.TaskDir.Dir, parts[0]) 1195 } 1196 1197 binds = append(binds, strings.Join(parts, ":")) 1198 } 1199 1200 if selinuxLabel := d.config.Read(dockerSELinuxLabelConfigOption); selinuxLabel != "" { 1201 // Apply SELinux Label to each volume 1202 for i := range binds { 1203 binds[i] = fmt.Sprintf("%s:%s", binds[i], selinuxLabel) 1204 } 1205 } 1206 1207 return binds, nil 1208 } 1209 1210 // createContainerConfig initializes a struct needed to call docker.client.CreateContainer() 1211 func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Task, 1212 driverConfig *DockerDriverConfig, syslogAddr string) (docker.CreateContainerOptions, error) { 1213 var c docker.CreateContainerOptions 1214 if task.Resources == nil { 1215 // Guard against missing resources. We should never have been able to 1216 // schedule a job without specifying this. 1217 d.logger.Println("[ERR] driver.docker: task.Resources is empty") 1218 return c, fmt.Errorf("task.Resources is empty") 1219 } 1220 1221 binds, err := d.containerBinds(driverConfig, ctx, task) 1222 if err != nil { 1223 return c, err 1224 } 1225 1226 // create the config block that will later be consumed by go-dockerclient 1227 config := &docker.Config{ 1228 Image: d.imageID, 1229 Entrypoint: driverConfig.Entrypoint, 1230 Hostname: driverConfig.Hostname, 1231 User: task.User, 1232 Tty: driverConfig.TTY, 1233 OpenStdin: driverConfig.Interactive, 1234 StopTimeout: int(task.KillTimeout.Seconds()), 1235 StopSignal: task.KillSignal, 1236 } 1237 1238 if driverConfig.WorkDir != "" { 1239 config.WorkingDir = driverConfig.WorkDir 1240 } 1241 1242 memLimit := int64(task.Resources.MemoryMB) * 1024 * 1024 1243 1244 if len(driverConfig.Logging) == 0 { 1245 if runtime.GOOS == "darwin" { 1246 d.logger.Printf("[DEBUG] driver.docker: deferring logging to docker on Docker for Mac") 1247 } else { 1248 d.logger.Printf("[DEBUG] driver.docker: Setting default logging options to syslog and %s", syslogAddr) 1249 driverConfig.Logging = []DockerLoggingOpts{ 1250 {Type: "syslog", Config: map[string]string{"syslog-address": syslogAddr}}, 1251 } 1252 } 1253 } 1254 1255 hostConfig := &docker.HostConfig{ 1256 // Convert MB to bytes. This is an absolute value. 1257 Memory: memLimit, 1258 // Convert Mhz to shares. This is a relative value. 1259 CPUShares: int64(task.Resources.CPU), 1260 1261 // Binds are used to mount a host volume into the container. We mount a 1262 // local directory for storage and a shared alloc directory that can be 1263 // used to share data between different tasks in the same task group. 1264 Binds: binds, 1265 1266 VolumeDriver: driverConfig.VolumeDriver, 1267 1268 PidsLimit: driverConfig.PidsLimit, 1269 } 1270 1271 // Calculate CPU Quota 1272 // cfs_quota_us is the time per core, so we must 1273 // multiply the time by the number of cores available 1274 // See https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/sec-cpu 1275 if driverConfig.CPUHardLimit { 1276 numCores := runtime.NumCPU() 1277 percentTicks := float64(task.Resources.CPU) / float64(d.node.Resources.CPU) 1278 hostConfig.CPUQuota = int64(percentTicks*defaultCFSPeriodUS) * int64(numCores) 1279 } 1280 1281 // Windows does not support MemorySwap/MemorySwappiness #2193 1282 if runtime.GOOS == "windows" { 1283 hostConfig.MemorySwap = 0 1284 hostConfig.MemorySwappiness = -1 1285 } else { 1286 hostConfig.MemorySwap = memLimit // MemorySwap is memory + swap. 1287 } 1288 1289 if len(driverConfig.Logging) != 0 { 1290 d.logger.Printf("[DEBUG] driver.docker: Using config for logging: %+v", driverConfig.Logging[0]) 1291 hostConfig.LogConfig = docker.LogConfig{ 1292 Type: driverConfig.Logging[0].Type, 1293 Config: driverConfig.Logging[0].Config, 1294 } 1295 } 1296 1297 d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name) 1298 d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name) 1299 if driverConfig.CPUHardLimit { 1300 d.logger.Printf("[DEBUG] driver.docker: using %dms cpu quota and %dms cpu period for %s", hostConfig.CPUQuota, defaultCFSPeriodUS, task.Name) 1301 } 1302 d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name) 1303 1304 // set privileged mode 1305 hostPrivileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false) 1306 if driverConfig.Privileged && !hostPrivileged { 1307 return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`) 1308 } 1309 hostConfig.Privileged = driverConfig.Privileged 1310 1311 // set capabilities 1312 hostCapsWhitelistConfig := d.config.ReadDefault( 1313 dockerCapsWhitelistConfigOption, dockerCapsWhitelistConfigDefault) 1314 hostCapsWhitelist := make(map[string]struct{}) 1315 for _, cap := range strings.Split(hostCapsWhitelistConfig, ",") { 1316 cap = strings.ToLower(strings.TrimSpace(cap)) 1317 hostCapsWhitelist[cap] = struct{}{} 1318 } 1319 1320 if _, ok := hostCapsWhitelist["all"]; !ok { 1321 effectiveCaps, err := tweakCapabilities( 1322 strings.Split(dockerBasicCaps, ","), 1323 driverConfig.CapAdd, 1324 driverConfig.CapDrop, 1325 ) 1326 if err != nil { 1327 return c, err 1328 } 1329 var missingCaps []string 1330 for _, cap := range effectiveCaps { 1331 cap = strings.ToLower(cap) 1332 if _, ok := hostCapsWhitelist[cap]; !ok { 1333 missingCaps = append(missingCaps, cap) 1334 } 1335 } 1336 if len(missingCaps) > 0 { 1337 return c, fmt.Errorf("Docker driver doesn't have the following caps whitelisted on this Nomad agent: %s", missingCaps) 1338 } 1339 } 1340 1341 hostConfig.CapAdd = driverConfig.CapAdd 1342 hostConfig.CapDrop = driverConfig.CapDrop 1343 1344 // set SHM size 1345 if driverConfig.ShmSize != 0 { 1346 hostConfig.ShmSize = driverConfig.ShmSize 1347 } 1348 1349 // set DNS servers 1350 for _, ip := range driverConfig.DNSServers { 1351 if net.ParseIP(ip) != nil { 1352 hostConfig.DNS = append(hostConfig.DNS, ip) 1353 } else { 1354 d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip) 1355 } 1356 } 1357 1358 if len(driverConfig.Devices) > 0 { 1359 var devices []docker.Device 1360 for _, device := range driverConfig.Devices { 1361 dev := docker.Device{ 1362 PathOnHost: device.HostPath, 1363 PathInContainer: device.ContainerPath, 1364 CgroupPermissions: device.CgroupPermissions} 1365 devices = append(devices, dev) 1366 } 1367 hostConfig.Devices = devices 1368 } 1369 1370 // Setup mounts 1371 for _, m := range driverConfig.Mounts { 1372 hm := docker.HostMount{ 1373 Target: m.Target, 1374 Source: m.Source, 1375 Type: m.Type, 1376 ReadOnly: m.ReadOnly, 1377 } 1378 1379 if hm.Type == "bind" { 1380 volumesEnabled := d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault) 1381 hm.Source = filepath.Clean(hm.Source) 1382 1383 if filepath.IsAbs(hm.Source) { 1384 d.logger.Println("[DEBUG] driver.docker: Job contains an absolute path.") 1385 if !volumesEnabled { 1386 // Disallow mounting arbitrary absolute paths 1387 return c, fmt.Errorf("%s is false; cannot bind mount host paths: %+q", dockerVolumesConfigOption, m.Source) 1388 } 1389 } else { 1390 // Expand path relative to alloc dir 1391 hm.Source = filepath.Join(ctx.TaskDir.Dir, m.Source) 1392 d.logger.Printf("[DEBUG] driver.docker: Job contains a relative path; expanding to %v", hm.Source) 1393 } 1394 1395 if len(m.BindOptions) == 1 { 1396 bo := m.BindOptions[0] 1397 hm.BindOptions = &docker.BindOptions{ 1398 Propagation: bo.Propagation, 1399 } 1400 } 1401 } else { 1402 if len(m.VolumeOptions) == 1 { 1403 vo := m.VolumeOptions[0] 1404 hm.VolumeOptions = &docker.VolumeOptions{ 1405 NoCopy: vo.NoCopy, 1406 } 1407 1408 if len(vo.DriverConfig) == 1 { 1409 dc := vo.DriverConfig[0] 1410 hm.VolumeOptions.DriverConfig = docker.VolumeDriverConfig{ 1411 Name: dc.Name, 1412 } 1413 if len(dc.Options) == 1 { 1414 hm.VolumeOptions.DriverConfig.Options = dc.Options[0] 1415 } 1416 } 1417 if len(vo.Labels) == 1 { 1418 hm.VolumeOptions.Labels = vo.Labels[0] 1419 } 1420 } 1421 } 1422 hostConfig.Mounts = append(hostConfig.Mounts, hm) 1423 } 1424 1425 // set DNS search domains and extra hosts 1426 hostConfig.DNSSearch = driverConfig.DNSSearchDomains 1427 hostConfig.DNSOptions = driverConfig.DNSOptions 1428 hostConfig.ExtraHosts = driverConfig.ExtraHosts 1429 1430 hostConfig.IpcMode = driverConfig.IpcMode 1431 hostConfig.PidMode = driverConfig.PidMode 1432 hostConfig.UTSMode = driverConfig.UTSMode 1433 hostConfig.UsernsMode = driverConfig.UsernsMode 1434 hostConfig.SecurityOpt = driverConfig.SecurityOpt 1435 hostConfig.Sysctls = driverConfig.Sysctl 1436 hostConfig.Ulimits = driverConfig.Ulimit 1437 hostConfig.ReadonlyRootfs = driverConfig.ReadonlyRootfs 1438 1439 hostConfig.NetworkMode = driverConfig.NetworkMode 1440 if hostConfig.NetworkMode == "" { 1441 // docker default 1442 d.logger.Printf("[DEBUG] driver.docker: networking mode not specified; defaulting to %s", defaultNetworkMode) 1443 hostConfig.NetworkMode = defaultNetworkMode 1444 } 1445 1446 // Setup port mapping and exposed ports 1447 if len(task.Resources.Networks) == 0 { 1448 d.logger.Println("[DEBUG] driver.docker: No network interfaces are available") 1449 if len(driverConfig.PortMap) > 0 { 1450 return c, fmt.Errorf("Trying to map ports but no network interface is available") 1451 } 1452 } else { 1453 // TODO add support for more than one network 1454 network := task.Resources.Networks[0] 1455 publishedPorts := map[docker.Port][]docker.PortBinding{} 1456 exposedPorts := map[docker.Port]struct{}{} 1457 1458 for _, port := range network.ReservedPorts { 1459 // By default we will map the allocated port 1:1 to the container 1460 containerPortInt := port.Value 1461 1462 // If the user has mapped a port using port_map we'll change it here 1463 if mapped, ok := driverConfig.PortMap[port.Label]; ok { 1464 containerPortInt = mapped 1465 } 1466 1467 hostPortStr := strconv.Itoa(port.Value) 1468 containerPort := docker.Port(strconv.Itoa(containerPortInt)) 1469 1470 publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr) 1471 publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr) 1472 d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value) 1473 1474 exposedPorts[containerPort+"/tcp"] = struct{}{} 1475 exposedPorts[containerPort+"/udp"] = struct{}{} 1476 d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value) 1477 } 1478 1479 for _, port := range network.DynamicPorts { 1480 // By default we will map the allocated port 1:1 to the container 1481 containerPortInt := port.Value 1482 1483 // If the user has mapped a port using port_map we'll change it here 1484 if mapped, ok := driverConfig.PortMap[port.Label]; ok { 1485 containerPortInt = mapped 1486 } 1487 1488 hostPortStr := strconv.Itoa(port.Value) 1489 containerPort := docker.Port(strconv.Itoa(containerPortInt)) 1490 1491 publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr) 1492 publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr) 1493 d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt) 1494 1495 exposedPorts[containerPort+"/tcp"] = struct{}{} 1496 exposedPorts[containerPort+"/udp"] = struct{}{} 1497 d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort) 1498 } 1499 1500 hostConfig.PortBindings = publishedPorts 1501 config.ExposedPorts = exposedPorts 1502 } 1503 1504 parsedArgs := ctx.TaskEnv.ParseAndReplace(driverConfig.Args) 1505 1506 // If the user specified a custom command to run, we'll inject it here. 1507 if driverConfig.Command != "" { 1508 // Validate command 1509 if err := validateCommand(driverConfig.Command, "args"); err != nil { 1510 return c, err 1511 } 1512 1513 cmd := []string{driverConfig.Command} 1514 if len(driverConfig.Args) != 0 { 1515 cmd = append(cmd, parsedArgs...) 1516 } 1517 d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " ")) 1518 config.Cmd = cmd 1519 } else if len(driverConfig.Args) != 0 { 1520 config.Cmd = parsedArgs 1521 } 1522 1523 if len(driverConfig.Labels) > 0 { 1524 config.Labels = driverConfig.Labels 1525 d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels) 1526 } 1527 1528 config.Env = ctx.TaskEnv.List() 1529 1530 containerName := fmt.Sprintf("%s-%s", task.Name, d.DriverContext.allocID) 1531 d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName) 1532 1533 var networkingConfig *docker.NetworkingConfig 1534 if len(driverConfig.NetworkAliases) > 0 || driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" { 1535 networkingConfig = &docker.NetworkingConfig{ 1536 EndpointsConfig: map[string]*docker.EndpointConfig{ 1537 hostConfig.NetworkMode: {}, 1538 }, 1539 } 1540 } 1541 1542 if len(driverConfig.NetworkAliases) > 0 { 1543 networkingConfig.EndpointsConfig[hostConfig.NetworkMode].Aliases = driverConfig.NetworkAliases 1544 d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with network aliases: %v", 1545 hostConfig.NetworkMode, strings.Join(driverConfig.NetworkAliases, ", ")) 1546 } 1547 1548 if driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" { 1549 networkingConfig.EndpointsConfig[hostConfig.NetworkMode].IPAMConfig = &docker.EndpointIPAMConfig{ 1550 IPv4Address: driverConfig.IPv4Address, 1551 IPv6Address: driverConfig.IPv6Address, 1552 } 1553 d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with ipv4: %q and ipv6: %q", 1554 hostConfig.NetworkMode, driverConfig.IPv4Address, driverConfig.IPv6Address) 1555 } 1556 1557 if driverConfig.MacAddress != "" { 1558 config.MacAddress = driverConfig.MacAddress 1559 d.logger.Printf("[DEBUG] driver.docker: using pinned mac address: %q", config.MacAddress) 1560 } 1561 1562 return docker.CreateContainerOptions{ 1563 Name: containerName, 1564 Config: config, 1565 HostConfig: hostConfig, 1566 NetworkingConfig: networkingConfig, 1567 }, nil 1568 } 1569 1570 func (d *DockerDriver) Periodic() (bool, time.Duration) { 1571 return true, 15 * time.Second 1572 } 1573 1574 // createImage creates a docker image either by pulling it from a registry or by 1575 // loading it from the file system 1576 func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir *allocdir.TaskDir) (string, error) { 1577 image := driverConfig.ImageName 1578 repo, tag := parseDockerImage(image) 1579 1580 coordinator, callerID := d.getDockerCoordinator(client) 1581 1582 // We're going to check whether the image is already downloaded. If the tag 1583 // is "latest", or ForcePull is set, we have to check for a new version every time so we don't 1584 // bother to check and cache the id here. We'll download first, then cache. 1585 if driverConfig.ForcePull { 1586 d.logger.Printf("[DEBUG] driver.docker: force pull image '%s' instead of inspecting local", dockerImageRef(repo, tag)) 1587 } else if tag != "latest" { 1588 if dockerImage, _ := client.InspectImage(image); dockerImage != nil { 1589 // Image exists so just increment its reference count 1590 coordinator.IncrementImageReference(dockerImage.ID, image, callerID) 1591 return dockerImage.ID, nil 1592 } 1593 } 1594 1595 // Load the image if specified 1596 if driverConfig.LoadImage != "" { 1597 return d.loadImage(driverConfig, client, taskDir) 1598 } 1599 1600 // Download the image 1601 return d.pullImage(driverConfig, client, repo, tag) 1602 } 1603 1604 // pullImage creates an image by pulling it from a docker registry 1605 func (d *DockerDriver) pullImage(driverConfig *DockerDriverConfig, client *docker.Client, repo, tag string) (id string, err error) { 1606 authOptions, err := d.resolveRegistryAuthentication(driverConfig, repo) 1607 if err != nil { 1608 if d.driverConfig.AuthSoftFail { 1609 d.logger.Printf("[WARN] Failed to find docker auth for repo %q: %v", repo, err) 1610 } else { 1611 return "", fmt.Errorf("Failed to find docker auth for repo %q: %v", repo, err) 1612 } 1613 } 1614 1615 if authIsEmpty(authOptions) { 1616 d.logger.Printf("[DEBUG] driver.docker: did not find docker auth for repo %q", repo) 1617 } 1618 1619 d.emitEvent("Downloading image %s", dockerImageRef(repo, tag)) 1620 coordinator, callerID := d.getDockerCoordinator(client) 1621 1622 return coordinator.PullImage(driverConfig.ImageName, authOptions, callerID, d.emitEvent) 1623 } 1624 1625 // authBackend encapsulates a function that resolves registry credentials. 1626 type authBackend func(string) (*docker.AuthConfiguration, error) 1627 1628 // resolveRegistryAuthentication attempts to retrieve auth credentials for the 1629 // repo, trying all authentication-backends possible. 1630 func (d *DockerDriver) resolveRegistryAuthentication(driverConfig *DockerDriverConfig, repo string) (*docker.AuthConfiguration, error) { 1631 return firstValidAuth(repo, []authBackend{ 1632 authFromTaskConfig(driverConfig), 1633 authFromDockerConfig(d.config.Read("docker.auth.config")), 1634 authFromHelper(d.config.Read("docker.auth.helper")), 1635 }) 1636 } 1637 1638 // loadImage creates an image by loading it from the file system 1639 func (d *DockerDriver) loadImage(driverConfig *DockerDriverConfig, client *docker.Client, 1640 taskDir *allocdir.TaskDir) (id string, err error) { 1641 1642 archive := filepath.Join(taskDir.LocalDir, driverConfig.LoadImage) 1643 d.logger.Printf("[DEBUG] driver.docker: loading image from: %v", archive) 1644 1645 f, err := os.Open(archive) 1646 if err != nil { 1647 return "", fmt.Errorf("unable to open image archive: %v", err) 1648 } 1649 1650 if err := client.LoadImage(docker.LoadImageOptions{InputStream: f}); err != nil { 1651 return "", err 1652 } 1653 f.Close() 1654 1655 dockerImage, err := client.InspectImage(driverConfig.ImageName) 1656 if err != nil { 1657 return "", recoverableErrTimeouts(err) 1658 } 1659 1660 coordinator, callerID := d.getDockerCoordinator(client) 1661 coordinator.IncrementImageReference(dockerImage.ID, driverConfig.ImageName, callerID) 1662 return dockerImage.ID, nil 1663 } 1664 1665 // createContainer creates the container given the passed configuration. It 1666 // attempts to handle any transient Docker errors. 1667 func (d *DockerDriver) createContainer(client createContainerClient, config docker.CreateContainerOptions) (*docker.Container, error) { 1668 // Create a container 1669 attempted := 0 1670 CREATE: 1671 container, createErr := client.CreateContainer(config) 1672 if createErr == nil { 1673 return container, nil 1674 } 1675 1676 d.logger.Printf("[DEBUG] driver.docker: failed to create container %q from image %q (ID: %q) (attempt %d): %v", 1677 config.Name, d.driverConfig.ImageName, d.imageID, attempted+1, createErr) 1678 1679 // Volume management tools like Portworx may not have detached a volume 1680 // from a previous node before Nomad started a task replacement task. 1681 // Treat these errors as recoverable so we retry. 1682 if strings.Contains(strings.ToLower(createErr.Error()), "volume is attached on another node") { 1683 return nil, structs.NewRecoverableError(createErr, true) 1684 } 1685 1686 // If the container already exists determine whether it's already 1687 // running or if it's dead and needs to be recreated. 1688 if strings.Contains(strings.ToLower(createErr.Error()), "container already exists") { 1689 containers, err := client.ListContainers(docker.ListContainersOptions{ 1690 All: true, 1691 }) 1692 if err != nil { 1693 d.logger.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name) 1694 return nil, recoverableErrTimeouts(fmt.Errorf("Failed to query list of containers: %s", err)) 1695 } 1696 1697 // Delete matching containers 1698 // Adding a / infront of the container name since Docker returns the 1699 // container names with a / pre-pended to the Nomad generated container names 1700 containerName := "/" + config.Name 1701 d.logger.Printf("[DEBUG] driver.docker: searching for container name %q to purge", containerName) 1702 for _, shimContainer := range containers { 1703 d.logger.Printf("[DEBUG] driver.docker: listed container %+v", shimContainer.Names) 1704 found := false 1705 for _, name := range shimContainer.Names { 1706 if name == containerName { 1707 d.logger.Printf("[DEBUG] driver.docker: Found container %v: %v", containerName, shimContainer.ID) 1708 found = true 1709 break 1710 } 1711 } 1712 1713 if !found { 1714 continue 1715 } 1716 1717 // Inspect the container and if the container isn't dead then return 1718 // the container 1719 container, err := client.InspectContainer(shimContainer.ID) 1720 if err != nil { 1721 err = fmt.Errorf("Failed to inspect container %s: %s", shimContainer.ID, err) 1722 1723 // This error is always recoverable as it could 1724 // be caused by races between listing 1725 // containers and this container being removed. 1726 // See #2802 1727 return nil, structs.NewRecoverableError(err, true) 1728 } 1729 if container != nil && container.State.Running { 1730 return container, nil 1731 } 1732 1733 err = client.RemoveContainer(docker.RemoveContainerOptions{ 1734 ID: container.ID, 1735 Force: true, 1736 }) 1737 if err != nil { 1738 d.logger.Printf("[ERR] driver.docker: failed to purge container %s", container.ID) 1739 return nil, recoverableErrTimeouts(fmt.Errorf("Failed to purge container %s: %s", container.ID, err)) 1740 } else if err == nil { 1741 d.logger.Printf("[INFO] driver.docker: purged container %s", container.ID) 1742 } 1743 } 1744 1745 if attempted < 5 { 1746 attempted++ 1747 time.Sleep(1 * time.Second) 1748 goto CREATE 1749 } 1750 } else if strings.Contains(strings.ToLower(createErr.Error()), "no such image") { 1751 // There is still a very small chance this is possible even with the 1752 // coordinator so retry. 1753 return nil, structs.NewRecoverableError(createErr, true) 1754 } 1755 1756 return nil, recoverableErrTimeouts(createErr) 1757 } 1758 1759 // startContainer starts the passed container. It attempts to handle any 1760 // transient Docker errors. 1761 func (d *DockerDriver) startContainer(c *docker.Container) error { 1762 // Start a container 1763 attempted := 0 1764 START: 1765 startErr := client.StartContainer(c.ID, c.HostConfig) 1766 if startErr == nil { 1767 return nil 1768 } 1769 1770 d.logger.Printf("[DEBUG] driver.docker: failed to start container %q (attempt %d): %v", c.ID, attempted+1, startErr) 1771 1772 // If it is a 500 error it is likely we can retry and be successful 1773 if strings.Contains(startErr.Error(), "API error (500)") { 1774 if attempted < 5 { 1775 attempted++ 1776 time.Sleep(1 * time.Second) 1777 goto START 1778 } 1779 return structs.NewRecoverableError(startErr, true) 1780 } 1781 1782 return recoverableErrTimeouts(startErr) 1783 } 1784 1785 func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) { 1786 // Split the handle 1787 pidBytes := []byte(strings.TrimPrefix(handleID, "DOCKER:")) 1788 pid := &dockerPID{} 1789 if err := json.Unmarshal(pidBytes, pid); err != nil { 1790 return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err) 1791 } 1792 d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", pid.ContainerID) 1793 d.logger.Printf("[DEBUG] driver.docker: re-attached to handle: %s", handleID) 1794 pluginConfig := &plugin.ClientConfig{ 1795 Reattach: pid.PluginConfig.PluginConfig(), 1796 } 1797 1798 client, waitClient, err := d.dockerClients() 1799 if err != nil { 1800 return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err) 1801 } 1802 1803 // Look for a running container with this ID 1804 containers, err := client.ListContainers(docker.ListContainersOptions{ 1805 Filters: map[string][]string{ 1806 "id": {pid.ContainerID}, 1807 }, 1808 }) 1809 if err != nil { 1810 return nil, fmt.Errorf("Failed to query for container %s: %v", pid.ContainerID, err) 1811 } 1812 1813 found := false 1814 for _, container := range containers { 1815 if container.ID == pid.ContainerID { 1816 found = true 1817 } 1818 } 1819 if !found { 1820 return nil, fmt.Errorf("Failed to find container %s", pid.ContainerID) 1821 } 1822 exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput) 1823 if err != nil { 1824 d.logger.Printf("[INFO] driver.docker: couldn't re-attach to the plugin process: %v", err) 1825 d.logger.Printf("[DEBUG] driver.docker: stopping container %q", pid.ContainerID) 1826 if e := client.StopContainer(pid.ContainerID, uint(pid.KillTimeout.Seconds())); e != nil { 1827 d.logger.Printf("[DEBUG] driver.docker: couldn't stop container: %v", e) 1828 } 1829 return nil, err 1830 } 1831 1832 ver, _ := exec.Version() 1833 d.logger.Printf("[DEBUG] driver.docker: version of executor: %v", ver.Version) 1834 1835 // Increment the reference count since we successfully attached to this 1836 // container 1837 coordinator, callerID := d.getDockerCoordinator(client) 1838 coordinator.IncrementImageReference(pid.ImageID, pid.Image, callerID) 1839 1840 // Return a driver handle 1841 h := &DockerHandle{ 1842 client: client, 1843 waitClient: waitClient, 1844 executor: exec, 1845 pluginClient: pluginClient, 1846 logger: d.logger, 1847 jobName: d.DriverContext.jobName, 1848 taskGroupName: d.DriverContext.taskGroupName, 1849 taskName: d.DriverContext.taskName, 1850 Image: pid.Image, 1851 ImageID: pid.ImageID, 1852 containerID: pid.ContainerID, 1853 version: pid.Version, 1854 killTimeout: pid.KillTimeout, 1855 maxKillTimeout: pid.MaxKillTimeout, 1856 doneCh: make(chan bool), 1857 waitCh: make(chan *dstructs.WaitResult, 1), 1858 } 1859 go h.collectStats() 1860 go h.run() 1861 return h, nil 1862 } 1863 1864 func (h *DockerHandle) ID() string { 1865 // Return a handle to the PID 1866 pid := dockerPID{ 1867 Version: h.version, 1868 ContainerID: h.containerID, 1869 Image: h.Image, 1870 ImageID: h.ImageID, 1871 KillTimeout: h.killTimeout, 1872 MaxKillTimeout: h.maxKillTimeout, 1873 PluginConfig: NewPluginReattachConfig(h.pluginClient.ReattachConfig()), 1874 } 1875 data, err := json.Marshal(pid) 1876 if err != nil { 1877 h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s", err) 1878 } 1879 return fmt.Sprintf("DOCKER:%s", string(data)) 1880 } 1881 1882 func (h *DockerHandle) ContainerID() string { 1883 return h.containerID 1884 } 1885 1886 func (h *DockerHandle) WaitCh() chan *dstructs.WaitResult { 1887 return h.waitCh 1888 } 1889 1890 func (h *DockerHandle) Update(task *structs.Task) error { 1891 // Store the updated kill timeout. 1892 h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout) 1893 if err := h.executor.UpdateTask(task); err != nil { 1894 h.logger.Printf("[DEBUG] driver.docker: failed to update log config: %v", err) 1895 } 1896 1897 // Update is not possible 1898 return nil 1899 } 1900 1901 func (h *DockerHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) { 1902 fullCmd := make([]string, len(args)+1) 1903 fullCmd[0] = cmd 1904 copy(fullCmd[1:], args) 1905 createExecOpts := docker.CreateExecOptions{ 1906 AttachStdin: false, 1907 AttachStdout: true, 1908 AttachStderr: true, 1909 Tty: false, 1910 Cmd: fullCmd, 1911 Container: h.containerID, 1912 Context: ctx, 1913 } 1914 exec, err := h.client.CreateExec(createExecOpts) 1915 if err != nil { 1916 return nil, 0, err 1917 } 1918 1919 output, _ := circbuf.NewBuffer(int64(dstructs.CheckBufSize)) 1920 startOpts := docker.StartExecOptions{ 1921 Detach: false, 1922 Tty: false, 1923 OutputStream: output, 1924 ErrorStream: output, 1925 Context: ctx, 1926 } 1927 if err := client.StartExec(exec.ID, startOpts); err != nil { 1928 return nil, 0, err 1929 } 1930 res, err := client.InspectExec(exec.ID) 1931 if err != nil { 1932 return output.Bytes(), 0, err 1933 } 1934 return output.Bytes(), res.ExitCode, nil 1935 } 1936 1937 func (h *DockerHandle) Signal(s os.Signal) error { 1938 // Convert types 1939 sysSig, ok := s.(syscall.Signal) 1940 if !ok { 1941 return fmt.Errorf("Failed to determine signal number") 1942 } 1943 1944 // TODO When we expose signals we will need a mapping layer that converts 1945 // MacOS signals to the correct signal number for docker. Or we change the 1946 // interface to take a signal string and leave it up to driver to map? 1947 1948 dockerSignal := docker.Signal(sysSig) 1949 opts := docker.KillContainerOptions{ 1950 ID: h.containerID, 1951 Signal: dockerSignal, 1952 } 1953 return h.client.KillContainer(opts) 1954 1955 } 1956 1957 // Kill is used to terminate the task. This uses `docker stop -t killTimeout` 1958 func (h *DockerHandle) Kill() error { 1959 // Stop the container 1960 err := h.waitClient.StopContainer(h.containerID, uint(h.killTimeout.Seconds())) 1961 if err != nil { 1962 h.executor.Exit() 1963 h.pluginClient.Kill() 1964 1965 // Container has already been removed. 1966 if strings.Contains(err.Error(), NoSuchContainerError) { 1967 h.logger.Printf("[DEBUG] driver.docker: attempted to stop nonexistent container %s", h.containerID) 1968 return nil 1969 } 1970 h.logger.Printf("[ERR] driver.docker: failed to stop container %s: %v", h.containerID, err) 1971 return fmt.Errorf("Failed to stop container %s: %s", h.containerID, err) 1972 } 1973 h.logger.Printf("[INFO] driver.docker: stopped container %s", h.containerID) 1974 return nil 1975 } 1976 1977 func (h *DockerHandle) Stats() (*cstructs.TaskResourceUsage, error) { 1978 h.resourceUsageLock.RLock() 1979 defer h.resourceUsageLock.RUnlock() 1980 var err error 1981 if h.resourceUsage == nil { 1982 err = fmt.Errorf("stats collection hasn't started yet") 1983 } 1984 return h.resourceUsage, err 1985 } 1986 1987 func (h *DockerHandle) run() { 1988 // Wait for it... 1989 exitCode, werr := h.waitClient.WaitContainer(h.containerID) 1990 if werr != nil { 1991 h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated", h.containerID) 1992 } 1993 1994 if exitCode != 0 { 1995 werr = fmt.Errorf("Docker container exited with non-zero exit code: %d", exitCode) 1996 } 1997 1998 container, ierr := h.waitClient.InspectContainer(h.containerID) 1999 if ierr != nil { 2000 h.logger.Printf("[ERR] driver.docker: failed to inspect container %s: %v", h.containerID, ierr) 2001 } else if container.State.OOMKilled { 2002 werr = fmt.Errorf("OOM Killed") 2003 labels := []metrics.Label{ 2004 { 2005 Name: "job", 2006 Value: h.jobName, 2007 }, 2008 { 2009 Name: "task_group", 2010 Value: h.taskGroupName, 2011 }, 2012 { 2013 Name: "task", 2014 Value: h.taskName, 2015 }, 2016 } 2017 metrics.IncrCounterWithLabels([]string{"driver", "docker", "oom"}, 1, labels) 2018 } 2019 2020 close(h.doneCh) 2021 2022 // Shutdown the syslog collector 2023 if err := h.executor.Exit(); err != nil { 2024 h.logger.Printf("[ERR] driver.docker: failed to kill the syslog collector: %v", err) 2025 } 2026 h.pluginClient.Kill() 2027 2028 // Stop the container just incase the docker daemon's wait returned 2029 // incorrectly 2030 if err := h.client.StopContainer(h.containerID, 0); err != nil { 2031 _, noSuchContainer := err.(*docker.NoSuchContainer) 2032 _, containerNotRunning := err.(*docker.ContainerNotRunning) 2033 if !containerNotRunning && !noSuchContainer { 2034 h.logger.Printf("[ERR] driver.docker: error stopping container: %v", err) 2035 } 2036 } 2037 2038 // Remove the container 2039 if h.removeContainerOnExit == true { 2040 if err := h.client.RemoveContainer(docker.RemoveContainerOptions{ID: h.containerID, RemoveVolumes: true, Force: true}); err != nil { 2041 h.logger.Printf("[ERR] driver.docker: error removing container: %v", err) 2042 } 2043 } else { 2044 h.logger.Printf("[DEBUG] driver.docker: not removing container %v because of config", h.containerID) 2045 } 2046 2047 // Send the results 2048 h.waitCh <- dstructs.NewWaitResult(exitCode, 0, werr) 2049 close(h.waitCh) 2050 } 2051 2052 // collectStats starts collecting resource usage stats of a docker container 2053 func (h *DockerHandle) collectStats() { 2054 statsCh := make(chan *docker.Stats) 2055 statsOpts := docker.StatsOptions{ID: h.containerID, Done: h.doneCh, Stats: statsCh, Stream: true} 2056 go func() { 2057 //TODO handle Stats error 2058 if err := h.waitClient.Stats(statsOpts); err != nil { 2059 h.logger.Printf("[DEBUG] driver.docker: error collecting stats from container %s: %v", h.containerID, err) 2060 } 2061 }() 2062 numCores := runtime.NumCPU() 2063 for { 2064 select { 2065 case s := <-statsCh: 2066 if s != nil { 2067 ms := &cstructs.MemoryStats{ 2068 RSS: s.MemoryStats.Stats.Rss, 2069 Cache: s.MemoryStats.Stats.Cache, 2070 Swap: s.MemoryStats.Stats.Swap, 2071 MaxUsage: s.MemoryStats.MaxUsage, 2072 Measured: DockerMeasuredMemStats, 2073 } 2074 2075 cs := &cstructs.CpuStats{ 2076 ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods, 2077 ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime, 2078 Measured: DockerMeasuredCpuStats, 2079 } 2080 2081 // Calculate percentage 2082 cs.Percent = calculatePercent( 2083 s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, 2084 s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, numCores) 2085 cs.SystemMode = calculatePercent( 2086 s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode, 2087 s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, numCores) 2088 cs.UserMode = calculatePercent( 2089 s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode, 2090 s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, numCores) 2091 cs.TotalTicks = (cs.Percent / 100) * shelpers.TotalTicksAvailable() / float64(numCores) 2092 2093 h.resourceUsageLock.Lock() 2094 h.resourceUsage = &cstructs.TaskResourceUsage{ 2095 ResourceUsage: &cstructs.ResourceUsage{ 2096 MemoryStats: ms, 2097 CpuStats: cs, 2098 }, 2099 Timestamp: s.Read.UTC().UnixNano(), 2100 } 2101 h.resourceUsageLock.Unlock() 2102 } 2103 case <-h.doneCh: 2104 return 2105 } 2106 } 2107 } 2108 2109 func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 { 2110 numerator := newSample - oldSample 2111 denom := newTotal - oldTotal 2112 if numerator <= 0 || denom <= 0 { 2113 return 0.0 2114 } 2115 2116 return (float64(numerator) / float64(denom)) * float64(cores) * 100.0 2117 } 2118 2119 // loadDockerConfig loads the docker config at the specified path, returning an 2120 // error if it couldn't be read. 2121 func loadDockerConfig(file string) (*configfile.ConfigFile, error) { 2122 f, err := os.Open(file) 2123 if err != nil { 2124 return nil, fmt.Errorf("Failed to open auth config file: %v, error: %v", file, err) 2125 } 2126 defer f.Close() 2127 2128 cfile := new(configfile.ConfigFile) 2129 if err = cfile.LoadFromReader(f); err != nil { 2130 return nil, fmt.Errorf("Failed to parse auth config file: %v", err) 2131 } 2132 return cfile, nil 2133 } 2134 2135 // parseRepositoryInfo takes a repo and returns the Docker RepositoryInfo. This 2136 // is useful for interacting with a Docker config object. 2137 func parseRepositoryInfo(repo string) (*registry.RepositoryInfo, error) { 2138 name, err := reference.ParseNamed(repo) 2139 if err != nil { 2140 return nil, fmt.Errorf("Failed to parse named repo %q: %v", repo, err) 2141 } 2142 2143 repoInfo, err := registry.ParseRepositoryInfo(name) 2144 if err != nil { 2145 return nil, fmt.Errorf("Failed to parse repository: %v", err) 2146 } 2147 2148 return repoInfo, nil 2149 } 2150 2151 // firstValidAuth tries a list of auth backends, returning first error or AuthConfiguration 2152 func firstValidAuth(repo string, backends []authBackend) (*docker.AuthConfiguration, error) { 2153 for _, backend := range backends { 2154 auth, err := backend(repo) 2155 if auth != nil || err != nil { 2156 return auth, err 2157 } 2158 } 2159 return nil, nil 2160 } 2161 2162 // authFromTaskConfig generates an authBackend for any auth given in the task-configuration 2163 func authFromTaskConfig(driverConfig *DockerDriverConfig) authBackend { 2164 return func(string) (*docker.AuthConfiguration, error) { 2165 if len(driverConfig.Auth) == 0 { 2166 return nil, nil 2167 } 2168 auth := driverConfig.Auth[0] 2169 return &docker.AuthConfiguration{ 2170 Username: auth.Username, 2171 Password: auth.Password, 2172 Email: auth.Email, 2173 ServerAddress: auth.ServerAddress, 2174 }, nil 2175 } 2176 } 2177 2178 // authFromDockerConfig generate an authBackend for a dockercfg-compatible file. 2179 // The authBacken can either be from explicit auth definitions or via credential 2180 // helpers 2181 func authFromDockerConfig(file string) authBackend { 2182 return func(repo string) (*docker.AuthConfiguration, error) { 2183 if file == "" { 2184 return nil, nil 2185 } 2186 repoInfo, err := parseRepositoryInfo(repo) 2187 if err != nil { 2188 return nil, err 2189 } 2190 2191 cfile, err := loadDockerConfig(file) 2192 if err != nil { 2193 return nil, err 2194 } 2195 2196 return firstValidAuth(repo, []authBackend{ 2197 func(string) (*docker.AuthConfiguration, error) { 2198 dockerAuthConfig := registry.ResolveAuthConfig(cfile.AuthConfigs, repoInfo.Index) 2199 auth := &docker.AuthConfiguration{ 2200 Username: dockerAuthConfig.Username, 2201 Password: dockerAuthConfig.Password, 2202 Email: dockerAuthConfig.Email, 2203 ServerAddress: dockerAuthConfig.ServerAddress, 2204 } 2205 if authIsEmpty(auth) { 2206 return nil, nil 2207 } 2208 return auth, nil 2209 }, 2210 authFromHelper(cfile.CredentialHelpers[registry.GetAuthConfigKey(repoInfo.Index)]), 2211 authFromHelper(cfile.CredentialsStore), 2212 }) 2213 } 2214 } 2215 2216 // authFromHelper generates an authBackend for a docker-credentials-helper; 2217 // A script taking the requested domain on input, outputting JSON with 2218 // "Username" and "Secret" 2219 func authFromHelper(helperName string) authBackend { 2220 return func(repo string) (*docker.AuthConfiguration, error) { 2221 if helperName == "" { 2222 return nil, nil 2223 } 2224 helper := dockerAuthHelperPrefix + helperName 2225 cmd := exec.Command(helper, "get") 2226 2227 repoParsed, err := reference.ParseNamed(repo) 2228 if err != nil { 2229 return nil, err 2230 } 2231 2232 // Ensure that the HTTPs prefix exists 2233 repoAddr := fmt.Sprintf("https://%s", repoParsed.Hostname()) 2234 2235 cmd.Stdin = strings.NewReader(repoAddr) 2236 output, err := cmd.Output() 2237 if err != nil { 2238 switch err.(type) { 2239 default: 2240 return nil, err 2241 case *exec.ExitError: 2242 return nil, fmt.Errorf("%s with input %q failed with stderr: %s", helper, repo, output) 2243 } 2244 } 2245 2246 var response map[string]string 2247 if err := json.Unmarshal(output, &response); err != nil { 2248 return nil, err 2249 } 2250 2251 auth := &docker.AuthConfiguration{ 2252 Username: response["Username"], 2253 Password: response["Secret"], 2254 } 2255 2256 if authIsEmpty(auth) { 2257 return nil, nil 2258 } 2259 return auth, nil 2260 } 2261 } 2262 2263 // authIsEmpty returns if auth is nil or an empty structure 2264 func authIsEmpty(auth *docker.AuthConfiguration) bool { 2265 if auth == nil { 2266 return false 2267 } 2268 return auth.Username == "" && 2269 auth.Password == "" && 2270 auth.Email == "" && 2271 auth.ServerAddress == "" 2272 } 2273 2274 // createContainerClient is the subset of Docker Client methods used by the 2275 // createContainer method to ease testing subtle error conditions. 2276 type createContainerClient interface { 2277 CreateContainer(docker.CreateContainerOptions) (*docker.Container, error) 2278 InspectContainer(id string) (*docker.Container, error) 2279 ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error) 2280 RemoveContainer(opts docker.RemoveContainerOptions) error 2281 } 2282 2283 func parseDockerImage(image string) (repo, tag string) { 2284 repo, tag = docker.ParseRepositoryTag(image) 2285 if tag != "" { 2286 return repo, tag 2287 } 2288 if i := strings.IndexRune(image, '@'); i > -1 { // Has digest (@sha256:...) 2289 // when pulling images with a digest, the repository contains the sha hash, and the tag is empty 2290 // see: https://github.com/fsouza/go-dockerclient/blob/master/image_test.go#L471 2291 repo = image 2292 } else { 2293 tag = "latest" 2294 } 2295 return repo, tag 2296 } 2297 2298 func dockerImageRef(repo string, tag string) string { 2299 if tag == "" { 2300 return repo 2301 } 2302 return fmt.Sprintf("%s:%s", repo, tag) 2303 }