github.com/anuvu/nomad@v0.8.7-atom1/client/driver/docker.go (about)

     1  package driver
     2  
     3  import (
     4  	"context"
     5  	"encoding/json"
     6  	"fmt"
     7  	"log"
     8  	"net"
     9  	"os"
    10  	"os/exec"
    11  	"path/filepath"
    12  	"runtime"
    13  	"strconv"
    14  	"strings"
    15  	"sync"
    16  	"syscall"
    17  	"time"
    18  
    19  	"github.com/armon/circbuf"
    20  	docker "github.com/fsouza/go-dockerclient"
    21  
    22  	"github.com/docker/docker/cli/config/configfile"
    23  	"github.com/docker/docker/reference"
    24  	"github.com/docker/docker/registry"
    25  
    26  	metrics "github.com/armon/go-metrics"
    27  	multierror "github.com/hashicorp/go-multierror"
    28  	plugin "github.com/hashicorp/go-plugin"
    29  	"github.com/hashicorp/nomad/client/allocdir"
    30  	"github.com/hashicorp/nomad/client/driver/env"
    31  	"github.com/hashicorp/nomad/client/driver/executor"
    32  	dstructs "github.com/hashicorp/nomad/client/driver/structs"
    33  	cstructs "github.com/hashicorp/nomad/client/structs"
    34  	"github.com/hashicorp/nomad/helper"
    35  	"github.com/hashicorp/nomad/helper/fields"
    36  	shelpers "github.com/hashicorp/nomad/helper/stats"
    37  	"github.com/hashicorp/nomad/nomad/structs"
    38  	"github.com/mitchellh/mapstructure"
    39  )
    40  
    41  var (
    42  	// createClientsLock is a lock that protects reading/writing global client
    43  	// variables
    44  	createClientsLock sync.Mutex
    45  
    46  	// client is a docker client with a timeout of 5 minutes. This is for doing
    47  	// all operations with the docker daemon besides which are not long running
    48  	// such as creating, killing containers, etc.
    49  	client *docker.Client
    50  
    51  	// waitClient is a docker client with no timeouts. This is used for long
    52  	// running operations such as waiting on containers and collect stats
    53  	waitClient *docker.Client
    54  
    55  	// healthCheckClient is a docker client with a timeout of 1 minute. This is
    56  	// necessary to have a shorter timeout than other API or fingerprint calls
    57  	healthCheckClient *docker.Client
    58  
    59  	// The statistics the Docker driver exposes
    60  	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"}
    61  	DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
    62  
    63  	// recoverableErrTimeouts returns a recoverable error if the error was due
    64  	// to timeouts
    65  	recoverableErrTimeouts = func(err error) error {
    66  		r := false
    67  		if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") ||
    68  			strings.Contains(err.Error(), "EOF") {
    69  			r = true
    70  		}
    71  		return structs.NewRecoverableError(err, r)
    72  	}
    73  )
    74  
    75  const (
    76  	// NoSuchContainerError is returned by the docker daemon if the container
    77  	// does not exist.
    78  	NoSuchContainerError = "No such container"
    79  
    80  	// The key populated in Node Attributes to indicate presence of the Docker
    81  	// driver
    82  	dockerDriverAttr = "driver.docker"
    83  
    84  	// dockerSELinuxLabelConfigOption is the key for configuring the
    85  	// SELinux label for binds.
    86  	dockerSELinuxLabelConfigOption = "docker.volumes.selinuxlabel"
    87  
    88  	// dockerVolumesConfigOption is the key for enabling the use of custom
    89  	// bind volumes to arbitrary host paths.
    90  	dockerVolumesConfigOption  = "docker.volumes.enabled"
    91  	dockerVolumesConfigDefault = true
    92  
    93  	// dockerPrivilegedConfigOption is the key for running containers in
    94  	// Docker's privileged mode.
    95  	dockerPrivilegedConfigOption = "docker.privileged.enabled"
    96  
    97  	// dockerCleanupImageConfigOption is the key for whether or not to
    98  	// cleanup images after the task exits.
    99  	dockerCleanupImageConfigOption  = "docker.cleanup.image"
   100  	dockerCleanupImageConfigDefault = true
   101  
   102  	// dockerPullTimeoutConfigOption is the key for setting an images pull
   103  	// timeout
   104  	dockerImageRemoveDelayConfigOption  = "docker.cleanup.image.delay"
   105  	dockerImageRemoveDelayConfigDefault = 3 * time.Minute
   106  
   107  	// dockerCapsWhitelistConfigOption is the key for setting the list of
   108  	// allowed Linux capabilities
   109  	dockerCapsWhitelistConfigOption  = "docker.caps.whitelist"
   110  	dockerCapsWhitelistConfigDefault = dockerBasicCaps
   111  
   112  	// dockerTimeout is the length of time a request can be outstanding before
   113  	// it is timed out.
   114  	dockerTimeout = 5 * time.Minute
   115  
   116  	// dockerHealthCheckTimeout is the length of time a request for a health
   117  	// check client can be outstanding before it is timed out.
   118  	dockerHealthCheckTimeout = 1 * time.Minute
   119  
   120  	// dockerImageResKey is the CreatedResources key for docker images
   121  	dockerImageResKey = "image"
   122  
   123  	// dockerAuthHelperPrefix is the prefix to attach to the credential helper
   124  	// and should be found in the $PATH. Example: ${prefix-}${helper-name}
   125  	dockerAuthHelperPrefix = "docker-credential-"
   126  
   127  	// dockerBasicCaps is comma-separated list of Linux capabilities that are
   128  	// allowed by docker by default, as documented in
   129  	// https://docs.docker.com/engine/reference/run/#block-io-bandwidth-blkio-constraint
   130  	dockerBasicCaps = "CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID," +
   131  		"SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE"
   132  
   133  	// This is cpu.cfs_period_us: the length of a period.
   134  	// The default values is 100 milliseconds (ms) represented in microseconds (us).
   135  	// Below is the documentation:
   136  	// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
   137  	// https://docs.docker.com/engine/api/v1.35/#
   138  	defaultCFSPeriodUS = 100000
   139  
   140  	// dockerCleanupContainerConfigOption is the key for whether or not to
   141  	// remove containers after the task exits.
   142  	dockerCleanupContainerConfigOption  = "docker.cleanup.container"
   143  	dockerCleanupContainerConfigDefault = true
   144  )
   145  
   146  type DockerDriver struct {
   147  	DriverContext
   148  
   149  	driverConfig *DockerDriverConfig
   150  	imageID      string
   151  
   152  	// A tri-state boolean to know if the fingerprinting has happened and
   153  	// whether it has been successful
   154  	fingerprintSuccess *bool
   155  }
   156  
   157  type DockerDriverAuth struct {
   158  	Username      string `mapstructure:"username"`       // username for the registry
   159  	Password      string `mapstructure:"password"`       // password to access the registry
   160  	Email         string `mapstructure:"email"`          // email address of the user who is allowed to access the registry
   161  	ServerAddress string `mapstructure:"server_address"` // server address of the registry
   162  }
   163  
   164  type DockerLoggingOpts struct {
   165  	Type      string              `mapstructure:"type"`
   166  	ConfigRaw []map[string]string `mapstructure:"config"`
   167  	Config    map[string]string   `mapstructure:"-"`
   168  }
   169  
   170  type DockerMount struct {
   171  	Type          string                 `mapstructure:"type"`
   172  	Target        string                 `mapstructure:"target"`
   173  	Source        string                 `mapstructure:"source"`
   174  	ReadOnly      bool                   `mapstructure:"readonly"`
   175  	BindOptions   []*DockerBindOptions   `mapstructure:"bind_options"`
   176  	VolumeOptions []*DockerVolumeOptions `mapstructure:"volume_options"`
   177  }
   178  
   179  type DockerDevice struct {
   180  	HostPath          string `mapstructure:"host_path"`
   181  	ContainerPath     string `mapstructure:"container_path"`
   182  	CgroupPermissions string `mapstructure:"cgroup_permissions"`
   183  }
   184  
   185  type DockerBindOptions struct {
   186  	Propagation string `mapstructure:"propagation"`
   187  }
   188  
   189  type DockerVolumeOptions struct {
   190  	NoCopy       bool                       `mapstructure:"no_copy"`
   191  	Labels       []map[string]string        `mapstructure:"labels"`
   192  	DriverConfig []DockerVolumeDriverConfig `mapstructure:"driver_config"`
   193  }
   194  
   195  // VolumeDriverConfig holds a map of volume driver specific options
   196  type DockerVolumeDriverConfig struct {
   197  	Name    string              `mapstructure:"name"`
   198  	Options []map[string]string `mapstructure:"options"`
   199  }
   200  
   201  // DockerDriverConfig defines the user specified config block in a jobspec
   202  type DockerDriverConfig struct {
   203  	ImageName            string              `mapstructure:"image"`                  // Container's Image Name
   204  	LoadImage            string              `mapstructure:"load"`                   // LoadImage is a path to an image archive file
   205  	Command              string              `mapstructure:"command"`                // The Command to run when the container starts up
   206  	Args                 []string            `mapstructure:"args"`                   // The arguments to the Command
   207  	Entrypoint           []string            `mapstructure:"entrypoint"`             // Override the containers entrypoint
   208  	IpcMode              string              `mapstructure:"ipc_mode"`               // The IPC mode of the container - host and none
   209  	NetworkMode          string              `mapstructure:"network_mode"`           // The network mode of the container - host, nat and none
   210  	NetworkAliases       []string            `mapstructure:"network_aliases"`        // The network-scoped alias for the container
   211  	IPv4Address          string              `mapstructure:"ipv4_address"`           // The container ipv4 address
   212  	IPv6Address          string              `mapstructure:"ipv6_address"`           // the container ipv6 address
   213  	PidMode              string              `mapstructure:"pid_mode"`               // The PID mode of the container - host and none
   214  	UTSMode              string              `mapstructure:"uts_mode"`               // The UTS mode of the container - host and none
   215  	UsernsMode           string              `mapstructure:"userns_mode"`            // The User namespace mode of the container - host and none
   216  	PortMapRaw           []map[string]string `mapstructure:"port_map"`               //
   217  	PortMap              map[string]int      `mapstructure:"-"`                      // A map of host port labels and the ports exposed on the container
   218  	Privileged           bool                `mapstructure:"privileged"`             // Flag to run the container in privileged mode
   219  	SysctlRaw            []map[string]string `mapstructure:"sysctl"`                 //
   220  	Sysctl               map[string]string   `mapstructure:"-"`                      // The sysctl custom configurations
   221  	UlimitRaw            []map[string]string `mapstructure:"ulimit"`                 //
   222  	Ulimit               []docker.ULimit     `mapstructure:"-"`                      // The ulimit custom configurations
   223  	DNSServers           []string            `mapstructure:"dns_servers"`            // DNS Server for containers
   224  	DNSSearchDomains     []string            `mapstructure:"dns_search_domains"`     // DNS Search domains for containers
   225  	DNSOptions           []string            `mapstructure:"dns_options"`            // DNS Options
   226  	ExtraHosts           []string            `mapstructure:"extra_hosts"`            // Add host to /etc/hosts (host:IP)
   227  	Hostname             string              `mapstructure:"hostname"`               // Hostname for containers
   228  	LabelsRaw            []map[string]string `mapstructure:"labels"`                 //
   229  	Labels               map[string]string   `mapstructure:"-"`                      // Labels to set when the container starts up
   230  	Auth                 []DockerDriverAuth  `mapstructure:"auth"`                   // Authentication credentials for a private Docker registry
   231  	AuthSoftFail         bool                `mapstructure:"auth_soft_fail"`         // Soft-fail if auth creds are provided but fail
   232  	TTY                  bool                `mapstructure:"tty"`                    // Allocate a Pseudo-TTY
   233  	Interactive          bool                `mapstructure:"interactive"`            // Keep STDIN open even if not attached
   234  	ShmSize              int64               `mapstructure:"shm_size"`               // Size of /dev/shm of the container in bytes
   235  	WorkDir              string              `mapstructure:"work_dir"`               // Working directory inside the container
   236  	Logging              []DockerLoggingOpts `mapstructure:"logging"`                // Logging options for syslog server
   237  	Volumes              []string            `mapstructure:"volumes"`                // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container
   238  	Mounts               []DockerMount       `mapstructure:"mounts"`                 // Docker volumes to mount
   239  	VolumeDriver         string              `mapstructure:"volume_driver"`          // Docker volume driver used for the container's volumes
   240  	ForcePull            bool                `mapstructure:"force_pull"`             // Always force pull before running image, useful if your tags are mutable
   241  	MacAddress           string              `mapstructure:"mac_address"`            // Pin mac address to container
   242  	SecurityOpt          []string            `mapstructure:"security_opt"`           // Flags to pass directly to security-opt
   243  	Devices              []DockerDevice      `mapstructure:"devices"`                // To allow mounting USB or other serial control devices
   244  	CapAdd               []string            `mapstructure:"cap_add"`                // Flags to pass directly to cap-add
   245  	CapDrop              []string            `mapstructure:"cap_drop"`               // Flags to pass directly to cap-drop
   246  	ReadonlyRootfs       bool                `mapstructure:"readonly_rootfs"`        // Mount the container’s root filesystem as read only
   247  	AdvertiseIPv6Address bool                `mapstructure:"advertise_ipv6_address"` // Flag to use the GlobalIPv6Address from the container as the detected IP
   248  	CPUHardLimit         bool                `mapstructure:"cpu_hard_limit"`         // Enforce CPU hard limit.
   249  	PidsLimit            int64               `mapstructure:"pids_limit"`             // Enforce Docker Pids limit
   250  }
   251  
   252  func sliceMergeUlimit(ulimitsRaw map[string]string) ([]docker.ULimit, error) {
   253  	var ulimits []docker.ULimit
   254  
   255  	for name, ulimitRaw := range ulimitsRaw {
   256  		if len(ulimitRaw) == 0 {
   257  			return []docker.ULimit{}, fmt.Errorf("Malformed ulimit specification %v: %q, cannot be empty", name, ulimitRaw)
   258  		}
   259  		// hard limit is optional
   260  		if strings.Contains(ulimitRaw, ":") == false {
   261  			ulimitRaw = ulimitRaw + ":" + ulimitRaw
   262  		}
   263  
   264  		splitted := strings.SplitN(ulimitRaw, ":", 2)
   265  		if len(splitted) < 2 {
   266  			return []docker.ULimit{}, fmt.Errorf("Malformed ulimit specification %v: %v", name, ulimitRaw)
   267  		}
   268  		soft, err := strconv.Atoi(splitted[0])
   269  		if err != nil {
   270  			return []docker.ULimit{}, fmt.Errorf("Malformed soft ulimit %v: %v", name, ulimitRaw)
   271  		}
   272  		hard, err := strconv.Atoi(splitted[1])
   273  		if err != nil {
   274  			return []docker.ULimit{}, fmt.Errorf("Malformed hard ulimit %v: %v", name, ulimitRaw)
   275  		}
   276  
   277  		ulimit := docker.ULimit{
   278  			Name: name,
   279  			Soft: int64(soft),
   280  			Hard: int64(hard),
   281  		}
   282  		ulimits = append(ulimits, ulimit)
   283  	}
   284  	return ulimits, nil
   285  }
   286  
   287  // Validate validates a docker driver config
   288  func (c *DockerDriverConfig) Validate() error {
   289  	if c.ImageName == "" {
   290  		return fmt.Errorf("Docker Driver needs an image name")
   291  	}
   292  	if len(c.Devices) > 0 {
   293  		for _, dev := range c.Devices {
   294  			if dev.HostPath == "" {
   295  				return fmt.Errorf("host path must be set in configuration for devices")
   296  			}
   297  			if dev.CgroupPermissions != "" {
   298  				for _, c := range dev.CgroupPermissions {
   299  					ch := string(c)
   300  					if ch != "r" && ch != "w" && ch != "m" {
   301  						return fmt.Errorf("invalid cgroup permission string: %q", dev.CgroupPermissions)
   302  					}
   303  				}
   304  			}
   305  		}
   306  	}
   307  	c.Sysctl = mapMergeStrStr(c.SysctlRaw...)
   308  	c.Labels = mapMergeStrStr(c.LabelsRaw...)
   309  	if len(c.Logging) > 0 {
   310  		c.Logging[0].Config = mapMergeStrStr(c.Logging[0].ConfigRaw...)
   311  	}
   312  
   313  	mergedUlimitsRaw := mapMergeStrStr(c.UlimitRaw...)
   314  	ulimit, err := sliceMergeUlimit(mergedUlimitsRaw)
   315  	if err != nil {
   316  		return err
   317  	}
   318  	c.Ulimit = ulimit
   319  	return nil
   320  }
   321  
   322  // NewDockerDriverConfig returns a docker driver config by parsing the HCL
   323  // config
   324  func NewDockerDriverConfig(task *structs.Task, env *env.TaskEnv) (*DockerDriverConfig, error) {
   325  	var dconf DockerDriverConfig
   326  
   327  	if err := mapstructure.WeakDecode(task.Config, &dconf); err != nil {
   328  		return nil, err
   329  	}
   330  
   331  	// Interpolate everything that is a string
   332  	dconf.ImageName = env.ReplaceEnv(dconf.ImageName)
   333  	dconf.Command = env.ReplaceEnv(dconf.Command)
   334  	dconf.Entrypoint = env.ParseAndReplace(dconf.Entrypoint)
   335  	dconf.IpcMode = env.ReplaceEnv(dconf.IpcMode)
   336  	dconf.NetworkMode = env.ReplaceEnv(dconf.NetworkMode)
   337  	dconf.NetworkAliases = env.ParseAndReplace(dconf.NetworkAliases)
   338  	dconf.IPv4Address = env.ReplaceEnv(dconf.IPv4Address)
   339  	dconf.IPv6Address = env.ReplaceEnv(dconf.IPv6Address)
   340  	dconf.PidMode = env.ReplaceEnv(dconf.PidMode)
   341  	dconf.UTSMode = env.ReplaceEnv(dconf.UTSMode)
   342  	dconf.Hostname = env.ReplaceEnv(dconf.Hostname)
   343  	dconf.WorkDir = env.ReplaceEnv(dconf.WorkDir)
   344  	dconf.LoadImage = env.ReplaceEnv(dconf.LoadImage)
   345  	dconf.Volumes = env.ParseAndReplace(dconf.Volumes)
   346  	dconf.VolumeDriver = env.ReplaceEnv(dconf.VolumeDriver)
   347  	dconf.DNSServers = env.ParseAndReplace(dconf.DNSServers)
   348  	dconf.DNSSearchDomains = env.ParseAndReplace(dconf.DNSSearchDomains)
   349  	dconf.DNSOptions = env.ParseAndReplace(dconf.DNSOptions)
   350  	dconf.ExtraHosts = env.ParseAndReplace(dconf.ExtraHosts)
   351  	dconf.MacAddress = env.ReplaceEnv(dconf.MacAddress)
   352  	dconf.SecurityOpt = env.ParseAndReplace(dconf.SecurityOpt)
   353  	dconf.CapAdd = env.ParseAndReplace(dconf.CapAdd)
   354  	dconf.CapDrop = env.ParseAndReplace(dconf.CapDrop)
   355  
   356  	for _, m := range dconf.SysctlRaw {
   357  		for k, v := range m {
   358  			delete(m, k)
   359  			m[env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   360  		}
   361  	}
   362  
   363  	for _, m := range dconf.UlimitRaw {
   364  		for k, v := range m {
   365  			delete(m, k)
   366  			m[env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   367  		}
   368  	}
   369  
   370  	for _, m := range dconf.LabelsRaw {
   371  		for k, v := range m {
   372  			delete(m, k)
   373  			m[env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   374  		}
   375  	}
   376  	dconf.Labels = mapMergeStrStr(dconf.LabelsRaw...)
   377  
   378  	for i, a := range dconf.Auth {
   379  		dconf.Auth[i].Username = env.ReplaceEnv(a.Username)
   380  		dconf.Auth[i].Password = env.ReplaceEnv(a.Password)
   381  		dconf.Auth[i].Email = env.ReplaceEnv(a.Email)
   382  		dconf.Auth[i].ServerAddress = env.ReplaceEnv(a.ServerAddress)
   383  	}
   384  
   385  	for i, l := range dconf.Logging {
   386  		dconf.Logging[i].Type = env.ReplaceEnv(l.Type)
   387  		for _, c := range l.ConfigRaw {
   388  			for k, v := range c {
   389  				delete(c, k)
   390  				c[env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   391  			}
   392  		}
   393  	}
   394  
   395  	for i, m := range dconf.Mounts {
   396  		m.Target = env.ReplaceEnv(m.Target)
   397  		m.Source = env.ReplaceEnv(m.Source)
   398  
   399  		if m.Type == "" {
   400  			// default to `volume` type for backwards compatibility
   401  			m.Type = "volume"
   402  		}
   403  
   404  		if m.Type != "bind" && m.Type != "volume" {
   405  			return nil, fmt.Errorf(`Docker mount type must be "bind" or "volume"`)
   406  		}
   407  
   408  		if m.Type == "bind" && len(m.VolumeOptions) > 0 {
   409  			return nil, fmt.Errorf(`volume_options invalid on "bind" mount type`)
   410  		}
   411  
   412  		if m.Type == "volume" && len(m.BindOptions) > 0 {
   413  			return nil, fmt.Errorf(`bind_options invalid on "volume" mount type`)
   414  		}
   415  
   416  		if len(m.BindOptions) > 1 {
   417  			return nil, fmt.Errorf("only one bind_options stanza allowed")
   418  		}
   419  
   420  		if len(m.VolumeOptions) > 1 {
   421  			return nil, fmt.Errorf("only one volume_options stanza allowed")
   422  		}
   423  
   424  		if len(m.VolumeOptions) == 1 {
   425  			vo := m.VolumeOptions[0]
   426  			if len(vo.Labels) > 1 {
   427  				return nil, fmt.Errorf("labels may only be specified once in volume_options stanza")
   428  			}
   429  
   430  			if len(vo.Labels) == 1 {
   431  				for k, v := range vo.Labels[0] {
   432  					if k != env.ReplaceEnv(k) {
   433  						delete(vo.Labels[0], k)
   434  					}
   435  					vo.Labels[0][env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   436  				}
   437  			}
   438  
   439  			if len(vo.DriverConfig) > 1 {
   440  				return nil, fmt.Errorf("volume driver config may only be specified once")
   441  			}
   442  			if len(vo.DriverConfig) == 1 {
   443  				vo.DriverConfig[0].Name = env.ReplaceEnv(vo.DriverConfig[0].Name)
   444  				if len(vo.DriverConfig[0].Options) > 1 {
   445  					return nil, fmt.Errorf("volume driver options may only be specified once")
   446  				}
   447  
   448  				if len(vo.DriverConfig[0].Options) == 1 {
   449  					options := vo.DriverConfig[0].Options[0]
   450  					for k, v := range options {
   451  						if k != env.ReplaceEnv(k) {
   452  							delete(options, k)
   453  						}
   454  						options[env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   455  					}
   456  				}
   457  			}
   458  		}
   459  
   460  		// Copy altered version back into slice
   461  		dconf.Mounts[i] = m
   462  	}
   463  
   464  	if len(dconf.Logging) > 0 {
   465  		dconf.Logging[0].Config = mapMergeStrStr(dconf.Logging[0].ConfigRaw...)
   466  	}
   467  
   468  	portMap := make(map[string]int)
   469  	for _, m := range dconf.PortMapRaw {
   470  		for k, v := range m {
   471  			ki, vi := env.ReplaceEnv(k), env.ReplaceEnv(v)
   472  			p, err := strconv.Atoi(vi)
   473  			if err != nil {
   474  				return nil, fmt.Errorf("failed to parse port map value %v to %v: %v", ki, vi, err)
   475  			}
   476  			portMap[ki] = p
   477  		}
   478  	}
   479  	dconf.PortMap = portMap
   480  
   481  	// Remove any http
   482  	if strings.Contains(dconf.ImageName, "https://") {
   483  		dconf.ImageName = strings.Replace(dconf.ImageName, "https://", "", 1)
   484  	}
   485  
   486  	// If devices are configured set default cgroup permissions
   487  	if len(dconf.Devices) > 0 {
   488  		for i, dev := range dconf.Devices {
   489  			if dev.CgroupPermissions == "" {
   490  				dev.CgroupPermissions = "rwm"
   491  			}
   492  			dconf.Devices[i] = dev
   493  		}
   494  	}
   495  
   496  	if err := dconf.Validate(); err != nil {
   497  		return nil, err
   498  	}
   499  	return &dconf, nil
   500  }
   501  
   502  type dockerPID struct {
   503  	Version        string
   504  	Image          string
   505  	ImageID        string
   506  	ContainerID    string
   507  	KillTimeout    time.Duration
   508  	MaxKillTimeout time.Duration
   509  	PluginConfig   *PluginReattachConfig
   510  }
   511  
   512  type DockerHandle struct {
   513  	pluginClient          *plugin.Client
   514  	executor              executor.Executor
   515  	client                *docker.Client
   516  	waitClient            *docker.Client
   517  	logger                *log.Logger
   518  	jobName               string
   519  	taskGroupName         string
   520  	taskName              string
   521  	Image                 string
   522  	ImageID               string
   523  	containerID           string
   524  	version               string
   525  	killTimeout           time.Duration
   526  	maxKillTimeout        time.Duration
   527  	resourceUsageLock     sync.RWMutex
   528  	resourceUsage         *cstructs.TaskResourceUsage
   529  	waitCh                chan *dstructs.WaitResult
   530  	doneCh                chan bool
   531  	removeContainerOnExit bool
   532  }
   533  
   534  func NewDockerDriver(ctx *DriverContext) Driver {
   535  	return &DockerDriver{DriverContext: *ctx}
   536  }
   537  
   538  func (d *DockerDriver) Fingerprint(req *cstructs.FingerprintRequest, resp *cstructs.FingerprintResponse) error {
   539  	client, _, err := d.dockerClients()
   540  	if err != nil {
   541  		if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
   542  			d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s", err)
   543  		}
   544  		d.fingerprintSuccess = helper.BoolToPtr(false)
   545  		return nil
   546  	}
   547  
   548  	// This is the first operation taken on the client so we'll try to
   549  	// establish a connection to the Docker daemon. If this fails it means
   550  	// Docker isn't available so we'll simply disable the docker driver.
   551  	env, err := client.Version()
   552  	if err != nil {
   553  		if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
   554  			d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon at %s: %s", client.Endpoint(), err)
   555  		}
   556  		d.fingerprintSuccess = helper.BoolToPtr(false)
   557  		resp.RemoveAttribute(dockerDriverAttr)
   558  		return nil
   559  	}
   560  
   561  	resp.AddAttribute(dockerDriverAttr, "1")
   562  	resp.AddAttribute("driver.docker.version", env.Get("Version"))
   563  	resp.Detected = true
   564  
   565  	privileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false)
   566  	if privileged {
   567  		resp.AddAttribute(dockerPrivilegedConfigOption, "1")
   568  	}
   569  
   570  	// Advertise if this node supports Docker volumes
   571  	if d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault) {
   572  		resp.AddAttribute("driver."+dockerVolumesConfigOption, "1")
   573  	}
   574  
   575  	// Detect bridge IP address - #2785
   576  	if nets, err := client.ListNetworks(); err != nil {
   577  		d.logger.Printf("[WARN] driver.docker: error discovering bridge IP: %v", err)
   578  	} else {
   579  		for _, n := range nets {
   580  			if n.Name != "bridge" {
   581  				continue
   582  			}
   583  
   584  			if len(n.IPAM.Config) == 0 {
   585  				d.logger.Printf("[WARN] driver.docker: no IPAM config for bridge network")
   586  				break
   587  			}
   588  
   589  			if n.IPAM.Config[0].Gateway != "" {
   590  				resp.AddAttribute("driver.docker.bridge_ip", n.IPAM.Config[0].Gateway)
   591  			} else if d.fingerprintSuccess == nil {
   592  				// Docker 17.09.0-ce dropped the Gateway IP from the bridge network
   593  				// See https://github.com/moby/moby/issues/32648
   594  				d.logger.Printf("[DEBUG] driver.docker: bridge_ip could not be discovered")
   595  			}
   596  			break
   597  		}
   598  	}
   599  
   600  	d.fingerprintSuccess = helper.BoolToPtr(true)
   601  	return nil
   602  }
   603  
   604  // HealthCheck implements the interface for the HealthCheck interface. This
   605  // performs a health check on the docker driver, asserting whether the docker
   606  // driver is responsive to a `docker ps` command.
   607  func (d *DockerDriver) HealthCheck(req *cstructs.HealthCheckRequest, resp *cstructs.HealthCheckResponse) error {
   608  	dinfo := &structs.DriverInfo{
   609  		UpdateTime: time.Now(),
   610  	}
   611  
   612  	healthCheckClient, err := d.dockerHealthCheckClient()
   613  	if err != nil {
   614  		d.logger.Printf("[WARN] driver.docker: failed to retrieve Docker client in the process of a docker health check: %v", err)
   615  		dinfo.HealthDescription = fmt.Sprintf("Failed retrieving Docker client: %v", err)
   616  		resp.AddDriverInfo("docker", dinfo)
   617  		return nil
   618  	}
   619  
   620  	_, err = healthCheckClient.ListContainers(docker.ListContainersOptions{All: false})
   621  	if err != nil {
   622  		d.logger.Printf("[WARN] driver.docker: failed to list Docker containers in the process of a Docker health check: %v", err)
   623  		dinfo.HealthDescription = fmt.Sprintf("Failed to list Docker containers: %v", err)
   624  		resp.AddDriverInfo("docker", dinfo)
   625  		return nil
   626  	}
   627  
   628  	d.logger.Printf("[TRACE] driver.docker: docker driver is available and is responsive to `docker ps`")
   629  	dinfo.Healthy = true
   630  	dinfo.HealthDescription = "Driver is available and responsive"
   631  	resp.AddDriverInfo("docker", dinfo)
   632  	return nil
   633  }
   634  
   635  // GetHealthChecks implements the interface for the HealthCheck interface. This
   636  // sets whether the driver is eligible for periodic health checks and the
   637  // interval at which to do them.
   638  func (d *DockerDriver) GetHealthCheckInterval(req *cstructs.HealthCheckIntervalRequest, resp *cstructs.HealthCheckIntervalResponse) error {
   639  	resp.Eligible = true
   640  	resp.Period = 1 * time.Minute
   641  	return nil
   642  }
   643  
   644  // Validate is used to validate the driver configuration
   645  func (d *DockerDriver) Validate(config map[string]interface{}) error {
   646  	fd := &fields.FieldData{
   647  		Raw: config,
   648  		Schema: map[string]*fields.FieldSchema{
   649  			"image": {
   650  				Type:     fields.TypeString,
   651  				Required: true,
   652  			},
   653  			"load": {
   654  				Type: fields.TypeString,
   655  			},
   656  			"command": {
   657  				Type: fields.TypeString,
   658  			},
   659  			"args": {
   660  				Type: fields.TypeArray,
   661  			},
   662  			"entrypoint": {
   663  				Type: fields.TypeArray,
   664  			},
   665  			"ipc_mode": {
   666  				Type: fields.TypeString,
   667  			},
   668  			"network_mode": {
   669  				Type: fields.TypeString,
   670  			},
   671  			"network_aliases": {
   672  				Type: fields.TypeArray,
   673  			},
   674  			"ipv4_address": {
   675  				Type: fields.TypeString,
   676  			},
   677  			"ipv6_address": {
   678  				Type: fields.TypeString,
   679  			},
   680  			"mac_address": {
   681  				Type: fields.TypeString,
   682  			},
   683  			"pid_mode": {
   684  				Type: fields.TypeString,
   685  			},
   686  			"uts_mode": {
   687  				Type: fields.TypeString,
   688  			},
   689  			"userns_mode": {
   690  				Type: fields.TypeString,
   691  			},
   692  			"sysctl": {
   693  				Type: fields.TypeArray,
   694  			},
   695  			"ulimit": {
   696  				Type: fields.TypeArray,
   697  			},
   698  			"port_map": {
   699  				Type: fields.TypeArray,
   700  			},
   701  			"privileged": {
   702  				Type: fields.TypeBool,
   703  			},
   704  			"dns_servers": {
   705  				Type: fields.TypeArray,
   706  			},
   707  			"dns_options": {
   708  				Type: fields.TypeArray,
   709  			},
   710  			"dns_search_domains": {
   711  				Type: fields.TypeArray,
   712  			},
   713  			"extra_hosts": {
   714  				Type: fields.TypeArray,
   715  			},
   716  			"hostname": {
   717  				Type: fields.TypeString,
   718  			},
   719  			"labels": {
   720  				Type: fields.TypeArray,
   721  			},
   722  			"auth": {
   723  				Type: fields.TypeArray,
   724  			},
   725  			"auth_soft_fail": {
   726  				Type: fields.TypeBool,
   727  			},
   728  			// COMPAT: Remove in 0.6.0. SSL is no longer needed
   729  			"ssl": {
   730  				Type: fields.TypeBool,
   731  			},
   732  			"tty": {
   733  				Type: fields.TypeBool,
   734  			},
   735  			"interactive": {
   736  				Type: fields.TypeBool,
   737  			},
   738  			"shm_size": {
   739  				Type: fields.TypeInt,
   740  			},
   741  			"work_dir": {
   742  				Type: fields.TypeString,
   743  			},
   744  			"logging": {
   745  				Type: fields.TypeArray,
   746  			},
   747  			"volumes": {
   748  				Type: fields.TypeArray,
   749  			},
   750  			"volume_driver": {
   751  				Type: fields.TypeString,
   752  			},
   753  			"mounts": {
   754  				Type: fields.TypeArray,
   755  			},
   756  			"force_pull": {
   757  				Type: fields.TypeBool,
   758  			},
   759  			"security_opt": {
   760  				Type: fields.TypeArray,
   761  			},
   762  			"devices": {
   763  				Type: fields.TypeArray,
   764  			},
   765  			"cap_add": {
   766  				Type: fields.TypeArray,
   767  			},
   768  			"cap_drop": {
   769  				Type: fields.TypeArray,
   770  			},
   771  			"readonly_rootfs": {
   772  				Type: fields.TypeBool,
   773  			},
   774  			"advertise_ipv6_address": {
   775  				Type: fields.TypeBool,
   776  			},
   777  			"cpu_hard_limit": {
   778  				Type: fields.TypeBool,
   779  			},
   780  			"pids_limit": {
   781  				Type: fields.TypeInt,
   782  			},
   783  		},
   784  	}
   785  
   786  	if err := fd.Validate(); err != nil {
   787  		return err
   788  	}
   789  
   790  	return nil
   791  }
   792  
   793  func (d *DockerDriver) Abilities() DriverAbilities {
   794  	return DriverAbilities{
   795  		SendSignals: true,
   796  		Exec:        true,
   797  	}
   798  }
   799  
   800  func (d *DockerDriver) FSIsolation() cstructs.FSIsolation {
   801  	return cstructs.FSIsolationImage
   802  }
   803  
   804  // getDockerCoordinator returns the docker coordinator and the caller ID to use when
   805  // interacting with the coordinator
   806  func (d *DockerDriver) getDockerCoordinator(client *docker.Client) (*dockerCoordinator, string) {
   807  	config := &dockerCoordinatorConfig{
   808  		client:      client,
   809  		cleanup:     d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault),
   810  		logger:      d.logger,
   811  		removeDelay: d.config.ReadDurationDefault(dockerImageRemoveDelayConfigOption, dockerImageRemoveDelayConfigDefault),
   812  	}
   813  
   814  	return GetDockerCoordinator(config), fmt.Sprintf("%s-%s", d.DriverContext.allocID, d.DriverContext.taskName)
   815  }
   816  
   817  func (d *DockerDriver) Prestart(ctx *ExecContext, task *structs.Task) (*PrestartResponse, error) {
   818  	driverConfig, err := NewDockerDriverConfig(task, ctx.TaskEnv)
   819  	if err != nil {
   820  		return nil, err
   821  	}
   822  
   823  	// Set state needed by Start
   824  	d.driverConfig = driverConfig
   825  
   826  	// Initialize docker API clients
   827  	client, _, err := d.dockerClients()
   828  	if err != nil {
   829  		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
   830  	}
   831  
   832  	// Ensure the image is available
   833  	id, err := d.createImage(driverConfig, client, ctx.TaskDir)
   834  	if err != nil {
   835  		return nil, err
   836  	}
   837  	d.imageID = id
   838  
   839  	resp := NewPrestartResponse()
   840  	resp.CreatedResources.Add(dockerImageResKey, id)
   841  
   842  	// Return the PortMap if it's set
   843  	if len(driverConfig.PortMap) > 0 {
   844  		resp.Network = &cstructs.DriverNetwork{
   845  			PortMap: driverConfig.PortMap,
   846  		}
   847  	}
   848  	return resp, nil
   849  }
   850  
   851  func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (*StartResponse, error) {
   852  	pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
   853  	executorConfig := &dstructs.ExecutorConfig{
   854  		LogFile:  pluginLogFile,
   855  		LogLevel: d.config.LogLevel,
   856  	}
   857  
   858  	exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig)
   859  	if err != nil {
   860  		return nil, err
   861  	}
   862  	executorCtx := &executor.ExecutorContext{
   863  		TaskEnv:        ctx.TaskEnv,
   864  		Task:           task,
   865  		Driver:         "docker",
   866  		LogDir:         ctx.TaskDir.LogDir,
   867  		TaskDir:        ctx.TaskDir.Dir,
   868  		PortLowerBound: d.config.ClientMinPort,
   869  		PortUpperBound: d.config.ClientMaxPort,
   870  	}
   871  	if err := exec.SetContext(executorCtx); err != nil {
   872  		pluginClient.Kill()
   873  		return nil, fmt.Errorf("failed to set executor context: %v", err)
   874  	}
   875  
   876  	// The user hasn't specified any logging options so launch our own syslog
   877  	// server if possible.
   878  	syslogAddr := ""
   879  	if len(d.driverConfig.Logging) == 0 {
   880  		if runtime.GOOS == "darwin" {
   881  			d.logger.Printf("[DEBUG] driver.docker: disabling syslog driver as Docker for Mac workaround")
   882  		} else {
   883  			ss, err := exec.LaunchSyslogServer()
   884  			if err != nil {
   885  				pluginClient.Kill()
   886  				return nil, fmt.Errorf("failed to start syslog collector: %v", err)
   887  			}
   888  			syslogAddr = ss.Addr
   889  		}
   890  	}
   891  
   892  	config, err := d.createContainerConfig(ctx, task, d.driverConfig, syslogAddr)
   893  	if err != nil {
   894  		d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err)
   895  		pluginClient.Kill()
   896  		return nil, fmt.Errorf("Failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err)
   897  	}
   898  
   899  	container, err := d.createContainer(client, config)
   900  	if err != nil {
   901  		wrapped := fmt.Sprintf("Failed to create container: %v", err)
   902  		d.logger.Printf("[ERR] driver.docker: %s", wrapped)
   903  		pluginClient.Kill()
   904  		return nil, structs.WrapRecoverable(wrapped, err)
   905  	}
   906  
   907  	d.logger.Printf("[INFO] driver.docker: created container %s", container.ID)
   908  
   909  	// We don't need to start the container if the container is already running
   910  	// since we don't create containers which are already present on the host
   911  	// and are running
   912  	if !container.State.Running {
   913  		// Start the container
   914  		if err := d.startContainer(container); err != nil {
   915  			d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err)
   916  			pluginClient.Kill()
   917  			return nil, structs.NewRecoverableError(fmt.Errorf("Failed to start container %s: %s", container.ID, err), structs.IsRecoverable(err))
   918  		}
   919  
   920  		// InspectContainer to get all of the container metadata as
   921  		// much of the metadata (eg networking) isn't populated until
   922  		// the container is started
   923  		runningContainer, err := client.InspectContainer(container.ID)
   924  		if err != nil {
   925  			err = fmt.Errorf("failed to inspect started container %s: %s", container.ID, err)
   926  			d.logger.Printf("[ERR] driver.docker: %v", err)
   927  			pluginClient.Kill()
   928  			return nil, structs.NewRecoverableError(err, true)
   929  		}
   930  		container = runningContainer
   931  		d.logger.Printf("[INFO] driver.docker: started container %s", container.ID)
   932  	} else {
   933  		d.logger.Printf("[DEBUG] driver.docker: re-attaching to container %s with status %q",
   934  			container.ID, container.State.String())
   935  	}
   936  
   937  	// Return a driver handle
   938  	maxKill := d.DriverContext.config.MaxKillTimeout
   939  	h := &DockerHandle{
   940  		client:                client,
   941  		waitClient:            waitClient,
   942  		executor:              exec,
   943  		pluginClient:          pluginClient,
   944  		logger:                d.logger,
   945  		jobName:               d.DriverContext.jobName,
   946  		taskGroupName:         d.DriverContext.taskGroupName,
   947  		taskName:              d.DriverContext.taskName,
   948  		Image:                 d.driverConfig.ImageName,
   949  		ImageID:               d.imageID,
   950  		containerID:           container.ID,
   951  		version:               d.config.Version.VersionNumber(),
   952  		killTimeout:           GetKillTimeout(task.KillTimeout, maxKill),
   953  		maxKillTimeout:        maxKill,
   954  		doneCh:                make(chan bool),
   955  		waitCh:                make(chan *dstructs.WaitResult, 1),
   956  		removeContainerOnExit: d.config.ReadBoolDefault(dockerCleanupContainerConfigOption, dockerCleanupContainerConfigDefault),
   957  	}
   958  	go h.collectStats()
   959  	go h.run()
   960  
   961  	// Detect container address
   962  	ip, autoUse := d.detectIP(container)
   963  
   964  	// Create a response with the driver handle and container network metadata
   965  	resp := &StartResponse{
   966  		Handle: h,
   967  		Network: &cstructs.DriverNetwork{
   968  			PortMap:       d.driverConfig.PortMap,
   969  			IP:            ip,
   970  			AutoAdvertise: autoUse,
   971  		},
   972  	}
   973  	return resp, nil
   974  }
   975  
   976  // detectIP of Docker container. Returns the first IP found as well as true if
   977  // the IP should be advertised (bridge network IPs return false). Returns an
   978  // empty string and false if no IP could be found.
   979  func (d *DockerDriver) detectIP(c *docker.Container) (string, bool) {
   980  	if c.NetworkSettings == nil {
   981  		// This should only happen if there's been a coding error (such
   982  		// as not calling InspectContainer after CreateContainer). Code
   983  		// defensively in case the Docker API changes subtly.
   984  		d.logger.Printf("[ERROR] driver.docker: no network settings for container %s", c.ID)
   985  		return "", false
   986  	}
   987  
   988  	ip, ipName := "", ""
   989  	auto := false
   990  	for name, net := range c.NetworkSettings.Networks {
   991  		if net.IPAddress == "" {
   992  			// Ignore networks without an IP address
   993  			continue
   994  		}
   995  
   996  		ip = net.IPAddress
   997  		if d.driverConfig.AdvertiseIPv6Address {
   998  			ip = net.GlobalIPv6Address
   999  			auto = true
  1000  		}
  1001  		ipName = name
  1002  
  1003  		// Don't auto-advertise IPs for default networks (bridge on
  1004  		// Linux, nat on Windows)
  1005  		if name != "bridge" && name != "nat" {
  1006  			auto = true
  1007  		}
  1008  
  1009  		break
  1010  	}
  1011  
  1012  	if n := len(c.NetworkSettings.Networks); n > 1 {
  1013  		d.logger.Printf("[WARN] driver.docker: task %s multiple (%d) Docker networks for container %q but Nomad only supports 1: choosing %q", d.taskName, n, c.ID, ipName)
  1014  	}
  1015  
  1016  	return ip, auto
  1017  }
  1018  
  1019  func (d *DockerDriver) Cleanup(_ *ExecContext, res *CreatedResources) error {
  1020  	retry := false
  1021  	var merr multierror.Error
  1022  	for key, resources := range res.Resources {
  1023  		switch key {
  1024  		case dockerImageResKey:
  1025  			for _, value := range resources {
  1026  				err := d.cleanupImage(value)
  1027  				if err != nil {
  1028  					if structs.IsRecoverable(err) {
  1029  						retry = true
  1030  					}
  1031  					merr.Errors = append(merr.Errors, err)
  1032  					continue
  1033  				}
  1034  
  1035  				// Remove cleaned image from resources
  1036  				res.Remove(dockerImageResKey, value)
  1037  			}
  1038  		default:
  1039  			d.logger.Printf("[ERR] driver.docker: unknown resource to cleanup: %q", key)
  1040  		}
  1041  	}
  1042  	return structs.NewRecoverableError(merr.ErrorOrNil(), retry)
  1043  }
  1044  
  1045  // cleanupImage removes a Docker image. No error is returned if the image
  1046  // doesn't exist or is still in use. Requires the global client to already be
  1047  // initialized.
  1048  func (d *DockerDriver) cleanupImage(imageID string) error {
  1049  	if !d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault) {
  1050  		// Config says not to cleanup
  1051  		return nil
  1052  	}
  1053  
  1054  	coordinator, callerID := d.getDockerCoordinator(client)
  1055  	coordinator.RemoveImage(imageID, callerID)
  1056  
  1057  	return nil
  1058  }
  1059  
  1060  // dockerHealthCheckClient creates a single *docker.Client with a timeout of
  1061  // one minute, which will be used when performing Docker health checks.
  1062  func (d *DockerDriver) dockerHealthCheckClient() (*docker.Client, error) {
  1063  	createClientsLock.Lock()
  1064  	defer createClientsLock.Unlock()
  1065  
  1066  	if healthCheckClient != nil {
  1067  		return healthCheckClient, nil
  1068  	}
  1069  
  1070  	var err error
  1071  	healthCheckClient, err = d.newDockerClient(dockerHealthCheckTimeout)
  1072  	if err != nil {
  1073  		return nil, err
  1074  	}
  1075  
  1076  	return healthCheckClient, nil
  1077  }
  1078  
  1079  // dockerClients creates two *docker.Client, one for long running operations and
  1080  // the other for shorter operations. In test / dev mode we can use ENV vars to
  1081  // connect to the docker daemon. In production mode we will read docker.endpoint
  1082  // from the config file.
  1083  func (d *DockerDriver) dockerClients() (*docker.Client, *docker.Client, error) {
  1084  	createClientsLock.Lock()
  1085  	defer createClientsLock.Unlock()
  1086  
  1087  	if client != nil && waitClient != nil {
  1088  		return client, waitClient, nil
  1089  	}
  1090  
  1091  	var err error
  1092  
  1093  	// Onlt initialize the client if it hasn't yet been done
  1094  	if client == nil {
  1095  		client, err = d.newDockerClient(dockerTimeout)
  1096  		if err != nil {
  1097  			return nil, nil, err
  1098  		}
  1099  	}
  1100  
  1101  	// Only initialize the waitClient if it hasn't yet been done
  1102  	if waitClient == nil {
  1103  		waitClient, err = d.newDockerClient(0 * time.Minute)
  1104  		if err != nil {
  1105  			return nil, nil, err
  1106  		}
  1107  	}
  1108  
  1109  	return client, waitClient, nil
  1110  }
  1111  
  1112  // newDockerClient creates a new *docker.Client with a configurable timeout
  1113  func (d *DockerDriver) newDockerClient(timeout time.Duration) (*docker.Client, error) {
  1114  	var err error
  1115  	var merr multierror.Error
  1116  	var newClient *docker.Client
  1117  
  1118  	// Default to using whatever is configured in docker.endpoint. If this is
  1119  	// not specified we'll fall back on NewClientFromEnv which reads config from
  1120  	// the DOCKER_* environment variables DOCKER_HOST, DOCKER_TLS_VERIFY, and
  1121  	// DOCKER_CERT_PATH. This allows us to lock down the config in production
  1122  	// but also accept the standard ENV configs for dev and test.
  1123  	dockerEndpoint := d.config.Read("docker.endpoint")
  1124  	if dockerEndpoint != "" {
  1125  		cert := d.config.Read("docker.tls.cert")
  1126  		key := d.config.Read("docker.tls.key")
  1127  		ca := d.config.Read("docker.tls.ca")
  1128  
  1129  		if cert+key+ca != "" {
  1130  			d.logger.Printf("[DEBUG] driver.docker: using TLS client connection to %s", dockerEndpoint)
  1131  			newClient, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca)
  1132  			if err != nil {
  1133  				merr.Errors = append(merr.Errors, err)
  1134  			}
  1135  		} else {
  1136  			d.logger.Printf("[DEBUG] driver.docker: using standard client connection to %s", dockerEndpoint)
  1137  			newClient, err = docker.NewClient(dockerEndpoint)
  1138  			if err != nil {
  1139  				merr.Errors = append(merr.Errors, err)
  1140  			}
  1141  		}
  1142  	} else {
  1143  		d.logger.Println("[DEBUG] driver.docker: using client connection initialized from environment")
  1144  		newClient, err = docker.NewClientFromEnv()
  1145  		if err != nil {
  1146  			merr.Errors = append(merr.Errors, err)
  1147  		}
  1148  	}
  1149  
  1150  	if timeout != 0 && newClient != nil {
  1151  		newClient.SetTimeout(timeout)
  1152  	}
  1153  	return newClient, merr.ErrorOrNil()
  1154  }
  1155  
  1156  func (d *DockerDriver) containerBinds(driverConfig *DockerDriverConfig, ctx *ExecContext,
  1157  	task *structs.Task) ([]string, error) {
  1158  
  1159  	allocDirBind := fmt.Sprintf("%s:%s", ctx.TaskDir.SharedAllocDir, ctx.TaskEnv.EnvMap[env.AllocDir])
  1160  	taskLocalBind := fmt.Sprintf("%s:%s", ctx.TaskDir.LocalDir, ctx.TaskEnv.EnvMap[env.TaskLocalDir])
  1161  	secretDirBind := fmt.Sprintf("%s:%s", ctx.TaskDir.SecretsDir, ctx.TaskEnv.EnvMap[env.SecretsDir])
  1162  	binds := []string{allocDirBind, taskLocalBind, secretDirBind}
  1163  
  1164  	volumesEnabled := d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault)
  1165  
  1166  	if !volumesEnabled && driverConfig.VolumeDriver != "" {
  1167  		return nil, fmt.Errorf("%s is false; cannot use volume driver %q", dockerVolumesConfigOption, driverConfig.VolumeDriver)
  1168  	}
  1169  
  1170  	for _, userbind := range driverConfig.Volumes {
  1171  		parts := strings.Split(userbind, ":")
  1172  		if len(parts) < 2 {
  1173  			return nil, fmt.Errorf("invalid docker volume: %q", userbind)
  1174  		}
  1175  
  1176  		// Resolve dotted path segments
  1177  		parts[0] = filepath.Clean(parts[0])
  1178  
  1179  		// Absolute paths aren't always supported
  1180  		if filepath.IsAbs(parts[0]) {
  1181  			if !volumesEnabled {
  1182  				// Disallow mounting arbitrary absolute paths
  1183  				return nil, fmt.Errorf("%s is false; cannot mount host paths: %+q", dockerVolumesConfigOption, userbind)
  1184  			}
  1185  			binds = append(binds, userbind)
  1186  			continue
  1187  		}
  1188  
  1189  		// Relative paths are always allowed as they mount within a container
  1190  		// When a VolumeDriver is set, we assume we receive a binding in the format volume-name:container-dest
  1191  		// Otherwise, we assume we receive a relative path binding in the format relative/to/task:/also/in/container
  1192  		if driverConfig.VolumeDriver == "" {
  1193  			// Expand path relative to alloc dir
  1194  			parts[0] = filepath.Join(ctx.TaskDir.Dir, parts[0])
  1195  		}
  1196  
  1197  		binds = append(binds, strings.Join(parts, ":"))
  1198  	}
  1199  
  1200  	if selinuxLabel := d.config.Read(dockerSELinuxLabelConfigOption); selinuxLabel != "" {
  1201  		// Apply SELinux Label to each volume
  1202  		for i := range binds {
  1203  			binds[i] = fmt.Sprintf("%s:%s", binds[i], selinuxLabel)
  1204  		}
  1205  	}
  1206  
  1207  	return binds, nil
  1208  }
  1209  
  1210  // createContainerConfig initializes a struct needed to call docker.client.CreateContainer()
  1211  func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Task,
  1212  	driverConfig *DockerDriverConfig, syslogAddr string) (docker.CreateContainerOptions, error) {
  1213  	var c docker.CreateContainerOptions
  1214  	if task.Resources == nil {
  1215  		// Guard against missing resources. We should never have been able to
  1216  		// schedule a job without specifying this.
  1217  		d.logger.Println("[ERR] driver.docker: task.Resources is empty")
  1218  		return c, fmt.Errorf("task.Resources is empty")
  1219  	}
  1220  
  1221  	binds, err := d.containerBinds(driverConfig, ctx, task)
  1222  	if err != nil {
  1223  		return c, err
  1224  	}
  1225  
  1226  	// create the config block that will later be consumed by go-dockerclient
  1227  	config := &docker.Config{
  1228  		Image:       d.imageID,
  1229  		Entrypoint:  driverConfig.Entrypoint,
  1230  		Hostname:    driverConfig.Hostname,
  1231  		User:        task.User,
  1232  		Tty:         driverConfig.TTY,
  1233  		OpenStdin:   driverConfig.Interactive,
  1234  		StopTimeout: int(task.KillTimeout.Seconds()),
  1235  		StopSignal:  task.KillSignal,
  1236  	}
  1237  
  1238  	if driverConfig.WorkDir != "" {
  1239  		config.WorkingDir = driverConfig.WorkDir
  1240  	}
  1241  
  1242  	memLimit := int64(task.Resources.MemoryMB) * 1024 * 1024
  1243  
  1244  	if len(driverConfig.Logging) == 0 {
  1245  		if runtime.GOOS == "darwin" {
  1246  			d.logger.Printf("[DEBUG] driver.docker: deferring logging to docker on Docker for Mac")
  1247  		} else {
  1248  			d.logger.Printf("[DEBUG] driver.docker: Setting default logging options to syslog and %s", syslogAddr)
  1249  			driverConfig.Logging = []DockerLoggingOpts{
  1250  				{Type: "syslog", Config: map[string]string{"syslog-address": syslogAddr}},
  1251  			}
  1252  		}
  1253  	}
  1254  
  1255  	hostConfig := &docker.HostConfig{
  1256  		// Convert MB to bytes. This is an absolute value.
  1257  		Memory: memLimit,
  1258  		// Convert Mhz to shares. This is a relative value.
  1259  		CPUShares: int64(task.Resources.CPU),
  1260  
  1261  		// Binds are used to mount a host volume into the container. We mount a
  1262  		// local directory for storage and a shared alloc directory that can be
  1263  		// used to share data between different tasks in the same task group.
  1264  		Binds: binds,
  1265  
  1266  		VolumeDriver: driverConfig.VolumeDriver,
  1267  
  1268  		PidsLimit: driverConfig.PidsLimit,
  1269  	}
  1270  
  1271  	// Calculate CPU Quota
  1272  	// cfs_quota_us is the time per core, so we must
  1273  	// multiply the time by the number of cores available
  1274  	// See https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/resource_management_guide/sec-cpu
  1275  	if driverConfig.CPUHardLimit {
  1276  		numCores := runtime.NumCPU()
  1277  		percentTicks := float64(task.Resources.CPU) / float64(d.node.Resources.CPU)
  1278  		hostConfig.CPUQuota = int64(percentTicks*defaultCFSPeriodUS) * int64(numCores)
  1279  	}
  1280  
  1281  	// Windows does not support MemorySwap/MemorySwappiness #2193
  1282  	if runtime.GOOS == "windows" {
  1283  		hostConfig.MemorySwap = 0
  1284  		hostConfig.MemorySwappiness = -1
  1285  	} else {
  1286  		hostConfig.MemorySwap = memLimit // MemorySwap is memory + swap.
  1287  	}
  1288  
  1289  	if len(driverConfig.Logging) != 0 {
  1290  		d.logger.Printf("[DEBUG] driver.docker: Using config for logging: %+v", driverConfig.Logging[0])
  1291  		hostConfig.LogConfig = docker.LogConfig{
  1292  			Type:   driverConfig.Logging[0].Type,
  1293  			Config: driverConfig.Logging[0].Config,
  1294  		}
  1295  	}
  1296  
  1297  	d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name)
  1298  	d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name)
  1299  	if driverConfig.CPUHardLimit {
  1300  		d.logger.Printf("[DEBUG] driver.docker: using %dms cpu quota and %dms cpu period for %s", hostConfig.CPUQuota, defaultCFSPeriodUS, task.Name)
  1301  	}
  1302  	d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name)
  1303  
  1304  	//  set privileged mode
  1305  	hostPrivileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false)
  1306  	if driverConfig.Privileged && !hostPrivileged {
  1307  		return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`)
  1308  	}
  1309  	hostConfig.Privileged = driverConfig.Privileged
  1310  
  1311  	// set capabilities
  1312  	hostCapsWhitelistConfig := d.config.ReadDefault(
  1313  		dockerCapsWhitelistConfigOption, dockerCapsWhitelistConfigDefault)
  1314  	hostCapsWhitelist := make(map[string]struct{})
  1315  	for _, cap := range strings.Split(hostCapsWhitelistConfig, ",") {
  1316  		cap = strings.ToLower(strings.TrimSpace(cap))
  1317  		hostCapsWhitelist[cap] = struct{}{}
  1318  	}
  1319  
  1320  	if _, ok := hostCapsWhitelist["all"]; !ok {
  1321  		effectiveCaps, err := tweakCapabilities(
  1322  			strings.Split(dockerBasicCaps, ","),
  1323  			driverConfig.CapAdd,
  1324  			driverConfig.CapDrop,
  1325  		)
  1326  		if err != nil {
  1327  			return c, err
  1328  		}
  1329  		var missingCaps []string
  1330  		for _, cap := range effectiveCaps {
  1331  			cap = strings.ToLower(cap)
  1332  			if _, ok := hostCapsWhitelist[cap]; !ok {
  1333  				missingCaps = append(missingCaps, cap)
  1334  			}
  1335  		}
  1336  		if len(missingCaps) > 0 {
  1337  			return c, fmt.Errorf("Docker driver doesn't have the following caps whitelisted on this Nomad agent: %s", missingCaps)
  1338  		}
  1339  	}
  1340  
  1341  	hostConfig.CapAdd = driverConfig.CapAdd
  1342  	hostConfig.CapDrop = driverConfig.CapDrop
  1343  
  1344  	// set SHM size
  1345  	if driverConfig.ShmSize != 0 {
  1346  		hostConfig.ShmSize = driverConfig.ShmSize
  1347  	}
  1348  
  1349  	// set DNS servers
  1350  	for _, ip := range driverConfig.DNSServers {
  1351  		if net.ParseIP(ip) != nil {
  1352  			hostConfig.DNS = append(hostConfig.DNS, ip)
  1353  		} else {
  1354  			d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip)
  1355  		}
  1356  	}
  1357  
  1358  	if len(driverConfig.Devices) > 0 {
  1359  		var devices []docker.Device
  1360  		for _, device := range driverConfig.Devices {
  1361  			dev := docker.Device{
  1362  				PathOnHost:        device.HostPath,
  1363  				PathInContainer:   device.ContainerPath,
  1364  				CgroupPermissions: device.CgroupPermissions}
  1365  			devices = append(devices, dev)
  1366  		}
  1367  		hostConfig.Devices = devices
  1368  	}
  1369  
  1370  	// Setup mounts
  1371  	for _, m := range driverConfig.Mounts {
  1372  		hm := docker.HostMount{
  1373  			Target:   m.Target,
  1374  			Source:   m.Source,
  1375  			Type:     m.Type,
  1376  			ReadOnly: m.ReadOnly,
  1377  		}
  1378  
  1379  		if hm.Type == "bind" {
  1380  			volumesEnabled := d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault)
  1381  			hm.Source = filepath.Clean(hm.Source)
  1382  
  1383  			if filepath.IsAbs(hm.Source) {
  1384  				d.logger.Println("[DEBUG] driver.docker: Job contains an absolute path.")
  1385  				if !volumesEnabled {
  1386  					// Disallow mounting arbitrary absolute paths
  1387  					return c, fmt.Errorf("%s is false; cannot bind mount host paths: %+q", dockerVolumesConfigOption, m.Source)
  1388  				}
  1389  			} else {
  1390  				// Expand path relative to alloc dir
  1391  				hm.Source = filepath.Join(ctx.TaskDir.Dir, m.Source)
  1392  				d.logger.Printf("[DEBUG] driver.docker: Job contains a relative path; expanding to %v", hm.Source)
  1393  			}
  1394  
  1395  			if len(m.BindOptions) == 1 {
  1396  				bo := m.BindOptions[0]
  1397  				hm.BindOptions = &docker.BindOptions{
  1398  					Propagation: bo.Propagation,
  1399  				}
  1400  			}
  1401  		} else {
  1402  			if len(m.VolumeOptions) == 1 {
  1403  				vo := m.VolumeOptions[0]
  1404  				hm.VolumeOptions = &docker.VolumeOptions{
  1405  					NoCopy: vo.NoCopy,
  1406  				}
  1407  
  1408  				if len(vo.DriverConfig) == 1 {
  1409  					dc := vo.DriverConfig[0]
  1410  					hm.VolumeOptions.DriverConfig = docker.VolumeDriverConfig{
  1411  						Name: dc.Name,
  1412  					}
  1413  					if len(dc.Options) == 1 {
  1414  						hm.VolumeOptions.DriverConfig.Options = dc.Options[0]
  1415  					}
  1416  				}
  1417  				if len(vo.Labels) == 1 {
  1418  					hm.VolumeOptions.Labels = vo.Labels[0]
  1419  				}
  1420  			}
  1421  		}
  1422  		hostConfig.Mounts = append(hostConfig.Mounts, hm)
  1423  	}
  1424  
  1425  	// set DNS search domains and extra hosts
  1426  	hostConfig.DNSSearch = driverConfig.DNSSearchDomains
  1427  	hostConfig.DNSOptions = driverConfig.DNSOptions
  1428  	hostConfig.ExtraHosts = driverConfig.ExtraHosts
  1429  
  1430  	hostConfig.IpcMode = driverConfig.IpcMode
  1431  	hostConfig.PidMode = driverConfig.PidMode
  1432  	hostConfig.UTSMode = driverConfig.UTSMode
  1433  	hostConfig.UsernsMode = driverConfig.UsernsMode
  1434  	hostConfig.SecurityOpt = driverConfig.SecurityOpt
  1435  	hostConfig.Sysctls = driverConfig.Sysctl
  1436  	hostConfig.Ulimits = driverConfig.Ulimit
  1437  	hostConfig.ReadonlyRootfs = driverConfig.ReadonlyRootfs
  1438  
  1439  	hostConfig.NetworkMode = driverConfig.NetworkMode
  1440  	if hostConfig.NetworkMode == "" {
  1441  		// docker default
  1442  		d.logger.Printf("[DEBUG] driver.docker: networking mode not specified; defaulting to %s", defaultNetworkMode)
  1443  		hostConfig.NetworkMode = defaultNetworkMode
  1444  	}
  1445  
  1446  	// Setup port mapping and exposed ports
  1447  	if len(task.Resources.Networks) == 0 {
  1448  		d.logger.Println("[DEBUG] driver.docker: No network interfaces are available")
  1449  		if len(driverConfig.PortMap) > 0 {
  1450  			return c, fmt.Errorf("Trying to map ports but no network interface is available")
  1451  		}
  1452  	} else {
  1453  		// TODO add support for more than one network
  1454  		network := task.Resources.Networks[0]
  1455  		publishedPorts := map[docker.Port][]docker.PortBinding{}
  1456  		exposedPorts := map[docker.Port]struct{}{}
  1457  
  1458  		for _, port := range network.ReservedPorts {
  1459  			// By default we will map the allocated port 1:1 to the container
  1460  			containerPortInt := port.Value
  1461  
  1462  			// If the user has mapped a port using port_map we'll change it here
  1463  			if mapped, ok := driverConfig.PortMap[port.Label]; ok {
  1464  				containerPortInt = mapped
  1465  			}
  1466  
  1467  			hostPortStr := strconv.Itoa(port.Value)
  1468  			containerPort := docker.Port(strconv.Itoa(containerPortInt))
  1469  
  1470  			publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr)
  1471  			publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr)
  1472  			d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value)
  1473  
  1474  			exposedPorts[containerPort+"/tcp"] = struct{}{}
  1475  			exposedPorts[containerPort+"/udp"] = struct{}{}
  1476  			d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value)
  1477  		}
  1478  
  1479  		for _, port := range network.DynamicPorts {
  1480  			// By default we will map the allocated port 1:1 to the container
  1481  			containerPortInt := port.Value
  1482  
  1483  			// If the user has mapped a port using port_map we'll change it here
  1484  			if mapped, ok := driverConfig.PortMap[port.Label]; ok {
  1485  				containerPortInt = mapped
  1486  			}
  1487  
  1488  			hostPortStr := strconv.Itoa(port.Value)
  1489  			containerPort := docker.Port(strconv.Itoa(containerPortInt))
  1490  
  1491  			publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr)
  1492  			publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr)
  1493  			d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt)
  1494  
  1495  			exposedPorts[containerPort+"/tcp"] = struct{}{}
  1496  			exposedPorts[containerPort+"/udp"] = struct{}{}
  1497  			d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort)
  1498  		}
  1499  
  1500  		hostConfig.PortBindings = publishedPorts
  1501  		config.ExposedPorts = exposedPorts
  1502  	}
  1503  
  1504  	parsedArgs := ctx.TaskEnv.ParseAndReplace(driverConfig.Args)
  1505  
  1506  	// If the user specified a custom command to run, we'll inject it here.
  1507  	if driverConfig.Command != "" {
  1508  		// Validate command
  1509  		if err := validateCommand(driverConfig.Command, "args"); err != nil {
  1510  			return c, err
  1511  		}
  1512  
  1513  		cmd := []string{driverConfig.Command}
  1514  		if len(driverConfig.Args) != 0 {
  1515  			cmd = append(cmd, parsedArgs...)
  1516  		}
  1517  		d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
  1518  		config.Cmd = cmd
  1519  	} else if len(driverConfig.Args) != 0 {
  1520  		config.Cmd = parsedArgs
  1521  	}
  1522  
  1523  	if len(driverConfig.Labels) > 0 {
  1524  		config.Labels = driverConfig.Labels
  1525  		d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels)
  1526  	}
  1527  
  1528  	config.Env = ctx.TaskEnv.List()
  1529  
  1530  	containerName := fmt.Sprintf("%s-%s", task.Name, d.DriverContext.allocID)
  1531  	d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName)
  1532  
  1533  	var networkingConfig *docker.NetworkingConfig
  1534  	if len(driverConfig.NetworkAliases) > 0 || driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" {
  1535  		networkingConfig = &docker.NetworkingConfig{
  1536  			EndpointsConfig: map[string]*docker.EndpointConfig{
  1537  				hostConfig.NetworkMode: {},
  1538  			},
  1539  		}
  1540  	}
  1541  
  1542  	if len(driverConfig.NetworkAliases) > 0 {
  1543  		networkingConfig.EndpointsConfig[hostConfig.NetworkMode].Aliases = driverConfig.NetworkAliases
  1544  		d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with network aliases: %v",
  1545  			hostConfig.NetworkMode, strings.Join(driverConfig.NetworkAliases, ", "))
  1546  	}
  1547  
  1548  	if driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" {
  1549  		networkingConfig.EndpointsConfig[hostConfig.NetworkMode].IPAMConfig = &docker.EndpointIPAMConfig{
  1550  			IPv4Address: driverConfig.IPv4Address,
  1551  			IPv6Address: driverConfig.IPv6Address,
  1552  		}
  1553  		d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with ipv4: %q and ipv6: %q",
  1554  			hostConfig.NetworkMode, driverConfig.IPv4Address, driverConfig.IPv6Address)
  1555  	}
  1556  
  1557  	if driverConfig.MacAddress != "" {
  1558  		config.MacAddress = driverConfig.MacAddress
  1559  		d.logger.Printf("[DEBUG] driver.docker: using pinned mac address: %q", config.MacAddress)
  1560  	}
  1561  
  1562  	return docker.CreateContainerOptions{
  1563  		Name:             containerName,
  1564  		Config:           config,
  1565  		HostConfig:       hostConfig,
  1566  		NetworkingConfig: networkingConfig,
  1567  	}, nil
  1568  }
  1569  
  1570  func (d *DockerDriver) Periodic() (bool, time.Duration) {
  1571  	return true, 15 * time.Second
  1572  }
  1573  
  1574  // createImage creates a docker image either by pulling it from a registry or by
  1575  // loading it from the file system
  1576  func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir *allocdir.TaskDir) (string, error) {
  1577  	image := driverConfig.ImageName
  1578  	repo, tag := parseDockerImage(image)
  1579  
  1580  	coordinator, callerID := d.getDockerCoordinator(client)
  1581  
  1582  	// We're going to check whether the image is already downloaded. If the tag
  1583  	// is "latest", or ForcePull is set, we have to check for a new version every time so we don't
  1584  	// bother to check and cache the id here. We'll download first, then cache.
  1585  	if driverConfig.ForcePull {
  1586  		d.logger.Printf("[DEBUG] driver.docker: force pull image '%s' instead of inspecting local", dockerImageRef(repo, tag))
  1587  	} else if tag != "latest" {
  1588  		if dockerImage, _ := client.InspectImage(image); dockerImage != nil {
  1589  			// Image exists so just increment its reference count
  1590  			coordinator.IncrementImageReference(dockerImage.ID, image, callerID)
  1591  			return dockerImage.ID, nil
  1592  		}
  1593  	}
  1594  
  1595  	// Load the image if specified
  1596  	if driverConfig.LoadImage != "" {
  1597  		return d.loadImage(driverConfig, client, taskDir)
  1598  	}
  1599  
  1600  	// Download the image
  1601  	return d.pullImage(driverConfig, client, repo, tag)
  1602  }
  1603  
  1604  // pullImage creates an image by pulling it from a docker registry
  1605  func (d *DockerDriver) pullImage(driverConfig *DockerDriverConfig, client *docker.Client, repo, tag string) (id string, err error) {
  1606  	authOptions, err := d.resolveRegistryAuthentication(driverConfig, repo)
  1607  	if err != nil {
  1608  		if d.driverConfig.AuthSoftFail {
  1609  			d.logger.Printf("[WARN] Failed to find docker auth for repo %q: %v", repo, err)
  1610  		} else {
  1611  			return "", fmt.Errorf("Failed to find docker auth for repo %q: %v", repo, err)
  1612  		}
  1613  	}
  1614  
  1615  	if authIsEmpty(authOptions) {
  1616  		d.logger.Printf("[DEBUG] driver.docker: did not find docker auth for repo %q", repo)
  1617  	}
  1618  
  1619  	d.emitEvent("Downloading image %s", dockerImageRef(repo, tag))
  1620  	coordinator, callerID := d.getDockerCoordinator(client)
  1621  
  1622  	return coordinator.PullImage(driverConfig.ImageName, authOptions, callerID, d.emitEvent)
  1623  }
  1624  
  1625  // authBackend encapsulates a function that resolves registry credentials.
  1626  type authBackend func(string) (*docker.AuthConfiguration, error)
  1627  
  1628  // resolveRegistryAuthentication attempts to retrieve auth credentials for the
  1629  // repo, trying all authentication-backends possible.
  1630  func (d *DockerDriver) resolveRegistryAuthentication(driverConfig *DockerDriverConfig, repo string) (*docker.AuthConfiguration, error) {
  1631  	return firstValidAuth(repo, []authBackend{
  1632  		authFromTaskConfig(driverConfig),
  1633  		authFromDockerConfig(d.config.Read("docker.auth.config")),
  1634  		authFromHelper(d.config.Read("docker.auth.helper")),
  1635  	})
  1636  }
  1637  
  1638  // loadImage creates an image by loading it from the file system
  1639  func (d *DockerDriver) loadImage(driverConfig *DockerDriverConfig, client *docker.Client,
  1640  	taskDir *allocdir.TaskDir) (id string, err error) {
  1641  
  1642  	archive := filepath.Join(taskDir.LocalDir, driverConfig.LoadImage)
  1643  	d.logger.Printf("[DEBUG] driver.docker: loading image from: %v", archive)
  1644  
  1645  	f, err := os.Open(archive)
  1646  	if err != nil {
  1647  		return "", fmt.Errorf("unable to open image archive: %v", err)
  1648  	}
  1649  
  1650  	if err := client.LoadImage(docker.LoadImageOptions{InputStream: f}); err != nil {
  1651  		return "", err
  1652  	}
  1653  	f.Close()
  1654  
  1655  	dockerImage, err := client.InspectImage(driverConfig.ImageName)
  1656  	if err != nil {
  1657  		return "", recoverableErrTimeouts(err)
  1658  	}
  1659  
  1660  	coordinator, callerID := d.getDockerCoordinator(client)
  1661  	coordinator.IncrementImageReference(dockerImage.ID, driverConfig.ImageName, callerID)
  1662  	return dockerImage.ID, nil
  1663  }
  1664  
  1665  // createContainer creates the container given the passed configuration. It
  1666  // attempts to handle any transient Docker errors.
  1667  func (d *DockerDriver) createContainer(client createContainerClient, config docker.CreateContainerOptions) (*docker.Container, error) {
  1668  	// Create a container
  1669  	attempted := 0
  1670  CREATE:
  1671  	container, createErr := client.CreateContainer(config)
  1672  	if createErr == nil {
  1673  		return container, nil
  1674  	}
  1675  
  1676  	d.logger.Printf("[DEBUG] driver.docker: failed to create container %q from image %q (ID: %q) (attempt %d): %v",
  1677  		config.Name, d.driverConfig.ImageName, d.imageID, attempted+1, createErr)
  1678  
  1679  	// Volume management tools like Portworx may not have detached a volume
  1680  	// from a previous node before Nomad started a task replacement task.
  1681  	// Treat these errors as recoverable so we retry.
  1682  	if strings.Contains(strings.ToLower(createErr.Error()), "volume is attached on another node") {
  1683  		return nil, structs.NewRecoverableError(createErr, true)
  1684  	}
  1685  
  1686  	// If the container already exists determine whether it's already
  1687  	// running or if it's dead and needs to be recreated.
  1688  	if strings.Contains(strings.ToLower(createErr.Error()), "container already exists") {
  1689  		containers, err := client.ListContainers(docker.ListContainersOptions{
  1690  			All: true,
  1691  		})
  1692  		if err != nil {
  1693  			d.logger.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name)
  1694  			return nil, recoverableErrTimeouts(fmt.Errorf("Failed to query list of containers: %s", err))
  1695  		}
  1696  
  1697  		// Delete matching containers
  1698  		// Adding a / infront of the container name since Docker returns the
  1699  		// container names with a / pre-pended to the Nomad generated container names
  1700  		containerName := "/" + config.Name
  1701  		d.logger.Printf("[DEBUG] driver.docker: searching for container name %q to purge", containerName)
  1702  		for _, shimContainer := range containers {
  1703  			d.logger.Printf("[DEBUG] driver.docker: listed container %+v", shimContainer.Names)
  1704  			found := false
  1705  			for _, name := range shimContainer.Names {
  1706  				if name == containerName {
  1707  					d.logger.Printf("[DEBUG] driver.docker: Found container %v: %v", containerName, shimContainer.ID)
  1708  					found = true
  1709  					break
  1710  				}
  1711  			}
  1712  
  1713  			if !found {
  1714  				continue
  1715  			}
  1716  
  1717  			// Inspect the container and if the container isn't dead then return
  1718  			// the container
  1719  			container, err := client.InspectContainer(shimContainer.ID)
  1720  			if err != nil {
  1721  				err = fmt.Errorf("Failed to inspect container %s: %s", shimContainer.ID, err)
  1722  
  1723  				// This error is always recoverable as it could
  1724  				// be caused by races between listing
  1725  				// containers and this container being removed.
  1726  				// See #2802
  1727  				return nil, structs.NewRecoverableError(err, true)
  1728  			}
  1729  			if container != nil && container.State.Running {
  1730  				return container, nil
  1731  			}
  1732  
  1733  			err = client.RemoveContainer(docker.RemoveContainerOptions{
  1734  				ID:    container.ID,
  1735  				Force: true,
  1736  			})
  1737  			if err != nil {
  1738  				d.logger.Printf("[ERR] driver.docker: failed to purge container %s", container.ID)
  1739  				return nil, recoverableErrTimeouts(fmt.Errorf("Failed to purge container %s: %s", container.ID, err))
  1740  			} else if err == nil {
  1741  				d.logger.Printf("[INFO] driver.docker: purged container %s", container.ID)
  1742  			}
  1743  		}
  1744  
  1745  		if attempted < 5 {
  1746  			attempted++
  1747  			time.Sleep(1 * time.Second)
  1748  			goto CREATE
  1749  		}
  1750  	} else if strings.Contains(strings.ToLower(createErr.Error()), "no such image") {
  1751  		// There is still a very small chance this is possible even with the
  1752  		// coordinator so retry.
  1753  		return nil, structs.NewRecoverableError(createErr, true)
  1754  	}
  1755  
  1756  	return nil, recoverableErrTimeouts(createErr)
  1757  }
  1758  
  1759  // startContainer starts the passed container. It attempts to handle any
  1760  // transient Docker errors.
  1761  func (d *DockerDriver) startContainer(c *docker.Container) error {
  1762  	// Start a container
  1763  	attempted := 0
  1764  START:
  1765  	startErr := client.StartContainer(c.ID, c.HostConfig)
  1766  	if startErr == nil {
  1767  		return nil
  1768  	}
  1769  
  1770  	d.logger.Printf("[DEBUG] driver.docker: failed to start container %q (attempt %d): %v", c.ID, attempted+1, startErr)
  1771  
  1772  	// If it is a 500 error it is likely we can retry and be successful
  1773  	if strings.Contains(startErr.Error(), "API error (500)") {
  1774  		if attempted < 5 {
  1775  			attempted++
  1776  			time.Sleep(1 * time.Second)
  1777  			goto START
  1778  		}
  1779  		return structs.NewRecoverableError(startErr, true)
  1780  	}
  1781  
  1782  	return recoverableErrTimeouts(startErr)
  1783  }
  1784  
  1785  func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
  1786  	// Split the handle
  1787  	pidBytes := []byte(strings.TrimPrefix(handleID, "DOCKER:"))
  1788  	pid := &dockerPID{}
  1789  	if err := json.Unmarshal(pidBytes, pid); err != nil {
  1790  		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
  1791  	}
  1792  	d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", pid.ContainerID)
  1793  	d.logger.Printf("[DEBUG] driver.docker: re-attached to handle: %s", handleID)
  1794  	pluginConfig := &plugin.ClientConfig{
  1795  		Reattach: pid.PluginConfig.PluginConfig(),
  1796  	}
  1797  
  1798  	client, waitClient, err := d.dockerClients()
  1799  	if err != nil {
  1800  		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
  1801  	}
  1802  
  1803  	// Look for a running container with this ID
  1804  	containers, err := client.ListContainers(docker.ListContainersOptions{
  1805  		Filters: map[string][]string{
  1806  			"id": {pid.ContainerID},
  1807  		},
  1808  	})
  1809  	if err != nil {
  1810  		return nil, fmt.Errorf("Failed to query for container %s: %v", pid.ContainerID, err)
  1811  	}
  1812  
  1813  	found := false
  1814  	for _, container := range containers {
  1815  		if container.ID == pid.ContainerID {
  1816  			found = true
  1817  		}
  1818  	}
  1819  	if !found {
  1820  		return nil, fmt.Errorf("Failed to find container %s", pid.ContainerID)
  1821  	}
  1822  	exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
  1823  	if err != nil {
  1824  		d.logger.Printf("[INFO] driver.docker: couldn't re-attach to the plugin process: %v", err)
  1825  		d.logger.Printf("[DEBUG] driver.docker: stopping container %q", pid.ContainerID)
  1826  		if e := client.StopContainer(pid.ContainerID, uint(pid.KillTimeout.Seconds())); e != nil {
  1827  			d.logger.Printf("[DEBUG] driver.docker: couldn't stop container: %v", e)
  1828  		}
  1829  		return nil, err
  1830  	}
  1831  
  1832  	ver, _ := exec.Version()
  1833  	d.logger.Printf("[DEBUG] driver.docker: version of executor: %v", ver.Version)
  1834  
  1835  	// Increment the reference count since we successfully attached to this
  1836  	// container
  1837  	coordinator, callerID := d.getDockerCoordinator(client)
  1838  	coordinator.IncrementImageReference(pid.ImageID, pid.Image, callerID)
  1839  
  1840  	// Return a driver handle
  1841  	h := &DockerHandle{
  1842  		client:         client,
  1843  		waitClient:     waitClient,
  1844  		executor:       exec,
  1845  		pluginClient:   pluginClient,
  1846  		logger:         d.logger,
  1847  		jobName:        d.DriverContext.jobName,
  1848  		taskGroupName:  d.DriverContext.taskGroupName,
  1849  		taskName:       d.DriverContext.taskName,
  1850  		Image:          pid.Image,
  1851  		ImageID:        pid.ImageID,
  1852  		containerID:    pid.ContainerID,
  1853  		version:        pid.Version,
  1854  		killTimeout:    pid.KillTimeout,
  1855  		maxKillTimeout: pid.MaxKillTimeout,
  1856  		doneCh:         make(chan bool),
  1857  		waitCh:         make(chan *dstructs.WaitResult, 1),
  1858  	}
  1859  	go h.collectStats()
  1860  	go h.run()
  1861  	return h, nil
  1862  }
  1863  
  1864  func (h *DockerHandle) ID() string {
  1865  	// Return a handle to the PID
  1866  	pid := dockerPID{
  1867  		Version:        h.version,
  1868  		ContainerID:    h.containerID,
  1869  		Image:          h.Image,
  1870  		ImageID:        h.ImageID,
  1871  		KillTimeout:    h.killTimeout,
  1872  		MaxKillTimeout: h.maxKillTimeout,
  1873  		PluginConfig:   NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
  1874  	}
  1875  	data, err := json.Marshal(pid)
  1876  	if err != nil {
  1877  		h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s", err)
  1878  	}
  1879  	return fmt.Sprintf("DOCKER:%s", string(data))
  1880  }
  1881  
  1882  func (h *DockerHandle) ContainerID() string {
  1883  	return h.containerID
  1884  }
  1885  
  1886  func (h *DockerHandle) WaitCh() chan *dstructs.WaitResult {
  1887  	return h.waitCh
  1888  }
  1889  
  1890  func (h *DockerHandle) Update(task *structs.Task) error {
  1891  	// Store the updated kill timeout.
  1892  	h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
  1893  	if err := h.executor.UpdateTask(task); err != nil {
  1894  		h.logger.Printf("[DEBUG] driver.docker: failed to update log config: %v", err)
  1895  	}
  1896  
  1897  	// Update is not possible
  1898  	return nil
  1899  }
  1900  
  1901  func (h *DockerHandle) Exec(ctx context.Context, cmd string, args []string) ([]byte, int, error) {
  1902  	fullCmd := make([]string, len(args)+1)
  1903  	fullCmd[0] = cmd
  1904  	copy(fullCmd[1:], args)
  1905  	createExecOpts := docker.CreateExecOptions{
  1906  		AttachStdin:  false,
  1907  		AttachStdout: true,
  1908  		AttachStderr: true,
  1909  		Tty:          false,
  1910  		Cmd:          fullCmd,
  1911  		Container:    h.containerID,
  1912  		Context:      ctx,
  1913  	}
  1914  	exec, err := h.client.CreateExec(createExecOpts)
  1915  	if err != nil {
  1916  		return nil, 0, err
  1917  	}
  1918  
  1919  	output, _ := circbuf.NewBuffer(int64(dstructs.CheckBufSize))
  1920  	startOpts := docker.StartExecOptions{
  1921  		Detach:       false,
  1922  		Tty:          false,
  1923  		OutputStream: output,
  1924  		ErrorStream:  output,
  1925  		Context:      ctx,
  1926  	}
  1927  	if err := client.StartExec(exec.ID, startOpts); err != nil {
  1928  		return nil, 0, err
  1929  	}
  1930  	res, err := client.InspectExec(exec.ID)
  1931  	if err != nil {
  1932  		return output.Bytes(), 0, err
  1933  	}
  1934  	return output.Bytes(), res.ExitCode, nil
  1935  }
  1936  
  1937  func (h *DockerHandle) Signal(s os.Signal) error {
  1938  	// Convert types
  1939  	sysSig, ok := s.(syscall.Signal)
  1940  	if !ok {
  1941  		return fmt.Errorf("Failed to determine signal number")
  1942  	}
  1943  
  1944  	// TODO When we expose signals we will need a mapping layer that converts
  1945  	// MacOS signals to the correct signal number for docker. Or we change the
  1946  	// interface to take a signal string and leave it up to driver to map?
  1947  
  1948  	dockerSignal := docker.Signal(sysSig)
  1949  	opts := docker.KillContainerOptions{
  1950  		ID:     h.containerID,
  1951  		Signal: dockerSignal,
  1952  	}
  1953  	return h.client.KillContainer(opts)
  1954  
  1955  }
  1956  
  1957  // Kill is used to terminate the task. This uses `docker stop -t killTimeout`
  1958  func (h *DockerHandle) Kill() error {
  1959  	// Stop the container
  1960  	err := h.waitClient.StopContainer(h.containerID, uint(h.killTimeout.Seconds()))
  1961  	if err != nil {
  1962  		h.executor.Exit()
  1963  		h.pluginClient.Kill()
  1964  
  1965  		// Container has already been removed.
  1966  		if strings.Contains(err.Error(), NoSuchContainerError) {
  1967  			h.logger.Printf("[DEBUG] driver.docker: attempted to stop nonexistent container %s", h.containerID)
  1968  			return nil
  1969  		}
  1970  		h.logger.Printf("[ERR] driver.docker: failed to stop container %s: %v", h.containerID, err)
  1971  		return fmt.Errorf("Failed to stop container %s: %s", h.containerID, err)
  1972  	}
  1973  	h.logger.Printf("[INFO] driver.docker: stopped container %s", h.containerID)
  1974  	return nil
  1975  }
  1976  
  1977  func (h *DockerHandle) Stats() (*cstructs.TaskResourceUsage, error) {
  1978  	h.resourceUsageLock.RLock()
  1979  	defer h.resourceUsageLock.RUnlock()
  1980  	var err error
  1981  	if h.resourceUsage == nil {
  1982  		err = fmt.Errorf("stats collection hasn't started yet")
  1983  	}
  1984  	return h.resourceUsage, err
  1985  }
  1986  
  1987  func (h *DockerHandle) run() {
  1988  	// Wait for it...
  1989  	exitCode, werr := h.waitClient.WaitContainer(h.containerID)
  1990  	if werr != nil {
  1991  		h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated", h.containerID)
  1992  	}
  1993  
  1994  	if exitCode != 0 {
  1995  		werr = fmt.Errorf("Docker container exited with non-zero exit code: %d", exitCode)
  1996  	}
  1997  
  1998  	container, ierr := h.waitClient.InspectContainer(h.containerID)
  1999  	if ierr != nil {
  2000  		h.logger.Printf("[ERR] driver.docker: failed to inspect container %s: %v", h.containerID, ierr)
  2001  	} else if container.State.OOMKilled {
  2002  		werr = fmt.Errorf("OOM Killed")
  2003  		labels := []metrics.Label{
  2004  			{
  2005  				Name:  "job",
  2006  				Value: h.jobName,
  2007  			},
  2008  			{
  2009  				Name:  "task_group",
  2010  				Value: h.taskGroupName,
  2011  			},
  2012  			{
  2013  				Name:  "task",
  2014  				Value: h.taskName,
  2015  			},
  2016  		}
  2017  		metrics.IncrCounterWithLabels([]string{"driver", "docker", "oom"}, 1, labels)
  2018  	}
  2019  
  2020  	close(h.doneCh)
  2021  
  2022  	// Shutdown the syslog collector
  2023  	if err := h.executor.Exit(); err != nil {
  2024  		h.logger.Printf("[ERR] driver.docker: failed to kill the syslog collector: %v", err)
  2025  	}
  2026  	h.pluginClient.Kill()
  2027  
  2028  	// Stop the container just incase the docker daemon's wait returned
  2029  	// incorrectly
  2030  	if err := h.client.StopContainer(h.containerID, 0); err != nil {
  2031  		_, noSuchContainer := err.(*docker.NoSuchContainer)
  2032  		_, containerNotRunning := err.(*docker.ContainerNotRunning)
  2033  		if !containerNotRunning && !noSuchContainer {
  2034  			h.logger.Printf("[ERR] driver.docker: error stopping container: %v", err)
  2035  		}
  2036  	}
  2037  
  2038  	// Remove the container
  2039  	if h.removeContainerOnExit == true {
  2040  		if err := h.client.RemoveContainer(docker.RemoveContainerOptions{ID: h.containerID, RemoveVolumes: true, Force: true}); err != nil {
  2041  			h.logger.Printf("[ERR] driver.docker: error removing container: %v", err)
  2042  		}
  2043  	} else {
  2044  		h.logger.Printf("[DEBUG] driver.docker: not removing container %v because of config", h.containerID)
  2045  	}
  2046  
  2047  	// Send the results
  2048  	h.waitCh <- dstructs.NewWaitResult(exitCode, 0, werr)
  2049  	close(h.waitCh)
  2050  }
  2051  
  2052  // collectStats starts collecting resource usage stats of a docker container
  2053  func (h *DockerHandle) collectStats() {
  2054  	statsCh := make(chan *docker.Stats)
  2055  	statsOpts := docker.StatsOptions{ID: h.containerID, Done: h.doneCh, Stats: statsCh, Stream: true}
  2056  	go func() {
  2057  		//TODO handle Stats error
  2058  		if err := h.waitClient.Stats(statsOpts); err != nil {
  2059  			h.logger.Printf("[DEBUG] driver.docker: error collecting stats from container %s: %v", h.containerID, err)
  2060  		}
  2061  	}()
  2062  	numCores := runtime.NumCPU()
  2063  	for {
  2064  		select {
  2065  		case s := <-statsCh:
  2066  			if s != nil {
  2067  				ms := &cstructs.MemoryStats{
  2068  					RSS:      s.MemoryStats.Stats.Rss,
  2069  					Cache:    s.MemoryStats.Stats.Cache,
  2070  					Swap:     s.MemoryStats.Stats.Swap,
  2071  					MaxUsage: s.MemoryStats.MaxUsage,
  2072  					Measured: DockerMeasuredMemStats,
  2073  				}
  2074  
  2075  				cs := &cstructs.CpuStats{
  2076  					ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
  2077  					ThrottledTime:    s.CPUStats.ThrottlingData.ThrottledTime,
  2078  					Measured:         DockerMeasuredCpuStats,
  2079  				}
  2080  
  2081  				// Calculate percentage
  2082  				cs.Percent = calculatePercent(
  2083  					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage,
  2084  					s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, numCores)
  2085  				cs.SystemMode = calculatePercent(
  2086  					s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode,
  2087  					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, numCores)
  2088  				cs.UserMode = calculatePercent(
  2089  					s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
  2090  					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, numCores)
  2091  				cs.TotalTicks = (cs.Percent / 100) * shelpers.TotalTicksAvailable() / float64(numCores)
  2092  
  2093  				h.resourceUsageLock.Lock()
  2094  				h.resourceUsage = &cstructs.TaskResourceUsage{
  2095  					ResourceUsage: &cstructs.ResourceUsage{
  2096  						MemoryStats: ms,
  2097  						CpuStats:    cs,
  2098  					},
  2099  					Timestamp: s.Read.UTC().UnixNano(),
  2100  				}
  2101  				h.resourceUsageLock.Unlock()
  2102  			}
  2103  		case <-h.doneCh:
  2104  			return
  2105  		}
  2106  	}
  2107  }
  2108  
  2109  func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 {
  2110  	numerator := newSample - oldSample
  2111  	denom := newTotal - oldTotal
  2112  	if numerator <= 0 || denom <= 0 {
  2113  		return 0.0
  2114  	}
  2115  
  2116  	return (float64(numerator) / float64(denom)) * float64(cores) * 100.0
  2117  }
  2118  
  2119  // loadDockerConfig loads the docker config at the specified path, returning an
  2120  // error if it couldn't be read.
  2121  func loadDockerConfig(file string) (*configfile.ConfigFile, error) {
  2122  	f, err := os.Open(file)
  2123  	if err != nil {
  2124  		return nil, fmt.Errorf("Failed to open auth config file: %v, error: %v", file, err)
  2125  	}
  2126  	defer f.Close()
  2127  
  2128  	cfile := new(configfile.ConfigFile)
  2129  	if err = cfile.LoadFromReader(f); err != nil {
  2130  		return nil, fmt.Errorf("Failed to parse auth config file: %v", err)
  2131  	}
  2132  	return cfile, nil
  2133  }
  2134  
  2135  // parseRepositoryInfo takes a repo and returns the Docker RepositoryInfo. This
  2136  // is useful for interacting with a Docker config object.
  2137  func parseRepositoryInfo(repo string) (*registry.RepositoryInfo, error) {
  2138  	name, err := reference.ParseNamed(repo)
  2139  	if err != nil {
  2140  		return nil, fmt.Errorf("Failed to parse named repo %q: %v", repo, err)
  2141  	}
  2142  
  2143  	repoInfo, err := registry.ParseRepositoryInfo(name)
  2144  	if err != nil {
  2145  		return nil, fmt.Errorf("Failed to parse repository: %v", err)
  2146  	}
  2147  
  2148  	return repoInfo, nil
  2149  }
  2150  
  2151  // firstValidAuth tries a list of auth backends, returning first error or AuthConfiguration
  2152  func firstValidAuth(repo string, backends []authBackend) (*docker.AuthConfiguration, error) {
  2153  	for _, backend := range backends {
  2154  		auth, err := backend(repo)
  2155  		if auth != nil || err != nil {
  2156  			return auth, err
  2157  		}
  2158  	}
  2159  	return nil, nil
  2160  }
  2161  
  2162  // authFromTaskConfig generates an authBackend for any auth given in the task-configuration
  2163  func authFromTaskConfig(driverConfig *DockerDriverConfig) authBackend {
  2164  	return func(string) (*docker.AuthConfiguration, error) {
  2165  		if len(driverConfig.Auth) == 0 {
  2166  			return nil, nil
  2167  		}
  2168  		auth := driverConfig.Auth[0]
  2169  		return &docker.AuthConfiguration{
  2170  			Username:      auth.Username,
  2171  			Password:      auth.Password,
  2172  			Email:         auth.Email,
  2173  			ServerAddress: auth.ServerAddress,
  2174  		}, nil
  2175  	}
  2176  }
  2177  
  2178  // authFromDockerConfig generate an authBackend for a dockercfg-compatible file.
  2179  // The authBacken can either be from explicit auth definitions or via credential
  2180  // helpers
  2181  func authFromDockerConfig(file string) authBackend {
  2182  	return func(repo string) (*docker.AuthConfiguration, error) {
  2183  		if file == "" {
  2184  			return nil, nil
  2185  		}
  2186  		repoInfo, err := parseRepositoryInfo(repo)
  2187  		if err != nil {
  2188  			return nil, err
  2189  		}
  2190  
  2191  		cfile, err := loadDockerConfig(file)
  2192  		if err != nil {
  2193  			return nil, err
  2194  		}
  2195  
  2196  		return firstValidAuth(repo, []authBackend{
  2197  			func(string) (*docker.AuthConfiguration, error) {
  2198  				dockerAuthConfig := registry.ResolveAuthConfig(cfile.AuthConfigs, repoInfo.Index)
  2199  				auth := &docker.AuthConfiguration{
  2200  					Username:      dockerAuthConfig.Username,
  2201  					Password:      dockerAuthConfig.Password,
  2202  					Email:         dockerAuthConfig.Email,
  2203  					ServerAddress: dockerAuthConfig.ServerAddress,
  2204  				}
  2205  				if authIsEmpty(auth) {
  2206  					return nil, nil
  2207  				}
  2208  				return auth, nil
  2209  			},
  2210  			authFromHelper(cfile.CredentialHelpers[registry.GetAuthConfigKey(repoInfo.Index)]),
  2211  			authFromHelper(cfile.CredentialsStore),
  2212  		})
  2213  	}
  2214  }
  2215  
  2216  // authFromHelper generates an authBackend for a docker-credentials-helper;
  2217  // A script taking the requested domain on input, outputting JSON with
  2218  // "Username" and "Secret"
  2219  func authFromHelper(helperName string) authBackend {
  2220  	return func(repo string) (*docker.AuthConfiguration, error) {
  2221  		if helperName == "" {
  2222  			return nil, nil
  2223  		}
  2224  		helper := dockerAuthHelperPrefix + helperName
  2225  		cmd := exec.Command(helper, "get")
  2226  
  2227  		repoParsed, err := reference.ParseNamed(repo)
  2228  		if err != nil {
  2229  			return nil, err
  2230  		}
  2231  
  2232  		// Ensure that the HTTPs prefix exists
  2233  		repoAddr := fmt.Sprintf("https://%s", repoParsed.Hostname())
  2234  
  2235  		cmd.Stdin = strings.NewReader(repoAddr)
  2236  		output, err := cmd.Output()
  2237  		if err != nil {
  2238  			switch err.(type) {
  2239  			default:
  2240  				return nil, err
  2241  			case *exec.ExitError:
  2242  				return nil, fmt.Errorf("%s with input %q failed with stderr: %s", helper, repo, output)
  2243  			}
  2244  		}
  2245  
  2246  		var response map[string]string
  2247  		if err := json.Unmarshal(output, &response); err != nil {
  2248  			return nil, err
  2249  		}
  2250  
  2251  		auth := &docker.AuthConfiguration{
  2252  			Username: response["Username"],
  2253  			Password: response["Secret"],
  2254  		}
  2255  
  2256  		if authIsEmpty(auth) {
  2257  			return nil, nil
  2258  		}
  2259  		return auth, nil
  2260  	}
  2261  }
  2262  
  2263  // authIsEmpty returns if auth is nil or an empty structure
  2264  func authIsEmpty(auth *docker.AuthConfiguration) bool {
  2265  	if auth == nil {
  2266  		return false
  2267  	}
  2268  	return auth.Username == "" &&
  2269  		auth.Password == "" &&
  2270  		auth.Email == "" &&
  2271  		auth.ServerAddress == ""
  2272  }
  2273  
  2274  // createContainerClient is the subset of Docker Client methods used by the
  2275  // createContainer method to ease testing subtle error conditions.
  2276  type createContainerClient interface {
  2277  	CreateContainer(docker.CreateContainerOptions) (*docker.Container, error)
  2278  	InspectContainer(id string) (*docker.Container, error)
  2279  	ListContainers(docker.ListContainersOptions) ([]docker.APIContainers, error)
  2280  	RemoveContainer(opts docker.RemoveContainerOptions) error
  2281  }
  2282  
  2283  func parseDockerImage(image string) (repo, tag string) {
  2284  	repo, tag = docker.ParseRepositoryTag(image)
  2285  	if tag != "" {
  2286  		return repo, tag
  2287  	}
  2288  	if i := strings.IndexRune(image, '@'); i > -1 { // Has digest (@sha256:...)
  2289  		// when pulling images with a digest, the repository contains the sha hash, and the tag is empty
  2290  		// see: https://github.com/fsouza/go-dockerclient/blob/master/image_test.go#L471
  2291  		repo = image
  2292  	} else {
  2293  		tag = "latest"
  2294  	}
  2295  	return repo, tag
  2296  }
  2297  
  2298  func dockerImageRef(repo string, tag string) string {
  2299  	if tag == "" {
  2300  		return repo
  2301  	}
  2302  	return fmt.Sprintf("%s:%s", repo, tag)
  2303  }