github.com/taylorchu/nomad@v0.5.3-rc1.0.20170407200202-db11e7dd7b55/client/driver/docker.go (about)

     1  package driver
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"log"
     7  	"net"
     8  	"os"
     9  	"path/filepath"
    10  	"runtime"
    11  	"strconv"
    12  	"strings"
    13  	"sync"
    14  	"syscall"
    15  	"time"
    16  
    17  	docker "github.com/fsouza/go-dockerclient"
    18  
    19  	"github.com/docker/docker/cli/config/configfile"
    20  	"github.com/docker/docker/reference"
    21  	"github.com/docker/docker/registry"
    22  
    23  	"github.com/hashicorp/go-multierror"
    24  	"github.com/hashicorp/go-plugin"
    25  	"github.com/hashicorp/nomad/client/allocdir"
    26  	"github.com/hashicorp/nomad/client/config"
    27  	"github.com/hashicorp/nomad/client/driver/env"
    28  	"github.com/hashicorp/nomad/client/driver/executor"
    29  	dstructs "github.com/hashicorp/nomad/client/driver/structs"
    30  	cstructs "github.com/hashicorp/nomad/client/structs"
    31  	"github.com/hashicorp/nomad/helper"
    32  	"github.com/hashicorp/nomad/helper/fields"
    33  	shelpers "github.com/hashicorp/nomad/helper/stats"
    34  	"github.com/hashicorp/nomad/nomad/structs"
    35  	"github.com/mitchellh/mapstructure"
    36  )
    37  
    38  var (
    39  	// We store the clients globally to cache the connection to the docker daemon.
    40  	createClients sync.Once
    41  
    42  	// client is a docker client with a timeout of 1 minute. This is for doing
    43  	// all operations with the docker daemon besides which are not long running
    44  	// such as creating, killing containers, etc.
    45  	client *docker.Client
    46  
    47  	// waitClient is a docker client with no timeouts. This is used for long
    48  	// running operations such as waiting on containers and collect stats
    49  	waitClient *docker.Client
    50  
    51  	// The statistics the Docker driver exposes
    52  	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"}
    53  	DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
    54  
    55  	// recoverableErrTimeouts returns a recoverable error if the error was due
    56  	// to timeouts
    57  	recoverableErrTimeouts = func(err error) error {
    58  		r := false
    59  		if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") ||
    60  			strings.Contains(err.Error(), "EOF") {
    61  			r = true
    62  		}
    63  		return structs.NewRecoverableError(err, r)
    64  	}
    65  )
    66  
    67  const (
    68  	// NoSuchContainerError is returned by the docker daemon if the container
    69  	// does not exist.
    70  	NoSuchContainerError = "No such container"
    71  
    72  	// The key populated in Node Attributes to indicate presence of the Docker
    73  	// driver
    74  	dockerDriverAttr = "driver.docker"
    75  
    76  	// dockerSELinuxLabelConfigOption is the key for configuring the
    77  	// SELinux label for binds.
    78  	dockerSELinuxLabelConfigOption = "docker.volumes.selinuxlabel"
    79  
    80  	// dockerVolumesConfigOption is the key for enabling the use of custom
    81  	// bind volumes to arbitrary host paths.
    82  	dockerVolumesConfigOption  = "docker.volumes.enabled"
    83  	dockerVolumesConfigDefault = true
    84  
    85  	// dockerPrivilegedConfigOption is the key for running containers in
    86  	// Docker's privileged mode.
    87  	dockerPrivilegedConfigOption = "docker.privileged.enabled"
    88  
    89  	// dockerCleanupImageConfigOption is the key for whether or not to
    90  	// cleanup images after the task exits.
    91  	dockerCleanupImageConfigOption  = "docker.cleanup.image"
    92  	dockerCleanupImageConfigDefault = true
    93  
    94  	// dockerPullTimeoutConfigOption is the key for setting an images pull
    95  	// timeout
    96  	dockerImageRemoveDelayConfigOption  = "docker.cleanup.image.delay"
    97  	dockerImageRemoveDelayConfigDefault = 3 * time.Minute
    98  
    99  	// dockerTimeout is the length of time a request can be outstanding before
   100  	// it is timed out.
   101  	dockerTimeout = 5 * time.Minute
   102  
   103  	// dockerImageResKey is the CreatedResources key for docker images
   104  	dockerImageResKey = "image"
   105  )
   106  
   107  type DockerDriver struct {
   108  	DriverContext
   109  
   110  	driverConfig *DockerDriverConfig
   111  	imageID      string
   112  
   113  	// A tri-state boolean to know if the fingerprinting has happened and
   114  	// whether it has been successful
   115  	fingerprintSuccess *bool
   116  }
   117  
   118  type DockerDriverAuth struct {
   119  	Username      string `mapstructure:"username"`       // username for the registry
   120  	Password      string `mapstructure:"password"`       // password to access the registry
   121  	Email         string `mapstructure:"email"`          // email address of the user who is allowed to access the registry
   122  	ServerAddress string `mapstructure:"server_address"` // server address of the registry
   123  }
   124  
   125  type DockerLoggingOpts struct {
   126  	Type      string              `mapstructure:"type"`
   127  	ConfigRaw []map[string]string `mapstructure:"config"`
   128  	Config    map[string]string   `mapstructure:"-"`
   129  }
   130  
   131  type DockerDriverConfig struct {
   132  	ImageName        string              `mapstructure:"image"`              // Container's Image Name
   133  	LoadImage        string              `mapstructure:"load"`               // LoadImage is a path to an image archive file
   134  	Command          string              `mapstructure:"command"`            // The Command to run when the container starts up
   135  	Args             []string            `mapstructure:"args"`               // The arguments to the Command
   136  	IpcMode          string              `mapstructure:"ipc_mode"`           // The IPC mode of the container - host and none
   137  	NetworkMode      string              `mapstructure:"network_mode"`       // The network mode of the container - host, nat and none
   138  	NetworkAliases   []string            `mapstructure:"network_aliases"`    // The network-scoped alias for the container
   139  	IPv4Address      string              `mapstructure:"ipv4_address"`       // The container ipv4 address
   140  	IPv6Address      string              `mapstructure:"ipv6_address"`       // the container ipv6 address
   141  	PidMode          string              `mapstructure:"pid_mode"`           // The PID mode of the container - host and none
   142  	UTSMode          string              `mapstructure:"uts_mode"`           // The UTS mode of the container - host and none
   143  	UsernsMode       string              `mapstructure:"userns_mode"`        // The User namespace mode of the container - host and none
   144  	PortMapRaw       []map[string]int    `mapstructure:"port_map"`           //
   145  	PortMap          map[string]int      `mapstructure:"-"`                  // A map of host port labels and the ports exposed on the container
   146  	Privileged       bool                `mapstructure:"privileged"`         // Flag to run the container in privileged mode
   147  	DNSServers       []string            `mapstructure:"dns_servers"`        // DNS Server for containers
   148  	DNSSearchDomains []string            `mapstructure:"dns_search_domains"` // DNS Search domains for containers
   149  	Hostname         string              `mapstructure:"hostname"`           // Hostname for containers
   150  	LabelsRaw        []map[string]string `mapstructure:"labels"`             //
   151  	Labels           map[string]string   `mapstructure:"-"`                  // Labels to set when the container starts up
   152  	Auth             []DockerDriverAuth  `mapstructure:"auth"`               // Authentication credentials for a private Docker registry
   153  	TTY              bool                `mapstructure:"tty"`                // Allocate a Pseudo-TTY
   154  	Interactive      bool                `mapstructure:"interactive"`        // Keep STDIN open even if not attached
   155  	ShmSize          int64               `mapstructure:"shm_size"`           // Size of /dev/shm of the container in bytes
   156  	WorkDir          string              `mapstructure:"work_dir"`           // Working directory inside the container
   157  	Logging          []DockerLoggingOpts `mapstructure:"logging"`            // Logging options for syslog server
   158  	Volumes          []string            `mapstructure:"volumes"`            // Host-Volumes to mount in, syntax: /path/to/host/directory:/destination/path/in/container
   159  	VolumeDriver     string              `mapstructure:"volume_driver"`      // Docker volume driver used for the container's volumes
   160  	ForcePull        bool                `mapstructure:"force_pull"`         // Always force pull before running image, useful if your tags are mutable
   161  }
   162  
   163  // Validate validates a docker driver config
   164  func (c *DockerDriverConfig) Validate() error {
   165  	if c.ImageName == "" {
   166  		return fmt.Errorf("Docker Driver needs an image name")
   167  	}
   168  
   169  	c.PortMap = mapMergeStrInt(c.PortMapRaw...)
   170  	c.Labels = mapMergeStrStr(c.LabelsRaw...)
   171  	if len(c.Logging) > 0 {
   172  		c.Logging[0].Config = mapMergeStrStr(c.Logging[0].ConfigRaw...)
   173  	}
   174  	return nil
   175  }
   176  
   177  // NewDockerDriverConfig returns a docker driver config by parsing the HCL
   178  // config
   179  func NewDockerDriverConfig(task *structs.Task, env *env.TaskEnvironment) (*DockerDriverConfig, error) {
   180  	var dconf DockerDriverConfig
   181  
   182  	if err := mapstructure.WeakDecode(task.Config, &dconf); err != nil {
   183  		return nil, err
   184  	}
   185  
   186  	// Interpolate everthing that is a string
   187  	dconf.ImageName = env.ReplaceEnv(dconf.ImageName)
   188  	dconf.Command = env.ReplaceEnv(dconf.Command)
   189  	dconf.IpcMode = env.ReplaceEnv(dconf.IpcMode)
   190  	dconf.NetworkMode = env.ReplaceEnv(dconf.NetworkMode)
   191  	dconf.NetworkAliases = env.ParseAndReplace(dconf.NetworkAliases)
   192  	dconf.IPv4Address = env.ReplaceEnv(dconf.IPv4Address)
   193  	dconf.IPv6Address = env.ReplaceEnv(dconf.IPv6Address)
   194  	dconf.PidMode = env.ReplaceEnv(dconf.PidMode)
   195  	dconf.UTSMode = env.ReplaceEnv(dconf.UTSMode)
   196  	dconf.Hostname = env.ReplaceEnv(dconf.Hostname)
   197  	dconf.WorkDir = env.ReplaceEnv(dconf.WorkDir)
   198  	dconf.LoadImage = env.ReplaceEnv(dconf.LoadImage)
   199  	dconf.Volumes = env.ParseAndReplace(dconf.Volumes)
   200  	dconf.VolumeDriver = env.ReplaceEnv(dconf.VolumeDriver)
   201  	dconf.DNSServers = env.ParseAndReplace(dconf.DNSServers)
   202  	dconf.DNSSearchDomains = env.ParseAndReplace(dconf.DNSSearchDomains)
   203  
   204  	for _, m := range dconf.LabelsRaw {
   205  		for k, v := range m {
   206  			delete(m, k)
   207  			m[env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   208  		}
   209  	}
   210  
   211  	for i, a := range dconf.Auth {
   212  		dconf.Auth[i].Username = env.ReplaceEnv(a.Username)
   213  		dconf.Auth[i].Password = env.ReplaceEnv(a.Password)
   214  		dconf.Auth[i].Email = env.ReplaceEnv(a.Email)
   215  		dconf.Auth[i].ServerAddress = env.ReplaceEnv(a.ServerAddress)
   216  	}
   217  
   218  	for i, l := range dconf.Logging {
   219  		dconf.Logging[i].Type = env.ReplaceEnv(l.Type)
   220  		for _, c := range l.ConfigRaw {
   221  			for k, v := range c {
   222  				delete(c, k)
   223  				c[env.ReplaceEnv(k)] = env.ReplaceEnv(v)
   224  			}
   225  		}
   226  	}
   227  
   228  	for _, m := range dconf.PortMapRaw {
   229  		for k, v := range m {
   230  			delete(m, k)
   231  			m[env.ReplaceEnv(k)] = v
   232  		}
   233  	}
   234  
   235  	// Remove any http
   236  	if strings.Contains(dconf.ImageName, "https://") {
   237  		dconf.ImageName = strings.Replace(dconf.ImageName, "https://", "", 1)
   238  	}
   239  
   240  	if err := dconf.Validate(); err != nil {
   241  		return nil, err
   242  	}
   243  	return &dconf, nil
   244  }
   245  
   246  type dockerPID struct {
   247  	Version        string
   248  	Image          string
   249  	ImageID        string
   250  	ContainerID    string
   251  	KillTimeout    time.Duration
   252  	MaxKillTimeout time.Duration
   253  	PluginConfig   *PluginReattachConfig
   254  }
   255  
   256  type DockerHandle struct {
   257  	pluginClient      *plugin.Client
   258  	executor          executor.Executor
   259  	client            *docker.Client
   260  	waitClient        *docker.Client
   261  	logger            *log.Logger
   262  	Image             string
   263  	ImageID           string
   264  	containerID       string
   265  	version           string
   266  	clkSpeed          float64
   267  	killTimeout       time.Duration
   268  	maxKillTimeout    time.Duration
   269  	resourceUsageLock sync.RWMutex
   270  	resourceUsage     *cstructs.TaskResourceUsage
   271  	waitCh            chan *dstructs.WaitResult
   272  	doneCh            chan bool
   273  }
   274  
   275  func NewDockerDriver(ctx *DriverContext) Driver {
   276  	return &DockerDriver{DriverContext: *ctx}
   277  }
   278  
   279  func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
   280  	// Initialize docker API clients
   281  	client, _, err := d.dockerClients()
   282  	if err != nil {
   283  		if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
   284  			d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s", err)
   285  		}
   286  		delete(node.Attributes, dockerDriverAttr)
   287  		d.fingerprintSuccess = helper.BoolToPtr(false)
   288  		return false, nil
   289  	}
   290  
   291  	// This is the first operation taken on the client so we'll try to
   292  	// establish a connection to the Docker daemon. If this fails it means
   293  	// Docker isn't available so we'll simply disable the docker driver.
   294  	env, err := client.Version()
   295  	if err != nil {
   296  		delete(node.Attributes, dockerDriverAttr)
   297  		if d.fingerprintSuccess == nil || *d.fingerprintSuccess {
   298  			d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon at %s: %s", client.Endpoint(), err)
   299  		}
   300  		d.fingerprintSuccess = helper.BoolToPtr(false)
   301  		return false, nil
   302  	}
   303  
   304  	node.Attributes[dockerDriverAttr] = "1"
   305  	node.Attributes["driver.docker.version"] = env.Get("Version")
   306  
   307  	privileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false)
   308  	if privileged {
   309  		node.Attributes[dockerPrivilegedConfigOption] = "1"
   310  	}
   311  
   312  	// Advertise if this node supports Docker volumes
   313  	if d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault) {
   314  		node.Attributes["driver."+dockerVolumesConfigOption] = "1"
   315  	}
   316  
   317  	d.fingerprintSuccess = helper.BoolToPtr(true)
   318  	return true, nil
   319  }
   320  
   321  // Validate is used to validate the driver configuration
   322  func (d *DockerDriver) Validate(config map[string]interface{}) error {
   323  	fd := &fields.FieldData{
   324  		Raw: config,
   325  		Schema: map[string]*fields.FieldSchema{
   326  			"image": &fields.FieldSchema{
   327  				Type:     fields.TypeString,
   328  				Required: true,
   329  			},
   330  			"load": &fields.FieldSchema{
   331  				Type: fields.TypeString,
   332  			},
   333  			"command": &fields.FieldSchema{
   334  				Type: fields.TypeString,
   335  			},
   336  			"args": &fields.FieldSchema{
   337  				Type: fields.TypeArray,
   338  			},
   339  			"ipc_mode": &fields.FieldSchema{
   340  				Type: fields.TypeString,
   341  			},
   342  			"network_mode": &fields.FieldSchema{
   343  				Type: fields.TypeString,
   344  			},
   345  			"network_aliases": &fields.FieldSchema{
   346  				Type: fields.TypeArray,
   347  			},
   348  			"ipv4_address": &fields.FieldSchema{
   349  				Type: fields.TypeString,
   350  			},
   351  			"ipv6_address": &fields.FieldSchema{
   352  				Type: fields.TypeString,
   353  			},
   354  			"pid_mode": &fields.FieldSchema{
   355  				Type: fields.TypeString,
   356  			},
   357  			"uts_mode": &fields.FieldSchema{
   358  				Type: fields.TypeString,
   359  			},
   360  			"userns_mode": &fields.FieldSchema{
   361  				Type: fields.TypeString,
   362  			},
   363  			"port_map": &fields.FieldSchema{
   364  				Type: fields.TypeArray,
   365  			},
   366  			"privileged": &fields.FieldSchema{
   367  				Type: fields.TypeBool,
   368  			},
   369  			"dns_servers": &fields.FieldSchema{
   370  				Type: fields.TypeArray,
   371  			},
   372  			"dns_search_domains": &fields.FieldSchema{
   373  				Type: fields.TypeArray,
   374  			},
   375  			"hostname": &fields.FieldSchema{
   376  				Type: fields.TypeString,
   377  			},
   378  			"labels": &fields.FieldSchema{
   379  				Type: fields.TypeArray,
   380  			},
   381  			"auth": &fields.FieldSchema{
   382  				Type: fields.TypeArray,
   383  			},
   384  			// COMPAT: Remove in 0.6.0. SSL is no longer needed
   385  			"ssl": &fields.FieldSchema{
   386  				Type: fields.TypeBool,
   387  			},
   388  			"tty": &fields.FieldSchema{
   389  				Type: fields.TypeBool,
   390  			},
   391  			"interactive": &fields.FieldSchema{
   392  				Type: fields.TypeBool,
   393  			},
   394  			"shm_size": &fields.FieldSchema{
   395  				Type: fields.TypeInt,
   396  			},
   397  			"work_dir": &fields.FieldSchema{
   398  				Type: fields.TypeString,
   399  			},
   400  			"logging": &fields.FieldSchema{
   401  				Type: fields.TypeArray,
   402  			},
   403  			"volumes": &fields.FieldSchema{
   404  				Type: fields.TypeArray,
   405  			},
   406  			"volume_driver": &fields.FieldSchema{
   407  				Type: fields.TypeString,
   408  			},
   409  			"force_pull": &fields.FieldSchema{
   410  				Type: fields.TypeBool,
   411  			},
   412  		},
   413  	}
   414  
   415  	if err := fd.Validate(); err != nil {
   416  		return err
   417  	}
   418  
   419  	return nil
   420  }
   421  
   422  func (d *DockerDriver) Abilities() DriverAbilities {
   423  	return DriverAbilities{
   424  		SendSignals: true,
   425  	}
   426  }
   427  
   428  func (d *DockerDriver) FSIsolation() cstructs.FSIsolation {
   429  	return cstructs.FSIsolationImage
   430  }
   431  
   432  // getDockerCoordinator returns the docker coordinator and the caller ID to use when
   433  // interacting with the coordinator
   434  func (d *DockerDriver) getDockerCoordinator(client *docker.Client) (*dockerCoordinator, string) {
   435  	config := &dockerCoordinatorConfig{
   436  		client:      client,
   437  		cleanup:     d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault),
   438  		logger:      d.logger,
   439  		removeDelay: d.config.ReadDurationDefault(dockerImageRemoveDelayConfigOption, dockerImageRemoveDelayConfigDefault),
   440  	}
   441  
   442  	return GetDockerCoordinator(config), fmt.Sprintf("%s-%s", d.DriverContext.allocID, d.DriverContext.taskName)
   443  }
   444  
   445  func (d *DockerDriver) Prestart(ctx *ExecContext, task *structs.Task) (*CreatedResources, error) {
   446  	driverConfig, err := NewDockerDriverConfig(task, d.taskEnv)
   447  	if err != nil {
   448  		return nil, err
   449  	}
   450  
   451  	// Set state needed by Start()
   452  	d.driverConfig = driverConfig
   453  
   454  	// Initialize docker API clients
   455  	client, _, err := d.dockerClients()
   456  	if err != nil {
   457  		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
   458  	}
   459  
   460  	// Ensure the image is available
   461  	id, err := d.createImage(driverConfig, client, ctx.TaskDir)
   462  	if err != nil {
   463  		return nil, err
   464  	}
   465  
   466  	res := NewCreatedResources()
   467  	res.Add(dockerImageResKey, id)
   468  	d.imageID = id
   469  	return res, nil
   470  }
   471  
   472  func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
   473  
   474  	pluginLogFile := filepath.Join(ctx.TaskDir.Dir, "executor.out")
   475  	executorConfig := &dstructs.ExecutorConfig{
   476  		LogFile:  pluginLogFile,
   477  		LogLevel: d.config.LogLevel,
   478  	}
   479  
   480  	exec, pluginClient, err := createExecutor(d.config.LogOutput, d.config, executorConfig)
   481  	if err != nil {
   482  		return nil, err
   483  	}
   484  	executorCtx := &executor.ExecutorContext{
   485  		TaskEnv:        d.taskEnv,
   486  		Task:           task,
   487  		Driver:         "docker",
   488  		AllocID:        d.DriverContext.allocID,
   489  		LogDir:         ctx.TaskDir.LogDir,
   490  		TaskDir:        ctx.TaskDir.Dir,
   491  		PortLowerBound: d.config.ClientMinPort,
   492  		PortUpperBound: d.config.ClientMaxPort,
   493  	}
   494  	if err := exec.SetContext(executorCtx); err != nil {
   495  		pluginClient.Kill()
   496  		return nil, fmt.Errorf("failed to set executor context: %v", err)
   497  	}
   498  
   499  	// Only launch syslog server if we're going to use it!
   500  	syslogAddr := ""
   501  	if runtime.GOOS == "darwin" && len(d.driverConfig.Logging) == 0 {
   502  		d.logger.Printf("[DEBUG] driver.docker: disabling syslog driver as Docker for Mac workaround")
   503  	} else if len(d.driverConfig.Logging) == 0 || d.driverConfig.Logging[0].Type == "syslog" {
   504  		ss, err := exec.LaunchSyslogServer()
   505  		if err != nil {
   506  			pluginClient.Kill()
   507  			return nil, fmt.Errorf("failed to start syslog collector: %v", err)
   508  		}
   509  		syslogAddr = ss.Addr
   510  	}
   511  
   512  	config, err := d.createContainerConfig(ctx, task, d.driverConfig, syslogAddr)
   513  	if err != nil {
   514  		d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err)
   515  		pluginClient.Kill()
   516  		return nil, fmt.Errorf("Failed to create container configuration for image %q (%q): %v", d.driverConfig.ImageName, d.imageID, err)
   517  	}
   518  
   519  	container, err := d.createContainer(config)
   520  	if err != nil {
   521  		wrapped := fmt.Sprintf("Failed to create container: %v", err)
   522  		d.logger.Printf("[ERR] driver.docker: %s", wrapped)
   523  		pluginClient.Kill()
   524  		return nil, structs.WrapRecoverable(wrapped, err)
   525  	}
   526  
   527  	d.logger.Printf("[INFO] driver.docker: created container %s", container.ID)
   528  
   529  	// We don't need to start the container if the container is already running
   530  	// since we don't create containers which are already present on the host
   531  	// and are running
   532  	if !container.State.Running {
   533  		// Start the container
   534  		if err := d.startContainer(container); err != nil {
   535  			d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err)
   536  			pluginClient.Kill()
   537  			return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err)
   538  		}
   539  		d.logger.Printf("[INFO] driver.docker: started container %s", container.ID)
   540  	} else {
   541  		d.logger.Printf("[DEBUG] driver.docker: re-attaching to container %s with status %q",
   542  			container.ID, container.State.String())
   543  	}
   544  
   545  	// Return a driver handle
   546  	maxKill := d.DriverContext.config.MaxKillTimeout
   547  	h := &DockerHandle{
   548  		client:         client,
   549  		waitClient:     waitClient,
   550  		executor:       exec,
   551  		pluginClient:   pluginClient,
   552  		logger:         d.logger,
   553  		Image:          d.driverConfig.ImageName,
   554  		ImageID:        d.imageID,
   555  		containerID:    container.ID,
   556  		version:        d.config.Version,
   557  		killTimeout:    GetKillTimeout(task.KillTimeout, maxKill),
   558  		maxKillTimeout: maxKill,
   559  		doneCh:         make(chan bool),
   560  		waitCh:         make(chan *dstructs.WaitResult, 1),
   561  	}
   562  	if err := exec.SyncServices(consulContext(d.config, container.ID)); err != nil {
   563  		d.logger.Printf("[ERR] driver.docker: error registering services with consul for task: %q: %v", task.Name, err)
   564  	}
   565  	go h.collectStats()
   566  	go h.run()
   567  	return h, nil
   568  }
   569  
   570  func (d *DockerDriver) Cleanup(_ *ExecContext, res *CreatedResources) error {
   571  	retry := false
   572  	var merr multierror.Error
   573  	for key, resources := range res.Resources {
   574  		switch key {
   575  		case dockerImageResKey:
   576  			for _, value := range resources {
   577  				err := d.cleanupImage(value)
   578  				if err != nil {
   579  					if structs.IsRecoverable(err) {
   580  						retry = true
   581  					}
   582  					merr.Errors = append(merr.Errors, err)
   583  					continue
   584  				}
   585  
   586  				// Remove cleaned image from resources
   587  				res.Remove(dockerImageResKey, value)
   588  			}
   589  		default:
   590  			d.logger.Printf("[ERR] driver.docker: unknown resource to cleanup: %q", key)
   591  		}
   592  	}
   593  	return structs.NewRecoverableError(merr.ErrorOrNil(), retry)
   594  }
   595  
   596  // cleanupImage removes a Docker image. No error is returned if the image
   597  // doesn't exist or is still in use. Requires the global client to already be
   598  // initialized.
   599  func (d *DockerDriver) cleanupImage(imageID string) error {
   600  	if !d.config.ReadBoolDefault(dockerCleanupImageConfigOption, dockerCleanupImageConfigDefault) {
   601  		// Config says not to cleanup
   602  		return nil
   603  	}
   604  
   605  	coordinator, callerID := d.getDockerCoordinator(client)
   606  	coordinator.RemoveImage(imageID, callerID)
   607  
   608  	return nil
   609  }
   610  
   611  // dockerClients creates two *docker.Client, one for long running operations and
   612  // the other for shorter operations. In test / dev mode we can use ENV vars to
   613  // connect to the docker daemon. In production mode we will read docker.endpoint
   614  // from the config file.
   615  func (d *DockerDriver) dockerClients() (*docker.Client, *docker.Client, error) {
   616  	if client != nil && waitClient != nil {
   617  		return client, waitClient, nil
   618  	}
   619  
   620  	var err error
   621  	var merr multierror.Error
   622  	createClients.Do(func() {
   623  		// Default to using whatever is configured in docker.endpoint. If this is
   624  		// not specified we'll fall back on NewClientFromEnv which reads config from
   625  		// the DOCKER_* environment variables DOCKER_HOST, DOCKER_TLS_VERIFY, and
   626  		// DOCKER_CERT_PATH. This allows us to lock down the config in production
   627  		// but also accept the standard ENV configs for dev and test.
   628  		dockerEndpoint := d.config.Read("docker.endpoint")
   629  		if dockerEndpoint != "" {
   630  			cert := d.config.Read("docker.tls.cert")
   631  			key := d.config.Read("docker.tls.key")
   632  			ca := d.config.Read("docker.tls.ca")
   633  
   634  			if cert+key+ca != "" {
   635  				d.logger.Printf("[DEBUG] driver.docker: using TLS client connection to %s", dockerEndpoint)
   636  				client, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca)
   637  				if err != nil {
   638  					merr.Errors = append(merr.Errors, err)
   639  				}
   640  				waitClient, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca)
   641  				if err != nil {
   642  					merr.Errors = append(merr.Errors, err)
   643  				}
   644  			} else {
   645  				d.logger.Printf("[DEBUG] driver.docker: using standard client connection to %s", dockerEndpoint)
   646  				client, err = docker.NewClient(dockerEndpoint)
   647  				if err != nil {
   648  					merr.Errors = append(merr.Errors, err)
   649  				}
   650  				waitClient, err = docker.NewClient(dockerEndpoint)
   651  				if err != nil {
   652  					merr.Errors = append(merr.Errors, err)
   653  				}
   654  			}
   655  			client.SetTimeout(dockerTimeout)
   656  			return
   657  		}
   658  
   659  		d.logger.Println("[DEBUG] driver.docker: using client connection initialized from environment")
   660  		client, err = docker.NewClientFromEnv()
   661  		if err != nil {
   662  			merr.Errors = append(merr.Errors, err)
   663  		}
   664  		client.SetTimeout(dockerTimeout)
   665  
   666  		waitClient, err = docker.NewClientFromEnv()
   667  		if err != nil {
   668  			merr.Errors = append(merr.Errors, err)
   669  		}
   670  	})
   671  	return client, waitClient, merr.ErrorOrNil()
   672  }
   673  
   674  func (d *DockerDriver) containerBinds(driverConfig *DockerDriverConfig, taskDir *allocdir.TaskDir,
   675  	task *structs.Task) ([]string, error) {
   676  
   677  	allocDirBind := fmt.Sprintf("%s:%s", taskDir.SharedAllocDir, allocdir.SharedAllocContainerPath)
   678  	taskLocalBind := fmt.Sprintf("%s:%s", taskDir.LocalDir, allocdir.TaskLocalContainerPath)
   679  	secretDirBind := fmt.Sprintf("%s:%s", taskDir.SecretsDir, allocdir.TaskSecretsContainerPath)
   680  	binds := []string{allocDirBind, taskLocalBind, secretDirBind}
   681  
   682  	volumesEnabled := d.config.ReadBoolDefault(dockerVolumesConfigOption, dockerVolumesConfigDefault)
   683  
   684  	if !volumesEnabled && driverConfig.VolumeDriver != "" {
   685  		return nil, fmt.Errorf("%s is false; cannot use volume driver %q", dockerVolumesConfigOption, driverConfig.VolumeDriver)
   686  	}
   687  
   688  	for _, userbind := range driverConfig.Volumes {
   689  		parts := strings.Split(userbind, ":")
   690  		if len(parts) < 2 {
   691  			return nil, fmt.Errorf("invalid docker volume: %q", userbind)
   692  		}
   693  
   694  		// Resolve dotted path segments
   695  		parts[0] = filepath.Clean(parts[0])
   696  
   697  		// Absolute paths aren't always supported
   698  		if filepath.IsAbs(parts[0]) {
   699  			if !volumesEnabled {
   700  				// Disallow mounting arbitrary absolute paths
   701  				return nil, fmt.Errorf("%s is false; cannot mount host paths: %+q", dockerVolumesConfigOption, userbind)
   702  			}
   703  			binds = append(binds, userbind)
   704  			continue
   705  		}
   706  
   707  		// Relative paths are always allowed as they mount within a container
   708  		// When a VolumeDriver is set, we assume we receive a binding in the format volume-name:container-dest
   709  		// Otherwise, we assume we receive a relative path binding in the format relative/to/task:/also/in/container
   710  		if driverConfig.VolumeDriver == "" {
   711  			// Expand path relative to alloc dir
   712  			parts[0] = filepath.Join(taskDir.Dir, parts[0])
   713  		}
   714  
   715  		binds = append(binds, strings.Join(parts, ":"))
   716  	}
   717  
   718  	if selinuxLabel := d.config.Read(dockerSELinuxLabelConfigOption); selinuxLabel != "" {
   719  		// Apply SELinux Label to each volume
   720  		for i := range binds {
   721  			binds[i] = fmt.Sprintf("%s:%s", binds[i], selinuxLabel)
   722  		}
   723  	}
   724  
   725  	return binds, nil
   726  }
   727  
   728  // createContainerConfig initializes a struct needed to call docker.client.CreateContainer()
   729  func (d *DockerDriver) createContainerConfig(ctx *ExecContext, task *structs.Task,
   730  	driverConfig *DockerDriverConfig, syslogAddr string) (docker.CreateContainerOptions, error) {
   731  	var c docker.CreateContainerOptions
   732  	if task.Resources == nil {
   733  		// Guard against missing resources. We should never have been able to
   734  		// schedule a job without specifying this.
   735  		d.logger.Println("[ERR] driver.docker: task.Resources is empty")
   736  		return c, fmt.Errorf("task.Resources is empty")
   737  	}
   738  
   739  	binds, err := d.containerBinds(driverConfig, ctx.TaskDir, task)
   740  	if err != nil {
   741  		return c, err
   742  	}
   743  
   744  	config := &docker.Config{
   745  		Image:     d.imageID,
   746  		Hostname:  driverConfig.Hostname,
   747  		User:      task.User,
   748  		Tty:       driverConfig.TTY,
   749  		OpenStdin: driverConfig.Interactive,
   750  	}
   751  
   752  	if driverConfig.WorkDir != "" {
   753  		config.WorkingDir = driverConfig.WorkDir
   754  	}
   755  
   756  	memLimit := int64(task.Resources.MemoryMB) * 1024 * 1024
   757  
   758  	if len(driverConfig.Logging) == 0 {
   759  		if runtime.GOOS != "darwin" {
   760  			d.logger.Printf("[DEBUG] driver.docker: Setting default logging options to syslog and %s", syslogAddr)
   761  			driverConfig.Logging = []DockerLoggingOpts{
   762  				{Type: "syslog", Config: map[string]string{"syslog-address": syslogAddr}},
   763  			}
   764  		} else {
   765  			d.logger.Printf("[DEBUG] driver.docker: deferring logging to docker on Docker for Mac")
   766  		}
   767  	}
   768  
   769  	hostConfig := &docker.HostConfig{
   770  		// Convert MB to bytes. This is an absolute value.
   771  		Memory: memLimit,
   772  		// Convert Mhz to shares. This is a relative value.
   773  		CPUShares: int64(task.Resources.CPU),
   774  
   775  		// Binds are used to mount a host volume into the container. We mount a
   776  		// local directory for storage and a shared alloc directory that can be
   777  		// used to share data between different tasks in the same task group.
   778  		Binds: binds,
   779  
   780  		VolumeDriver: driverConfig.VolumeDriver,
   781  	}
   782  
   783  	// Windows does not support MemorySwap #2193
   784  	if runtime.GOOS != "windows" {
   785  		hostConfig.MemorySwap = memLimit // MemorySwap is memory + swap.
   786  	}
   787  
   788  	if len(driverConfig.Logging) != 0 {
   789  		d.logger.Printf("[DEBUG] driver.docker: Using config for logging: %+v", driverConfig.Logging[0])
   790  		hostConfig.LogConfig = docker.LogConfig{
   791  			Type:   driverConfig.Logging[0].Type,
   792  			Config: driverConfig.Logging[0].Config,
   793  		}
   794  	}
   795  
   796  	d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Name)
   797  	d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Name)
   798  	d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Name)
   799  
   800  	//  set privileged mode
   801  	hostPrivileged := d.config.ReadBoolDefault(dockerPrivilegedConfigOption, false)
   802  	if driverConfig.Privileged && !hostPrivileged {
   803  		return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`)
   804  	}
   805  	hostConfig.Privileged = driverConfig.Privileged
   806  
   807  	// set SHM size
   808  	if driverConfig.ShmSize != 0 {
   809  		hostConfig.ShmSize = driverConfig.ShmSize
   810  	}
   811  
   812  	// set DNS servers
   813  	for _, ip := range driverConfig.DNSServers {
   814  		if net.ParseIP(ip) != nil {
   815  			hostConfig.DNS = append(hostConfig.DNS, ip)
   816  		} else {
   817  			d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip)
   818  		}
   819  	}
   820  
   821  	// set DNS search domains
   822  	for _, domain := range driverConfig.DNSSearchDomains {
   823  		hostConfig.DNSSearch = append(hostConfig.DNSSearch, domain)
   824  	}
   825  
   826  	hostConfig.IpcMode = driverConfig.IpcMode
   827  	hostConfig.PidMode = driverConfig.PidMode
   828  	hostConfig.UTSMode = driverConfig.UTSMode
   829  	hostConfig.UsernsMode = driverConfig.UsernsMode
   830  
   831  	hostConfig.NetworkMode = driverConfig.NetworkMode
   832  	if hostConfig.NetworkMode == "" {
   833  		// docker default
   834  		d.logger.Printf("[DEBUG] driver.docker: networking mode not specified; defaulting to %s", defaultNetworkMode)
   835  		hostConfig.NetworkMode = defaultNetworkMode
   836  	}
   837  
   838  	// Setup port mapping and exposed ports
   839  	if len(task.Resources.Networks) == 0 {
   840  		d.logger.Println("[DEBUG] driver.docker: No network interfaces are available")
   841  		if len(driverConfig.PortMap) > 0 {
   842  			return c, fmt.Errorf("Trying to map ports but no network interface is available")
   843  		}
   844  	} else {
   845  		// TODO add support for more than one network
   846  		network := task.Resources.Networks[0]
   847  		publishedPorts := map[docker.Port][]docker.PortBinding{}
   848  		exposedPorts := map[docker.Port]struct{}{}
   849  
   850  		for _, port := range network.ReservedPorts {
   851  			// By default we will map the allocated port 1:1 to the container
   852  			containerPortInt := port.Value
   853  
   854  			// If the user has mapped a port using port_map we'll change it here
   855  			if mapped, ok := driverConfig.PortMap[port.Label]; ok {
   856  				containerPortInt = mapped
   857  			}
   858  
   859  			hostPortStr := strconv.Itoa(port.Value)
   860  			containerPort := docker.Port(strconv.Itoa(containerPortInt))
   861  
   862  			publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr)
   863  			publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr)
   864  			d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value)
   865  
   866  			exposedPorts[containerPort+"/tcp"] = struct{}{}
   867  			exposedPorts[containerPort+"/udp"] = struct{}{}
   868  			d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value)
   869  		}
   870  
   871  		for _, port := range network.DynamicPorts {
   872  			// By default we will map the allocated port 1:1 to the container
   873  			containerPortInt := port.Value
   874  
   875  			// If the user has mapped a port using port_map we'll change it here
   876  			if mapped, ok := driverConfig.PortMap[port.Label]; ok {
   877  				containerPortInt = mapped
   878  			}
   879  
   880  			hostPortStr := strconv.Itoa(port.Value)
   881  			containerPort := docker.Port(strconv.Itoa(containerPortInt))
   882  
   883  			publishedPorts[containerPort+"/tcp"] = getPortBinding(network.IP, hostPortStr)
   884  			publishedPorts[containerPort+"/udp"] = getPortBinding(network.IP, hostPortStr)
   885  			d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt)
   886  
   887  			exposedPorts[containerPort+"/tcp"] = struct{}{}
   888  			exposedPorts[containerPort+"/udp"] = struct{}{}
   889  			d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort)
   890  		}
   891  
   892  		d.taskEnv.SetPortMap(driverConfig.PortMap)
   893  
   894  		hostConfig.PortBindings = publishedPorts
   895  		config.ExposedPorts = exposedPorts
   896  	}
   897  
   898  	d.taskEnv.Build()
   899  	parsedArgs := d.taskEnv.ParseAndReplace(driverConfig.Args)
   900  
   901  	// If the user specified a custom command to run, we'll inject it here.
   902  	if driverConfig.Command != "" {
   903  		// Validate command
   904  		if err := validateCommand(driverConfig.Command, "args"); err != nil {
   905  			return c, err
   906  		}
   907  
   908  		cmd := []string{driverConfig.Command}
   909  		if len(driverConfig.Args) != 0 {
   910  			cmd = append(cmd, parsedArgs...)
   911  		}
   912  		d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
   913  		config.Cmd = cmd
   914  	} else if len(driverConfig.Args) != 0 {
   915  		config.Cmd = parsedArgs
   916  	}
   917  
   918  	if len(driverConfig.Labels) > 0 {
   919  		config.Labels = driverConfig.Labels
   920  		d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels)
   921  	}
   922  
   923  	config.Env = d.taskEnv.EnvList()
   924  
   925  	containerName := fmt.Sprintf("%s-%s", task.Name, d.DriverContext.allocID)
   926  	d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName)
   927  
   928  	var networkingConfig *docker.NetworkingConfig
   929  	if len(driverConfig.NetworkAliases) > 0 || driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" {
   930  		networkingConfig = &docker.NetworkingConfig{
   931  			EndpointsConfig: map[string]*docker.EndpointConfig{
   932  				hostConfig.NetworkMode: &docker.EndpointConfig{},
   933  			},
   934  		}
   935  	}
   936  
   937  	if len(driverConfig.NetworkAliases) > 0 {
   938  		networkingConfig.EndpointsConfig[hostConfig.NetworkMode].Aliases = driverConfig.NetworkAliases
   939  		d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with network aliases: %v",
   940  			hostConfig.NetworkMode, strings.Join(driverConfig.NetworkAliases, ", "))
   941  	}
   942  
   943  	if driverConfig.IPv4Address != "" || driverConfig.IPv6Address != "" {
   944  		networkingConfig.EndpointsConfig[hostConfig.NetworkMode].IPAMConfig = &docker.EndpointIPAMConfig{
   945  			IPv4Address: driverConfig.IPv4Address,
   946  			IPv6Address: driverConfig.IPv6Address,
   947  		}
   948  		d.logger.Printf("[DEBUG] driver.docker: using network_mode %q with ipv4: %q and ipv6: %q",
   949  			hostConfig.NetworkMode, driverConfig.IPv4Address, driverConfig.IPv6Address)
   950  	}
   951  
   952  	return docker.CreateContainerOptions{
   953  		Name:             containerName,
   954  		Config:           config,
   955  		HostConfig:       hostConfig,
   956  		NetworkingConfig: networkingConfig,
   957  	}, nil
   958  }
   959  
   960  func (d *DockerDriver) Periodic() (bool, time.Duration) {
   961  	return true, 15 * time.Second
   962  }
   963  
   964  // createImage creates a docker image either by pulling it from a registry or by
   965  // loading it from the file system
   966  func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir *allocdir.TaskDir) (string, error) {
   967  	image := driverConfig.ImageName
   968  	repo, tag := docker.ParseRepositoryTag(image)
   969  	if tag == "" {
   970  		tag = "latest"
   971  	}
   972  
   973  	coordinator, callerID := d.getDockerCoordinator(client)
   974  
   975  	// We're going to check whether the image is already downloaded. If the tag
   976  	// is "latest", or ForcePull is set, we have to check for a new version every time so we don't
   977  	// bother to check and cache the id here. We'll download first, then cache.
   978  	if driverConfig.ForcePull {
   979  		d.logger.Printf("[DEBUG] driver.docker: force pull image '%s:%s' instead of inspecting local", repo, tag)
   980  	} else if tag != "latest" {
   981  		if dockerImage, _ := client.InspectImage(image); dockerImage != nil {
   982  			// Image exists so just increment its reference count
   983  			coordinator.IncrementImageReference(dockerImage.ID, image, callerID)
   984  			return dockerImage.ID, nil
   985  		}
   986  	}
   987  
   988  	// Load the image if specified
   989  	if driverConfig.LoadImage != "" {
   990  		return d.loadImage(driverConfig, client, taskDir)
   991  	}
   992  
   993  	// Download the image
   994  	return d.pullImage(driverConfig, client, repo, tag)
   995  }
   996  
   997  // pullImage creates an image by pulling it from a docker registry
   998  func (d *DockerDriver) pullImage(driverConfig *DockerDriverConfig, client *docker.Client, repo, tag string) (id string, err error) {
   999  	var authOptions *docker.AuthConfiguration
  1000  	if len(driverConfig.Auth) != 0 {
  1001  		authOptions = &docker.AuthConfiguration{
  1002  			Username:      driverConfig.Auth[0].Username,
  1003  			Password:      driverConfig.Auth[0].Password,
  1004  			Email:         driverConfig.Auth[0].Email,
  1005  			ServerAddress: driverConfig.Auth[0].ServerAddress,
  1006  		}
  1007  	} else if authConfigFile := d.config.Read("docker.auth.config"); authConfigFile != "" {
  1008  		var err error
  1009  		authOptions, err = authOptionFrom(authConfigFile, repo)
  1010  		if err != nil {
  1011  			d.logger.Printf("[INFO] driver.docker: failed to find docker auth for repo %q: %v", repo, err)
  1012  			return "", fmt.Errorf("Failed to find docker auth for repo %q: %v", repo, err)
  1013  		}
  1014  
  1015  		if authOptions.Email == "" && authOptions.Password == "" &&
  1016  			authOptions.ServerAddress == "" && authOptions.Username == "" {
  1017  			d.logger.Printf("[DEBUG] driver.docker: did not find docker auth for repo %q", repo)
  1018  		}
  1019  	}
  1020  
  1021  	d.emitEvent("Downloading image %s:%s", repo, tag)
  1022  	coordinator, callerID := d.getDockerCoordinator(client)
  1023  	return coordinator.PullImage(driverConfig.ImageName, authOptions, callerID)
  1024  }
  1025  
  1026  // loadImage creates an image by loading it from the file system
  1027  func (d *DockerDriver) loadImage(driverConfig *DockerDriverConfig, client *docker.Client,
  1028  	taskDir *allocdir.TaskDir) (id string, err error) {
  1029  
  1030  	archive := filepath.Join(taskDir.LocalDir, driverConfig.LoadImage)
  1031  	d.logger.Printf("[DEBUG] driver.docker: loading image from: %v", archive)
  1032  
  1033  	f, err := os.Open(archive)
  1034  	if err != nil {
  1035  		return "", fmt.Errorf("unable to open image archive: %v", err)
  1036  	}
  1037  
  1038  	if err := client.LoadImage(docker.LoadImageOptions{InputStream: f}); err != nil {
  1039  		return "", err
  1040  	}
  1041  	f.Close()
  1042  
  1043  	dockerImage, err := client.InspectImage(driverConfig.ImageName)
  1044  	if err != nil {
  1045  		return "", recoverableErrTimeouts(err)
  1046  	}
  1047  
  1048  	coordinator, callerID := d.getDockerCoordinator(client)
  1049  	coordinator.IncrementImageReference(dockerImage.ID, driverConfig.ImageName, callerID)
  1050  	return dockerImage.ID, nil
  1051  }
  1052  
  1053  // createContainer creates the container given the passed configuration. It
  1054  // attempts to handle any transient Docker errors.
  1055  func (d *DockerDriver) createContainer(config docker.CreateContainerOptions) (*docker.Container, error) {
  1056  	// Create a container
  1057  	attempted := 0
  1058  CREATE:
  1059  	container, createErr := client.CreateContainer(config)
  1060  	if createErr == nil {
  1061  		return container, nil
  1062  	}
  1063  
  1064  	d.logger.Printf("[DEBUG] driver.docker: failed to create container %q from image %q (ID: %q) (attempt %d): %v",
  1065  		config.Name, d.driverConfig.ImageName, d.imageID, attempted+1, createErr)
  1066  	if strings.Contains(strings.ToLower(createErr.Error()), "container already exists") {
  1067  		containers, err := client.ListContainers(docker.ListContainersOptions{
  1068  			All: true,
  1069  		})
  1070  		if err != nil {
  1071  			d.logger.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name)
  1072  			return nil, recoverableErrTimeouts(fmt.Errorf("Failed to query list of containers: %s", err))
  1073  		}
  1074  
  1075  		// Delete matching containers
  1076  		// Adding a / infront of the container name since Docker returns the
  1077  		// container names with a / pre-pended to the Nomad generated container names
  1078  		containerName := "/" + config.Name
  1079  		d.logger.Printf("[DEBUG] driver.docker: searching for container name %q to purge", containerName)
  1080  		for _, container := range containers {
  1081  			d.logger.Printf("[DEBUG] driver.docker: listed container %+v", container)
  1082  			found := false
  1083  			for _, name := range container.Names {
  1084  				if name == containerName {
  1085  					found = true
  1086  					break
  1087  				}
  1088  			}
  1089  
  1090  			if !found {
  1091  				continue
  1092  			}
  1093  
  1094  			// Inspect the container and if the container isn't dead then return
  1095  			// the container
  1096  			container, err := client.InspectContainer(container.ID)
  1097  			if err != nil {
  1098  				return nil, recoverableErrTimeouts(fmt.Errorf("Failed to inspect container %s: %s", container.ID, err))
  1099  			}
  1100  			if container != nil && (container.State.Running || container.State.FinishedAt.IsZero()) {
  1101  				return container, nil
  1102  			}
  1103  
  1104  			err = client.RemoveContainer(docker.RemoveContainerOptions{
  1105  				ID:    container.ID,
  1106  				Force: true,
  1107  			})
  1108  			if err != nil {
  1109  				d.logger.Printf("[ERR] driver.docker: failed to purge container %s", container.ID)
  1110  				return nil, recoverableErrTimeouts(fmt.Errorf("Failed to purge container %s: %s", container.ID, err))
  1111  			} else if err == nil {
  1112  				d.logger.Printf("[INFO] driver.docker: purged container %s", container.ID)
  1113  			}
  1114  		}
  1115  
  1116  		if attempted < 5 {
  1117  			attempted++
  1118  			time.Sleep(1 * time.Second)
  1119  			goto CREATE
  1120  		}
  1121  	} else if strings.Contains(strings.ToLower(createErr.Error()), "no such image") {
  1122  		// There is still a very small chance this is possible even with the
  1123  		// coordinator so retry.
  1124  		return nil, structs.NewRecoverableError(createErr, true)
  1125  	}
  1126  
  1127  	return nil, recoverableErrTimeouts(createErr)
  1128  }
  1129  
  1130  // startContainer starts the passed container. It attempts to handle any
  1131  // transient Docker errors.
  1132  func (d *DockerDriver) startContainer(c *docker.Container) error {
  1133  	// Start a container
  1134  	attempted := 0
  1135  START:
  1136  	startErr := client.StartContainer(c.ID, c.HostConfig)
  1137  	if startErr == nil {
  1138  		return nil
  1139  	}
  1140  
  1141  	d.logger.Printf("[DEBUG] driver.docker: failed to start container %q (attempt %d): %v", c.ID, attempted+1, startErr)
  1142  
  1143  	// If it is a 500 error it is likely we can retry and be successful
  1144  	if strings.Contains(startErr.Error(), "API error (500)") {
  1145  		if attempted < 5 {
  1146  			attempted++
  1147  			time.Sleep(1 * time.Second)
  1148  			goto START
  1149  		}
  1150  	}
  1151  
  1152  	return recoverableErrTimeouts(startErr)
  1153  }
  1154  
  1155  func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
  1156  	// Split the handle
  1157  	pidBytes := []byte(strings.TrimPrefix(handleID, "DOCKER:"))
  1158  	pid := &dockerPID{}
  1159  	if err := json.Unmarshal(pidBytes, pid); err != nil {
  1160  		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
  1161  	}
  1162  	d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", pid.ContainerID)
  1163  	d.logger.Printf("[DEBUG] driver.docker: re-attached to handle: %s", handleID)
  1164  	pluginConfig := &plugin.ClientConfig{
  1165  		Reattach: pid.PluginConfig.PluginConfig(),
  1166  	}
  1167  
  1168  	client, waitClient, err := d.dockerClients()
  1169  	if err != nil {
  1170  		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
  1171  	}
  1172  
  1173  	// Look for a running container with this ID
  1174  	containers, err := client.ListContainers(docker.ListContainersOptions{
  1175  		Filters: map[string][]string{
  1176  			"id": []string{pid.ContainerID},
  1177  		},
  1178  	})
  1179  	if err != nil {
  1180  		return nil, fmt.Errorf("Failed to query for container %s: %v", pid.ContainerID, err)
  1181  	}
  1182  
  1183  	found := false
  1184  	for _, container := range containers {
  1185  		if container.ID == pid.ContainerID {
  1186  			found = true
  1187  		}
  1188  	}
  1189  	if !found {
  1190  		return nil, fmt.Errorf("Failed to find container %s", pid.ContainerID)
  1191  	}
  1192  	exec, pluginClient, err := createExecutorWithConfig(pluginConfig, d.config.LogOutput)
  1193  	if err != nil {
  1194  		d.logger.Printf("[INFO] driver.docker: couldn't re-attach to the plugin process: %v", err)
  1195  		d.logger.Printf("[DEBUG] driver.docker: stopping container %q", pid.ContainerID)
  1196  		if e := client.StopContainer(pid.ContainerID, uint(pid.KillTimeout.Seconds())); e != nil {
  1197  			d.logger.Printf("[DEBUG] driver.docker: couldn't stop container: %v", e)
  1198  		}
  1199  		return nil, err
  1200  	}
  1201  
  1202  	ver, _ := exec.Version()
  1203  	d.logger.Printf("[DEBUG] driver.docker: version of executor: %v", ver.Version)
  1204  
  1205  	// Increment the reference count since we successfully attached to this
  1206  	// container
  1207  	coordinator, callerID := d.getDockerCoordinator(client)
  1208  	coordinator.IncrementImageReference(pid.ImageID, pid.Image, callerID)
  1209  
  1210  	// Return a driver handle
  1211  	h := &DockerHandle{
  1212  		client:         client,
  1213  		waitClient:     waitClient,
  1214  		executor:       exec,
  1215  		pluginClient:   pluginClient,
  1216  		logger:         d.logger,
  1217  		Image:          pid.Image,
  1218  		ImageID:        pid.ImageID,
  1219  		containerID:    pid.ContainerID,
  1220  		version:        pid.Version,
  1221  		killTimeout:    pid.KillTimeout,
  1222  		maxKillTimeout: pid.MaxKillTimeout,
  1223  		doneCh:         make(chan bool),
  1224  		waitCh:         make(chan *dstructs.WaitResult, 1),
  1225  	}
  1226  	if err := exec.SyncServices(consulContext(d.config, pid.ContainerID)); err != nil {
  1227  		h.logger.Printf("[ERR] driver.docker: error registering services with consul: %v", err)
  1228  	}
  1229  
  1230  	go h.collectStats()
  1231  	go h.run()
  1232  	return h, nil
  1233  }
  1234  
  1235  func (h *DockerHandle) ID() string {
  1236  	// Return a handle to the PID
  1237  	pid := dockerPID{
  1238  		Version:        h.version,
  1239  		ContainerID:    h.containerID,
  1240  		Image:          h.Image,
  1241  		ImageID:        h.ImageID,
  1242  		KillTimeout:    h.killTimeout,
  1243  		MaxKillTimeout: h.maxKillTimeout,
  1244  		PluginConfig:   NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
  1245  	}
  1246  	data, err := json.Marshal(pid)
  1247  	if err != nil {
  1248  		h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s", err)
  1249  	}
  1250  	return fmt.Sprintf("DOCKER:%s", string(data))
  1251  }
  1252  
  1253  func (h *DockerHandle) ContainerID() string {
  1254  	return h.containerID
  1255  }
  1256  
  1257  func (h *DockerHandle) WaitCh() chan *dstructs.WaitResult {
  1258  	return h.waitCh
  1259  }
  1260  
  1261  func (h *DockerHandle) Update(task *structs.Task) error {
  1262  	// Store the updated kill timeout.
  1263  	h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
  1264  	if err := h.executor.UpdateTask(task); err != nil {
  1265  		h.logger.Printf("[DEBUG] driver.docker: failed to update log config: %v", err)
  1266  	}
  1267  
  1268  	// Update is not possible
  1269  	return nil
  1270  }
  1271  
  1272  func (h *DockerHandle) Signal(s os.Signal) error {
  1273  	// Convert types
  1274  	sysSig, ok := s.(syscall.Signal)
  1275  	if !ok {
  1276  		return fmt.Errorf("Failed to determine signal number")
  1277  	}
  1278  
  1279  	dockerSignal := docker.Signal(sysSig)
  1280  	opts := docker.KillContainerOptions{
  1281  		ID:     h.containerID,
  1282  		Signal: dockerSignal,
  1283  	}
  1284  	return h.client.KillContainer(opts)
  1285  
  1286  }
  1287  
  1288  // Kill is used to terminate the task. This uses `docker stop -t killTimeout`
  1289  func (h *DockerHandle) Kill() error {
  1290  	// Stop the container
  1291  	err := h.client.StopContainer(h.containerID, uint(h.killTimeout.Seconds()))
  1292  	if err != nil {
  1293  		h.executor.Exit()
  1294  		h.pluginClient.Kill()
  1295  
  1296  		// Container has already been removed.
  1297  		if strings.Contains(err.Error(), NoSuchContainerError) {
  1298  			h.logger.Printf("[DEBUG] driver.docker: attempted to stop non-existent container %s", h.containerID)
  1299  			return nil
  1300  		}
  1301  		h.logger.Printf("[ERR] driver.docker: failed to stop container %s: %v", h.containerID, err)
  1302  		return fmt.Errorf("Failed to stop container %s: %s", h.containerID, err)
  1303  	}
  1304  	h.logger.Printf("[INFO] driver.docker: stopped container %s", h.containerID)
  1305  	return nil
  1306  }
  1307  
  1308  func (h *DockerHandle) Stats() (*cstructs.TaskResourceUsage, error) {
  1309  	h.resourceUsageLock.RLock()
  1310  	defer h.resourceUsageLock.RUnlock()
  1311  	var err error
  1312  	if h.resourceUsage == nil {
  1313  		err = fmt.Errorf("stats collection hasn't started yet")
  1314  	}
  1315  	return h.resourceUsage, err
  1316  }
  1317  
  1318  func (h *DockerHandle) run() {
  1319  	// Wait for it...
  1320  	exitCode, werr := h.waitClient.WaitContainer(h.containerID)
  1321  	if werr != nil {
  1322  		h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated", h.containerID)
  1323  	}
  1324  
  1325  	if exitCode != 0 {
  1326  		werr = fmt.Errorf("Docker container exited with non-zero exit code: %d", exitCode)
  1327  	}
  1328  
  1329  	close(h.doneCh)
  1330  
  1331  	// Remove services
  1332  	if err := h.executor.DeregisterServices(); err != nil {
  1333  		h.logger.Printf("[ERR] driver.docker: error deregistering services: %v", err)
  1334  	}
  1335  
  1336  	// Shutdown the syslog collector
  1337  	if err := h.executor.Exit(); err != nil {
  1338  		h.logger.Printf("[ERR] driver.docker: failed to kill the syslog collector: %v", err)
  1339  	}
  1340  	h.pluginClient.Kill()
  1341  
  1342  	// Stop the container just incase the docker daemon's wait returned
  1343  	// incorrectly
  1344  	if err := h.client.StopContainer(h.containerID, 0); err != nil {
  1345  		_, noSuchContainer := err.(*docker.NoSuchContainer)
  1346  		_, containerNotRunning := err.(*docker.ContainerNotRunning)
  1347  		if !containerNotRunning && !noSuchContainer {
  1348  			h.logger.Printf("[ERR] driver.docker: error stopping container: %v", err)
  1349  		}
  1350  	}
  1351  
  1352  	// Remove the container
  1353  	if err := h.client.RemoveContainer(docker.RemoveContainerOptions{ID: h.containerID, RemoveVolumes: true, Force: true}); err != nil {
  1354  		h.logger.Printf("[ERR] driver.docker: error removing container: %v", err)
  1355  	}
  1356  
  1357  	// Send the results
  1358  	h.waitCh <- dstructs.NewWaitResult(exitCode, 0, werr)
  1359  	close(h.waitCh)
  1360  }
  1361  
  1362  // collectStats starts collecting resource usage stats of a docker container
  1363  func (h *DockerHandle) collectStats() {
  1364  	statsCh := make(chan *docker.Stats)
  1365  	statsOpts := docker.StatsOptions{ID: h.containerID, Done: h.doneCh, Stats: statsCh, Stream: true}
  1366  	go func() {
  1367  		//TODO handle Stats error
  1368  		if err := h.waitClient.Stats(statsOpts); err != nil {
  1369  			h.logger.Printf("[DEBUG] driver.docker: error collecting stats from container %s: %v", h.containerID, err)
  1370  		}
  1371  	}()
  1372  	numCores := runtime.NumCPU()
  1373  	for {
  1374  		select {
  1375  		case s := <-statsCh:
  1376  			if s != nil {
  1377  				ms := &cstructs.MemoryStats{
  1378  					RSS:      s.MemoryStats.Stats.Rss,
  1379  					Cache:    s.MemoryStats.Stats.Cache,
  1380  					Swap:     s.MemoryStats.Stats.Swap,
  1381  					MaxUsage: s.MemoryStats.MaxUsage,
  1382  					Measured: DockerMeasuredMemStats,
  1383  				}
  1384  
  1385  				cs := &cstructs.CpuStats{
  1386  					ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods,
  1387  					ThrottledTime:    s.CPUStats.ThrottlingData.ThrottledTime,
  1388  					Measured:         DockerMeasuredCpuStats,
  1389  				}
  1390  
  1391  				// Calculate percentage
  1392  				cores := len(s.CPUStats.CPUUsage.PercpuUsage)
  1393  				cs.Percent = calculatePercent(
  1394  					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage,
  1395  					s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, cores)
  1396  				cs.SystemMode = calculatePercent(
  1397  					s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode,
  1398  					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores)
  1399  				cs.UserMode = calculatePercent(
  1400  					s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode,
  1401  					s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, cores)
  1402  				cs.TotalTicks = (cs.Percent / 100) * shelpers.TotalTicksAvailable() / float64(numCores)
  1403  
  1404  				h.resourceUsageLock.Lock()
  1405  				h.resourceUsage = &cstructs.TaskResourceUsage{
  1406  					ResourceUsage: &cstructs.ResourceUsage{
  1407  						MemoryStats: ms,
  1408  						CpuStats:    cs,
  1409  					},
  1410  					Timestamp: s.Read.UTC().UnixNano(),
  1411  				}
  1412  				h.resourceUsageLock.Unlock()
  1413  			}
  1414  		case <-h.doneCh:
  1415  			return
  1416  		}
  1417  	}
  1418  }
  1419  
  1420  func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 {
  1421  	numerator := newSample - oldSample
  1422  	denom := newTotal - oldTotal
  1423  	if numerator <= 0 || denom <= 0 {
  1424  		return 0.0
  1425  	}
  1426  
  1427  	return (float64(numerator) / float64(denom)) * float64(cores) * 100.0
  1428  }
  1429  
  1430  // authOptionFrom takes the Docker auth config file and the repo being pulled
  1431  // and returns an AuthConfiguration or an error if the file/repo could not be
  1432  // parsed or looked up.
  1433  func authOptionFrom(file, repo string) (*docker.AuthConfiguration, error) {
  1434  	name, err := reference.ParseNamed(repo)
  1435  	if err != nil {
  1436  		return nil, fmt.Errorf("Failed to parse named repo %q: %v", repo, err)
  1437  	}
  1438  
  1439  	repoInfo, err := registry.ParseRepositoryInfo(name)
  1440  	if err != nil {
  1441  		return nil, fmt.Errorf("Failed to parse repository: %v", err)
  1442  	}
  1443  
  1444  	f, err := os.Open(file)
  1445  	if err != nil {
  1446  		return nil, fmt.Errorf("Failed to open auth config file: %v, error: %v", file, err)
  1447  	}
  1448  	defer f.Close()
  1449  
  1450  	cfile := new(configfile.ConfigFile)
  1451  	if err := cfile.LoadFromReader(f); err != nil {
  1452  		return nil, fmt.Errorf("Failed to parse auth config file: %v", err)
  1453  	}
  1454  
  1455  	dockerAuthConfig := registry.ResolveAuthConfig(cfile.AuthConfigs, repoInfo.Index)
  1456  
  1457  	// Convert to Api version
  1458  	apiAuthConfig := &docker.AuthConfiguration{
  1459  		Username:      dockerAuthConfig.Username,
  1460  		Password:      dockerAuthConfig.Password,
  1461  		Email:         dockerAuthConfig.Email,
  1462  		ServerAddress: dockerAuthConfig.ServerAddress,
  1463  	}
  1464  
  1465  	return apiAuthConfig, nil
  1466  }