github.com/dkerwin/nomad@v0.3.3-0.20160525181927-74554135514b/client/driver/docker.go (about)

     1  package driver
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"log"
     7  	"net"
     8  	"os"
     9  	"os/exec"
    10  	"path/filepath"
    11  	"regexp"
    12  	"strconv"
    13  	"strings"
    14  	"sync"
    15  	"time"
    16  
    17  	docker "github.com/fsouza/go-dockerclient"
    18  
    19  	"github.com/hashicorp/go-multierror"
    20  	"github.com/hashicorp/go-plugin"
    21  	"github.com/hashicorp/nomad/client/allocdir"
    22  	"github.com/hashicorp/nomad/client/config"
    23  	"github.com/hashicorp/nomad/client/driver/executor"
    24  	cstructs "github.com/hashicorp/nomad/client/driver/structs"
    25  	"github.com/hashicorp/nomad/helper/discover"
    26  	"github.com/hashicorp/nomad/helper/fields"
    27  	"github.com/hashicorp/nomad/nomad/structs"
    28  	"github.com/mitchellh/mapstructure"
    29  )
    30  
    31  var (
    32  	// We store the client globally to cache the connection to the docker daemon.
    33  	createClient sync.Once
    34  	client       *docker.Client
    35  )
    36  
    37  const (
    38  	// NoSuchContainerError is returned by the docker daemon if the container
    39  	// does not exist.
    40  	NoSuchContainerError = "No such container"
    41  
    42  	// The key populated in Node Attributes to indicate presence of the Docker
    43  	// driver
    44  	dockerDriverAttr = "driver.docker"
    45  
    46  	// dockerTimeout is the length of time a request can be outstanding before
    47  	// it is timed out.
    48  	dockerTimeout = 1 * time.Minute
    49  )
    50  
    51  type DockerDriver struct {
    52  	DriverContext
    53  }
    54  
    55  type DockerDriverAuth struct {
    56  	Username      string `mapstructure:"username"`       // username for the registry
    57  	Password      string `mapstructure:"password"`       // password to access the registry
    58  	Email         string `mapstructure:"email"`          // email address of the user who is allowed to access the registry
    59  	ServerAddress string `mapstructure:"server_address"` // server address of the registry
    60  }
    61  
    62  type DockerDriverConfig struct {
    63  	ImageName        string              `mapstructure:"image"`              // Container's Image Name
    64  	LoadImages       []string            `mapstructure:"load"`               // LoadImage is array of paths to image archive files
    65  	Command          string              `mapstructure:"command"`            // The Command/Entrypoint to run when the container starts up
    66  	Args             []string            `mapstructure:"args"`               // The arguments to the Command/Entrypoint
    67  	IpcMode          string              `mapstructure:"ipc_mode"`           // The IPC mode of the container - host and none
    68  	NetworkMode      string              `mapstructure:"network_mode"`       // The network mode of the container - host, net and none
    69  	PidMode          string              `mapstructure:"pid_mode"`           // The PID mode of the container - host and none
    70  	UTSMode          string              `mapstructure:"uts_mode"`           // The UTS mode of the container - host and none
    71  	PortMapRaw       []map[string]int    `mapstructure:"port_map"`           //
    72  	PortMap          map[string]int      `mapstructure:"-"`                  // A map of host port labels and the ports exposed on the container
    73  	Privileged       bool                `mapstructure:"privileged"`         // Flag to run the container in privileged mode
    74  	DNSServers       []string            `mapstructure:"dns_servers"`        // DNS Server for containers
    75  	DNSSearchDomains []string            `mapstructure:"dns_search_domains"` // DNS Search domains for containers
    76  	Hostname         string              `mapstructure:"hostname"`           // Hostname for containers
    77  	LabelsRaw        []map[string]string `mapstructure:"labels"`             //
    78  	Labels           map[string]string   `mapstructure:"-"`                  // Labels to set when the container starts up
    79  	Auth             []DockerDriverAuth  `mapstructure:"auth"`               // Authentication credentials for a private Docker registry
    80  	SSL              bool                `mapstructure:"ssl"`                // Flag indicating repository is served via https
    81  	TTY              bool                `mapstructure:"tty"`                // Allocate a Pseudo-TTY
    82  	Interactive      bool                `mapstructure:"interactive"`        // Keep STDIN open even if not attached
    83  }
    84  
    85  func (c *DockerDriverConfig) Init() error {
    86  	if strings.Contains(c.ImageName, "https://") {
    87  		c.SSL = true
    88  		c.ImageName = strings.Replace(c.ImageName, "https://", "", 1)
    89  	}
    90  
    91  	return nil
    92  }
    93  
    94  func (c *DockerDriverConfig) Validate() error {
    95  	if c.ImageName == "" {
    96  		return fmt.Errorf("Docker Driver needs an image name")
    97  	}
    98  
    99  	c.PortMap = mapMergeStrInt(c.PortMapRaw...)
   100  	c.Labels = mapMergeStrStr(c.LabelsRaw...)
   101  
   102  	return nil
   103  }
   104  
   105  type dockerPID struct {
   106  	Version        string
   107  	ImageID        string
   108  	ContainerID    string
   109  	KillTimeout    time.Duration
   110  	MaxKillTimeout time.Duration
   111  	PluginConfig   *PluginReattachConfig
   112  }
   113  
   114  type DockerHandle struct {
   115  	pluginClient   *plugin.Client
   116  	executor       executor.Executor
   117  	client         *docker.Client
   118  	logger         *log.Logger
   119  	cleanupImage   bool
   120  	imageID        string
   121  	containerID    string
   122  	version        string
   123  	killTimeout    time.Duration
   124  	maxKillTimeout time.Duration
   125  	waitCh         chan *cstructs.WaitResult
   126  	doneCh         chan struct{}
   127  }
   128  
   129  func NewDockerDriver(ctx *DriverContext) Driver {
   130  	return &DockerDriver{DriverContext: *ctx}
   131  }
   132  
   133  // Validate is used to validate the driver configuration
   134  func (d *DockerDriver) Validate(config map[string]interface{}) error {
   135  	fd := &fields.FieldData{
   136  		Raw: config,
   137  		Schema: map[string]*fields.FieldSchema{
   138  			"image": &fields.FieldSchema{
   139  				Type:     fields.TypeString,
   140  				Required: true,
   141  			},
   142  			"load": &fields.FieldSchema{
   143  				Type: fields.TypeArray,
   144  			},
   145  			"command": &fields.FieldSchema{
   146  				Type: fields.TypeString,
   147  			},
   148  			"args": &fields.FieldSchema{
   149  				Type: fields.TypeArray,
   150  			},
   151  			"ipc_mode": &fields.FieldSchema{
   152  				Type: fields.TypeString,
   153  			},
   154  			"network_mode": &fields.FieldSchema{
   155  				Type: fields.TypeString,
   156  			},
   157  			"pid_mode": &fields.FieldSchema{
   158  				Type: fields.TypeString,
   159  			},
   160  			"uts_mode": &fields.FieldSchema{
   161  				Type: fields.TypeString,
   162  			},
   163  			"port_map": &fields.FieldSchema{
   164  				Type: fields.TypeArray,
   165  			},
   166  			"privileged": &fields.FieldSchema{
   167  				Type: fields.TypeBool,
   168  			},
   169  			"dns_servers": &fields.FieldSchema{
   170  				Type: fields.TypeArray,
   171  			},
   172  			"dns_search_domains": &fields.FieldSchema{
   173  				Type: fields.TypeArray,
   174  			},
   175  			"hostname": &fields.FieldSchema{
   176  				Type: fields.TypeString,
   177  			},
   178  			"labels": &fields.FieldSchema{
   179  				Type: fields.TypeArray,
   180  			},
   181  			"auth": &fields.FieldSchema{
   182  				Type: fields.TypeArray,
   183  			},
   184  			"ssl": &fields.FieldSchema{
   185  				Type: fields.TypeBool,
   186  			},
   187  			"tty": &fields.FieldSchema{
   188  				Type: fields.TypeBool,
   189  			},
   190  			"interactive": &fields.FieldSchema{
   191  				Type: fields.TypeBool,
   192  			},
   193  		},
   194  	}
   195  
   196  	if err := fd.Validate(); err != nil {
   197  		return err
   198  	}
   199  
   200  	return nil
   201  }
   202  
   203  // dockerClient creates *docker.Client. In test / dev mode we can use ENV vars
   204  // to connect to the docker daemon. In production mode we will read
   205  // docker.endpoint from the config file.
   206  func (d *DockerDriver) dockerClient() (*docker.Client, error) {
   207  	if client != nil {
   208  		return client, nil
   209  	}
   210  
   211  	var err error
   212  	createClient.Do(func() {
   213  		// Default to using whatever is configured in docker.endpoint. If this is
   214  		// not specified we'll fall back on NewClientFromEnv which reads config from
   215  		// the DOCKER_* environment variables DOCKER_HOST, DOCKER_TLS_VERIFY, and
   216  		// DOCKER_CERT_PATH. This allows us to lock down the config in production
   217  		// but also accept the standard ENV configs for dev and test.
   218  		dockerEndpoint := d.config.Read("docker.endpoint")
   219  		if dockerEndpoint != "" {
   220  			cert := d.config.Read("docker.tls.cert")
   221  			key := d.config.Read("docker.tls.key")
   222  			ca := d.config.Read("docker.tls.ca")
   223  
   224  			if cert+key+ca != "" {
   225  				d.logger.Printf("[DEBUG] driver.docker: using TLS client connection to %s", dockerEndpoint)
   226  				client, err = docker.NewTLSClient(dockerEndpoint, cert, key, ca)
   227  			} else {
   228  				d.logger.Printf("[DEBUG] driver.docker: using standard client connection to %s", dockerEndpoint)
   229  				client, err = docker.NewClient(dockerEndpoint)
   230  			}
   231  			client.HTTPClient.Timeout = dockerTimeout
   232  			return
   233  		}
   234  
   235  		d.logger.Println("[DEBUG] driver.docker: using client connection initialized from environment")
   236  		client, err = docker.NewClientFromEnv()
   237  		client.HTTPClient.Timeout = dockerTimeout
   238  	})
   239  	return client, err
   240  }
   241  
   242  func (d *DockerDriver) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
   243  	// Get the current status so that we can log any debug messages only if the
   244  	// state changes
   245  	_, currentlyEnabled := node.Attributes[dockerDriverAttr]
   246  
   247  	// Initialize docker API client
   248  	client, err := d.dockerClient()
   249  	if err != nil {
   250  		delete(node.Attributes, dockerDriverAttr)
   251  		if currentlyEnabled {
   252  			d.logger.Printf("[INFO] driver.docker: failed to initialize client: %s", err)
   253  		}
   254  		return false, nil
   255  	}
   256  
   257  	privileged := d.config.ReadBoolDefault("docker.privileged.enabled", false)
   258  	if privileged {
   259  		node.Attributes["docker.privileged.enabled"] = "1"
   260  	}
   261  
   262  	// This is the first operation taken on the client so we'll try to
   263  	// establish a connection to the Docker daemon. If this fails it means
   264  	// Docker isn't available so we'll simply disable the docker driver.
   265  	env, err := client.Version()
   266  	if err != nil {
   267  		if currentlyEnabled {
   268  			d.logger.Printf("[DEBUG] driver.docker: could not connect to docker daemon at %s: %s", client.Endpoint(), err)
   269  		}
   270  		delete(node.Attributes, dockerDriverAttr)
   271  		return false, nil
   272  	}
   273  
   274  	node.Attributes[dockerDriverAttr] = "1"
   275  	node.Attributes["driver.docker.version"] = env.Get("Version")
   276  	return true, nil
   277  }
   278  
   279  func (d *DockerDriver) containerBinds(alloc *allocdir.AllocDir, task *structs.Task) ([]string, error) {
   280  	shared := alloc.SharedDir
   281  	local, ok := alloc.TaskDirs[task.Name]
   282  	if !ok {
   283  		return nil, fmt.Errorf("Failed to find task local directory: %v", task.Name)
   284  	}
   285  
   286  	return []string{
   287  		// "z" and "Z" option is to allocate directory with SELinux label.
   288  		fmt.Sprintf("%s:/%s:rw,z", shared, allocdir.SharedAllocName),
   289  		// capital "Z" will label with Multi-Category Security (MCS) labels
   290  		fmt.Sprintf("%s:/%s:rw,Z", local, allocdir.TaskLocal),
   291  	}, nil
   292  }
   293  
   294  // createContainer initializes a struct needed to call docker.client.CreateContainer()
   295  func (d *DockerDriver) createContainer(ctx *ExecContext, task *structs.Task,
   296  	driverConfig *DockerDriverConfig, syslogAddr string) (docker.CreateContainerOptions, error) {
   297  	var c docker.CreateContainerOptions
   298  	if task.Resources == nil {
   299  		// Guard against missing resources. We should never have been able to
   300  		// schedule a job without specifying this.
   301  		d.logger.Println("[ERR] driver.docker: task.Resources is empty")
   302  		return c, fmt.Errorf("task.Resources is empty")
   303  	}
   304  
   305  	binds, err := d.containerBinds(ctx.AllocDir, task)
   306  	if err != nil {
   307  		return c, err
   308  	}
   309  
   310  	// Set environment variables.
   311  	d.taskEnv.SetAllocDir(filepath.Join("/", allocdir.SharedAllocName))
   312  	d.taskEnv.SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal))
   313  
   314  	config := &docker.Config{
   315  		Image:     driverConfig.ImageName,
   316  		Hostname:  driverConfig.Hostname,
   317  		User:      task.User,
   318  		Tty:       driverConfig.TTY,
   319  		OpenStdin: driverConfig.Interactive,
   320  	}
   321  
   322  	hostConfig := &docker.HostConfig{
   323  		// Convert MB to bytes. This is an absolute value.
   324  		Memory:     int64(task.Resources.MemoryMB) * 1024 * 1024,
   325  		MemorySwap: -1,
   326  		// Convert Mhz to shares. This is a relative value.
   327  		CPUShares: int64(task.Resources.CPU),
   328  
   329  		// Binds are used to mount a host volume into the container. We mount a
   330  		// local directory for storage and a shared alloc directory that can be
   331  		// used to share data between different tasks in the same task group.
   332  		Binds: binds,
   333  		LogConfig: docker.LogConfig{
   334  			Type: "syslog",
   335  			Config: map[string]string{
   336  				"syslog-address": syslogAddr,
   337  			},
   338  		},
   339  	}
   340  
   341  	d.logger.Printf("[DEBUG] driver.docker: using %d bytes memory for %s", hostConfig.Memory, task.Config["image"])
   342  	d.logger.Printf("[DEBUG] driver.docker: using %d cpu shares for %s", hostConfig.CPUShares, task.Config["image"])
   343  	d.logger.Printf("[DEBUG] driver.docker: binding directories %#v for %s", hostConfig.Binds, task.Config["image"])
   344  
   345  	//  set privileged mode
   346  	hostPrivileged := d.config.ReadBoolDefault("docker.privileged.enabled", false)
   347  	if driverConfig.Privileged && !hostPrivileged {
   348  		return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent`)
   349  	}
   350  	hostConfig.Privileged = hostPrivileged
   351  
   352  	// set DNS servers
   353  	for _, ip := range driverConfig.DNSServers {
   354  		if net.ParseIP(ip) != nil {
   355  			hostConfig.DNS = append(hostConfig.DNS, ip)
   356  		} else {
   357  			d.logger.Printf("[ERR] driver.docker: invalid ip address for container dns server: %s", ip)
   358  		}
   359  	}
   360  
   361  	// set DNS search domains
   362  	for _, domain := range driverConfig.DNSSearchDomains {
   363  		hostConfig.DNSSearch = append(hostConfig.DNSSearch, domain)
   364  	}
   365  
   366  	if driverConfig.IpcMode != "" {
   367  		if !hostPrivileged {
   368  			return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting ipc mode not allowed`)
   369  		}
   370  		d.logger.Printf("[DEBUG] driver.docker: setting ipc mode to %s", driverConfig.IpcMode)
   371  	}
   372  	hostConfig.IpcMode = driverConfig.IpcMode
   373  
   374  	if driverConfig.PidMode != "" {
   375  		if !hostPrivileged {
   376  			return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting pid mode not allowed`)
   377  		}
   378  		d.logger.Printf("[DEBUG] driver.docker: setting pid mode to %s", driverConfig.PidMode)
   379  	}
   380  	hostConfig.PidMode = driverConfig.PidMode
   381  
   382  	if driverConfig.UTSMode != "" {
   383  		if !hostPrivileged {
   384  			return c, fmt.Errorf(`Docker privileged mode is disabled on this Nomad agent, setting UTS mode not allowed`)
   385  		}
   386  		d.logger.Printf("[DEBUG] driver.docker: setting UTS mode to %s", driverConfig.UTSMode)
   387  	}
   388  	hostConfig.UTSMode = driverConfig.UTSMode
   389  
   390  	hostConfig.NetworkMode = driverConfig.NetworkMode
   391  	if hostConfig.NetworkMode == "" {
   392  		// docker default
   393  		d.logger.Println("[DEBUG] driver.docker: networking mode not specified; defaulting to bridge")
   394  		hostConfig.NetworkMode = "bridge"
   395  	}
   396  
   397  	// Setup port mapping and exposed ports
   398  	if len(task.Resources.Networks) == 0 {
   399  		d.logger.Println("[DEBUG] driver.docker: No network interfaces are available")
   400  		if len(driverConfig.PortMap) > 0 {
   401  			return c, fmt.Errorf("Trying to map ports but no network interface is available")
   402  		}
   403  	} else {
   404  		// TODO add support for more than one network
   405  		network := task.Resources.Networks[0]
   406  		publishedPorts := map[docker.Port][]docker.PortBinding{}
   407  		exposedPorts := map[docker.Port]struct{}{}
   408  
   409  		for _, port := range network.ReservedPorts {
   410  			// By default we will map the allocated port 1:1 to the container
   411  			containerPortInt := port.Value
   412  
   413  			// If the user has mapped a port using port_map we'll change it here
   414  			if mapped, ok := driverConfig.PortMap[port.Label]; ok {
   415  				containerPortInt = mapped
   416  			}
   417  
   418  			hostPortStr := strconv.Itoa(port.Value)
   419  			containerPort := docker.Port(strconv.Itoa(containerPortInt))
   420  
   421  			publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
   422  			publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
   423  			d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (static)", network.IP, port.Value, port.Value)
   424  
   425  			exposedPorts[containerPort+"/tcp"] = struct{}{}
   426  			exposedPorts[containerPort+"/udp"] = struct{}{}
   427  			d.logger.Printf("[DEBUG] driver.docker: exposed port %d", port.Value)
   428  		}
   429  
   430  		for _, port := range network.DynamicPorts {
   431  			// By default we will map the allocated port 1:1 to the container
   432  			containerPortInt := port.Value
   433  
   434  			// If the user has mapped a port using port_map we'll change it here
   435  			if mapped, ok := driverConfig.PortMap[port.Label]; ok {
   436  				containerPortInt = mapped
   437  			}
   438  
   439  			hostPortStr := strconv.Itoa(port.Value)
   440  			containerPort := docker.Port(strconv.Itoa(containerPortInt))
   441  
   442  			publishedPorts[containerPort+"/tcp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
   443  			publishedPorts[containerPort+"/udp"] = []docker.PortBinding{docker.PortBinding{HostIP: network.IP, HostPort: hostPortStr}}
   444  			d.logger.Printf("[DEBUG] driver.docker: allocated port %s:%d -> %d (mapped)", network.IP, port.Value, containerPortInt)
   445  
   446  			exposedPorts[containerPort+"/tcp"] = struct{}{}
   447  			exposedPorts[containerPort+"/udp"] = struct{}{}
   448  			d.logger.Printf("[DEBUG] driver.docker: exposed port %s", containerPort)
   449  		}
   450  
   451  		d.taskEnv.SetPortMap(driverConfig.PortMap)
   452  
   453  		hostConfig.PortBindings = publishedPorts
   454  		config.ExposedPorts = exposedPorts
   455  	}
   456  
   457  	d.taskEnv.Build()
   458  	parsedArgs := d.taskEnv.ParseAndReplace(driverConfig.Args)
   459  
   460  	// If the user specified a custom command to run as their entrypoint, we'll
   461  	// inject it here.
   462  	if driverConfig.Command != "" {
   463  		// Validate command
   464  		if err := validateCommand(driverConfig.Command, "args"); err != nil {
   465  			return c, err
   466  		}
   467  
   468  		cmd := []string{driverConfig.Command}
   469  		if len(driverConfig.Args) != 0 {
   470  			cmd = append(cmd, parsedArgs...)
   471  		}
   472  		d.logger.Printf("[DEBUG] driver.docker: setting container startup command to: %s", strings.Join(cmd, " "))
   473  		config.Cmd = cmd
   474  	} else if len(driverConfig.Args) != 0 {
   475  		d.logger.Println("[DEBUG] driver.docker: ignoring command arguments because command is not specified")
   476  	}
   477  
   478  	if len(driverConfig.Labels) > 0 {
   479  		config.Labels = driverConfig.Labels
   480  		d.logger.Printf("[DEBUG] driver.docker: applied labels on the container: %+v", config.Labels)
   481  	}
   482  
   483  	config.Env = d.taskEnv.EnvList()
   484  
   485  	containerName := fmt.Sprintf("%s-%s", task.Name, ctx.AllocID)
   486  	d.logger.Printf("[DEBUG] driver.docker: setting container name to: %s", containerName)
   487  
   488  	return docker.CreateContainerOptions{
   489  		Name:       containerName,
   490  		Config:     config,
   491  		HostConfig: hostConfig,
   492  	}, nil
   493  }
   494  
   495  var (
   496  	// imageNotFoundMatcher is a regex expression that matches the image not
   497  	// found error Docker returns.
   498  	imageNotFoundMatcher = regexp.MustCompile(`Error: image .+ not found`)
   499  )
   500  
   501  // recoverablePullError wraps the error gotten when trying to pull and image if
   502  // the error is recoverable.
   503  func (d *DockerDriver) recoverablePullError(err error, image string) error {
   504  	recoverable := true
   505  	if imageNotFoundMatcher.MatchString(err.Error()) {
   506  		recoverable = false
   507  	}
   508  	return cstructs.NewRecoverableError(fmt.Errorf("Failed to pull `%s`: %s", image, err), recoverable)
   509  }
   510  
   511  func (d *DockerDriver) Periodic() (bool, time.Duration) {
   512  	return true, 15 * time.Second
   513  }
   514  
   515  // createImage creates a docker image either by pulling it from a registry or by
   516  // loading it from the file system
   517  func (d *DockerDriver) createImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir string) error {
   518  	image := driverConfig.ImageName
   519  	repo, tag := docker.ParseRepositoryTag(image)
   520  	if tag == "" {
   521  		tag = "latest"
   522  	}
   523  
   524  	var dockerImage *docker.Image
   525  	var err error
   526  	// We're going to check whether the image is already downloaded. If the tag
   527  	// is "latest" we have to check for a new version every time so we don't
   528  	// bother to check and cache the id here. We'll download first, then cache.
   529  	if tag != "latest" {
   530  		dockerImage, err = client.InspectImage(image)
   531  	}
   532  
   533  	// Download the image
   534  	if dockerImage == nil {
   535  		if len(driverConfig.LoadImages) > 0 {
   536  			return d.loadImage(driverConfig, client, taskDir)
   537  		}
   538  
   539  		return d.pullImage(driverConfig, client, repo, tag)
   540  	}
   541  	return err
   542  }
   543  
   544  // pullImage creates an image by pulling it from a docker registry
   545  func (d *DockerDriver) pullImage(driverConfig *DockerDriverConfig, client *docker.Client, repo string, tag string) error {
   546  	pullOptions := docker.PullImageOptions{
   547  		Repository: repo,
   548  		Tag:        tag,
   549  	}
   550  
   551  	authOptions := docker.AuthConfiguration{}
   552  	if len(driverConfig.Auth) != 0 {
   553  		authOptions = docker.AuthConfiguration{
   554  			Username:      driverConfig.Auth[0].Username,
   555  			Password:      driverConfig.Auth[0].Password,
   556  			Email:         driverConfig.Auth[0].Email,
   557  			ServerAddress: driverConfig.Auth[0].ServerAddress,
   558  		}
   559  	}
   560  
   561  	if authConfigFile := d.config.Read("docker.auth.config"); authConfigFile != "" {
   562  		if f, err := os.Open(authConfigFile); err == nil {
   563  			defer f.Close()
   564  			var authConfigurations *docker.AuthConfigurations
   565  			if authConfigurations, err = docker.NewAuthConfigurations(f); err != nil {
   566  				return fmt.Errorf("Failed to create docker auth object: %v", err)
   567  			}
   568  
   569  			authConfigurationKey := ""
   570  			if driverConfig.SSL {
   571  				authConfigurationKey += "https://"
   572  			}
   573  
   574  			authConfigurationKey += strings.Split(driverConfig.ImageName, "/")[0]
   575  			if authConfiguration, ok := authConfigurations.Configs[authConfigurationKey]; ok {
   576  				authOptions = authConfiguration
   577  			}
   578  		} else {
   579  			return fmt.Errorf("Failed to open auth config file: %v, error: %v", authConfigFile, err)
   580  		}
   581  	}
   582  
   583  	err := client.PullImage(pullOptions, authOptions)
   584  	if err != nil {
   585  		d.logger.Printf("[ERR] driver.docker: failed pulling container %s:%s: %s", repo, tag, err)
   586  		return d.recoverablePullError(err, driverConfig.ImageName)
   587  	}
   588  	d.logger.Printf("[DEBUG] driver.docker: docker pull %s:%s succeeded", repo, tag)
   589  	return nil
   590  }
   591  
   592  // loadImage creates an image by loading it from the file system
   593  func (d *DockerDriver) loadImage(driverConfig *DockerDriverConfig, client *docker.Client, taskDir string) error {
   594  	var errors multierror.Error
   595  	for _, image := range driverConfig.LoadImages {
   596  		archive := filepath.Join(taskDir, allocdir.TaskLocal, image)
   597  		d.logger.Printf("[DEBUG] driver.docker: loading image from: %v", archive)
   598  		f, err := os.Open(archive)
   599  		if err != nil {
   600  			errors.Errors = append(errors.Errors, fmt.Errorf("unable to open image archive: %v", err))
   601  			continue
   602  		}
   603  		if err := client.LoadImage(docker.LoadImageOptions{InputStream: f}); err != nil {
   604  			errors.Errors = append(errors.Errors, err)
   605  		}
   606  		f.Close()
   607  	}
   608  	return errors.ErrorOrNil()
   609  }
   610  
   611  func (d *DockerDriver) Start(ctx *ExecContext, task *structs.Task) (DriverHandle, error) {
   612  	var driverConfig DockerDriverConfig
   613  	if err := mapstructure.WeakDecode(task.Config, &driverConfig); err != nil {
   614  		return nil, err
   615  	}
   616  
   617  	if err := driverConfig.Init(); err != nil {
   618  		return nil, err
   619  	}
   620  
   621  	if err := driverConfig.Validate(); err != nil {
   622  		return nil, err
   623  	}
   624  
   625  	cleanupImage := d.config.ReadBoolDefault("docker.cleanup.image", true)
   626  
   627  	taskDir, ok := ctx.AllocDir.TaskDirs[d.DriverContext.taskName]
   628  	if !ok {
   629  		return nil, fmt.Errorf("Could not find task directory for task: %v", d.DriverContext.taskName)
   630  	}
   631  
   632  	// Initialize docker API client
   633  	client, err := d.dockerClient()
   634  	if err != nil {
   635  		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
   636  	}
   637  
   638  	if err := d.createImage(&driverConfig, client, taskDir); err != nil {
   639  		return nil, fmt.Errorf("failed to create image: %v", err)
   640  	}
   641  
   642  	image := driverConfig.ImageName
   643  	// Now that we have the image we can get the image id
   644  	dockerImage, err := client.InspectImage(image)
   645  	if err != nil {
   646  		d.logger.Printf("[ERR] driver.docker: failed getting image id for %s: %s", image, err)
   647  		return nil, fmt.Errorf("Failed to determine image id for `%s`: %s", image, err)
   648  	}
   649  	d.logger.Printf("[DEBUG] driver.docker: identified image %s as %s", image, dockerImage.ID)
   650  
   651  	bin, err := discover.NomadExecutable()
   652  	if err != nil {
   653  		return nil, fmt.Errorf("unable to find the nomad binary: %v", err)
   654  	}
   655  	pluginLogFile := filepath.Join(taskDir, fmt.Sprintf("%s-executor.out", task.Name))
   656  	pluginConfig := &plugin.ClientConfig{
   657  		Cmd: exec.Command(bin, "executor", pluginLogFile),
   658  	}
   659  
   660  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   661  	if err != nil {
   662  		return nil, err
   663  	}
   664  	executorCtx := &executor.ExecutorContext{
   665  		TaskEnv:        d.taskEnv,
   666  		Task:           task,
   667  		Driver:         "docker",
   668  		AllocDir:       ctx.AllocDir,
   669  		AllocID:        ctx.AllocID,
   670  		PortLowerBound: d.config.ClientMinPort,
   671  		PortUpperBound: d.config.ClientMaxPort,
   672  	}
   673  	ss, err := exec.LaunchSyslogServer(executorCtx)
   674  	if err != nil {
   675  		return nil, fmt.Errorf("failed to start syslog collector: %v", err)
   676  	}
   677  
   678  	config, err := d.createContainer(ctx, task, &driverConfig, ss.Addr)
   679  	if err != nil {
   680  		d.logger.Printf("[ERR] driver.docker: failed to create container configuration for image %s: %s", image, err)
   681  		pluginClient.Kill()
   682  		return nil, fmt.Errorf("Failed to create container configuration for image %s: %s", image, err)
   683  	}
   684  	// Create a container
   685  	container, err := client.CreateContainer(config)
   686  	if err != nil {
   687  		// If the container already exists because of a previous failure we'll
   688  		// try to purge it and re-create it.
   689  		if strings.Contains(err.Error(), "container already exists") {
   690  			// Get the ID of the existing container so we can delete it
   691  			containers, err := client.ListContainers(docker.ListContainersOptions{
   692  				// The image might be in use by a stopped container, so check everything
   693  				All: true,
   694  				Filters: map[string][]string{
   695  					"name": []string{config.Name},
   696  				},
   697  			})
   698  			if err != nil {
   699  				d.logger.Printf("[ERR] driver.docker: failed to query list of containers matching name:%s", config.Name)
   700  				pluginClient.Kill()
   701  				return nil, fmt.Errorf("Failed to query list of containers: %s", err)
   702  			}
   703  
   704  			// Couldn't find any matching containers
   705  			if len(containers) == 0 {
   706  				d.logger.Printf("[ERR] driver.docker: failed to get id for container %s: %#v", config.Name, containers)
   707  				pluginClient.Kill()
   708  				return nil, fmt.Errorf("Failed to get id for container %s", config.Name)
   709  			}
   710  
   711  			// Delete matching containers
   712  			d.logger.Printf("[INFO] driver.docker: a container with the name %s already exists; will attempt to purge and re-create", config.Name)
   713  			for _, container := range containers {
   714  				err = client.RemoveContainer(docker.RemoveContainerOptions{
   715  					ID: container.ID,
   716  				})
   717  				if err != nil {
   718  					d.logger.Printf("[ERR] driver.docker: failed to purge container %s", container.ID)
   719  					pluginClient.Kill()
   720  					return nil, fmt.Errorf("Failed to purge container %s: %s", container.ID, err)
   721  				}
   722  				d.logger.Printf("[INFO] driver.docker: purged container %s", container.ID)
   723  			}
   724  
   725  			container, err = client.CreateContainer(config)
   726  			if err != nil {
   727  				d.logger.Printf("[ERR] driver.docker: failed to re-create container %s; aborting", config.Name)
   728  				pluginClient.Kill()
   729  				return nil, fmt.Errorf("Failed to re-create container %s; aborting", config.Name)
   730  			}
   731  		} else {
   732  			// We failed to create the container for some other reason.
   733  			d.logger.Printf("[ERR] driver.docker: failed to create container from image %s: %s", image, err)
   734  			pluginClient.Kill()
   735  			return nil, fmt.Errorf("Failed to create container from image %s: %s", image, err)
   736  		}
   737  	}
   738  	d.logger.Printf("[INFO] driver.docker: created container %s", container.ID)
   739  
   740  	// Start the container
   741  	err = client.StartContainer(container.ID, container.HostConfig)
   742  	if err != nil {
   743  		d.logger.Printf("[ERR] driver.docker: failed to start container %s: %s", container.ID, err)
   744  		pluginClient.Kill()
   745  		return nil, fmt.Errorf("Failed to start container %s: %s", container.ID, err)
   746  	}
   747  	d.logger.Printf("[INFO] driver.docker: started container %s", container.ID)
   748  
   749  	// Return a driver handle
   750  	maxKill := d.DriverContext.config.MaxKillTimeout
   751  	h := &DockerHandle{
   752  		client:         client,
   753  		executor:       exec,
   754  		pluginClient:   pluginClient,
   755  		cleanupImage:   cleanupImage,
   756  		logger:         d.logger,
   757  		imageID:        dockerImage.ID,
   758  		containerID:    container.ID,
   759  		version:        d.config.Version,
   760  		killTimeout:    GetKillTimeout(task.KillTimeout, maxKill),
   761  		maxKillTimeout: maxKill,
   762  		doneCh:         make(chan struct{}),
   763  		waitCh:         make(chan *cstructs.WaitResult, 1),
   764  	}
   765  	if err := exec.SyncServices(consulContext(d.config, container.ID)); err != nil {
   766  		d.logger.Printf("[ERR] driver.docker: error registering services with consul for task: %q: %v", task.Name, err)
   767  	}
   768  	go h.run()
   769  	return h, nil
   770  }
   771  
   772  func (d *DockerDriver) Open(ctx *ExecContext, handleID string) (DriverHandle, error) {
   773  	cleanupImage := d.config.ReadBoolDefault("docker.cleanup.image", true)
   774  
   775  	// Split the handle
   776  	pidBytes := []byte(strings.TrimPrefix(handleID, "DOCKER:"))
   777  	pid := &dockerPID{}
   778  	if err := json.Unmarshal(pidBytes, pid); err != nil {
   779  		return nil, fmt.Errorf("Failed to parse handle '%s': %v", handleID, err)
   780  	}
   781  	d.logger.Printf("[INFO] driver.docker: re-attaching to docker process: %s", pid.ContainerID)
   782  	d.logger.Printf("[DEBUG] driver.docker: re-attached to handle: %s", handleID)
   783  	pluginConfig := &plugin.ClientConfig{
   784  		Reattach: pid.PluginConfig.PluginConfig(),
   785  	}
   786  
   787  	client, err := d.dockerClient()
   788  	if err != nil {
   789  		return nil, fmt.Errorf("Failed to connect to docker daemon: %s", err)
   790  	}
   791  
   792  	// Look for a running container with this ID
   793  	containers, err := client.ListContainers(docker.ListContainersOptions{
   794  		Filters: map[string][]string{
   795  			"id": []string{pid.ContainerID},
   796  		},
   797  	})
   798  	if err != nil {
   799  		return nil, fmt.Errorf("Failed to query for container %s: %v", pid.ContainerID, err)
   800  	}
   801  
   802  	found := false
   803  	for _, container := range containers {
   804  		if container.ID == pid.ContainerID {
   805  			found = true
   806  		}
   807  	}
   808  	if !found {
   809  		return nil, fmt.Errorf("Failed to find container %s: %v", pid.ContainerID, err)
   810  	}
   811  	exec, pluginClient, err := createExecutor(pluginConfig, d.config.LogOutput, d.config)
   812  	if err != nil {
   813  		d.logger.Printf("[INFO] driver.docker: couldn't re-attach to the plugin process: %v", err)
   814  		if e := client.StopContainer(pid.ContainerID, uint(pid.KillTimeout*time.Second)); e != nil {
   815  			d.logger.Printf("[DEBUG] driver.docker: couldn't stop container: %v", e)
   816  		}
   817  		return nil, err
   818  	}
   819  
   820  	ver, _ := exec.Version()
   821  	d.logger.Printf("[DEBUG] driver.docker: version of executor: %v", ver.Version)
   822  
   823  	// Return a driver handle
   824  	h := &DockerHandle{
   825  		client:         client,
   826  		executor:       exec,
   827  		pluginClient:   pluginClient,
   828  		cleanupImage:   cleanupImage,
   829  		logger:         d.logger,
   830  		imageID:        pid.ImageID,
   831  		containerID:    pid.ContainerID,
   832  		version:        pid.Version,
   833  		killTimeout:    pid.KillTimeout,
   834  		maxKillTimeout: pid.MaxKillTimeout,
   835  		doneCh:         make(chan struct{}),
   836  		waitCh:         make(chan *cstructs.WaitResult, 1),
   837  	}
   838  	if err := exec.SyncServices(consulContext(d.config, pid.ContainerID)); err != nil {
   839  		h.logger.Printf("[ERR] driver.docker: error registering services with consul: %v", err)
   840  	}
   841  
   842  	go h.run()
   843  	return h, nil
   844  }
   845  
   846  func (h *DockerHandle) ID() string {
   847  	// Return a handle to the PID
   848  	pid := dockerPID{
   849  		Version:        h.version,
   850  		ImageID:        h.imageID,
   851  		ContainerID:    h.containerID,
   852  		KillTimeout:    h.killTimeout,
   853  		MaxKillTimeout: h.maxKillTimeout,
   854  		PluginConfig:   NewPluginReattachConfig(h.pluginClient.ReattachConfig()),
   855  	}
   856  	data, err := json.Marshal(pid)
   857  	if err != nil {
   858  		h.logger.Printf("[ERR] driver.docker: failed to marshal docker PID to JSON: %s", err)
   859  	}
   860  	return fmt.Sprintf("DOCKER:%s", string(data))
   861  }
   862  
   863  func (h *DockerHandle) ContainerID() string {
   864  	return h.containerID
   865  }
   866  
   867  func (h *DockerHandle) WaitCh() chan *cstructs.WaitResult {
   868  	return h.waitCh
   869  }
   870  
   871  func (h *DockerHandle) Update(task *structs.Task) error {
   872  	// Store the updated kill timeout.
   873  	h.killTimeout = GetKillTimeout(task.KillTimeout, h.maxKillTimeout)
   874  	if err := h.executor.UpdateTask(task); err != nil {
   875  		h.logger.Printf("[DEBUG] driver.docker: failed to update log config: %v", err)
   876  	}
   877  
   878  	// Update is not possible
   879  	return nil
   880  }
   881  
   882  // Kill is used to terminate the task. This uses `docker stop -t killTimeout`
   883  func (h *DockerHandle) Kill() error {
   884  	// Stop the container
   885  	err := h.client.StopContainer(h.containerID, uint(h.killTimeout.Seconds()))
   886  	if err != nil {
   887  		h.executor.Exit()
   888  		h.pluginClient.Kill()
   889  
   890  		// Container has already been removed.
   891  		if strings.Contains(err.Error(), NoSuchContainerError) {
   892  			h.logger.Printf("[DEBUG] driver.docker: attempted to stop non-existent container %s", h.containerID)
   893  			return nil
   894  		}
   895  		h.logger.Printf("[ERR] driver.docker: failed to stop container %s: %v", h.containerID, err)
   896  		return fmt.Errorf("Failed to stop container %s: %s", h.containerID, err)
   897  	}
   898  	h.logger.Printf("[INFO] driver.docker: stopped container %s", h.containerID)
   899  	return nil
   900  }
   901  
   902  func (h *DockerHandle) run() {
   903  	// Wait for it...
   904  	exitCode, err := h.client.WaitContainer(h.containerID)
   905  	if err != nil {
   906  		h.logger.Printf("[ERR] driver.docker: failed to wait for %s; container already terminated", h.containerID)
   907  	}
   908  
   909  	if exitCode != 0 {
   910  		err = fmt.Errorf("Docker container exited with non-zero exit code: %d", exitCode)
   911  	}
   912  
   913  	close(h.doneCh)
   914  	h.waitCh <- cstructs.NewWaitResult(exitCode, 0, err)
   915  	close(h.waitCh)
   916  
   917  	// Remove services
   918  	if err := h.executor.DeregisterServices(); err != nil {
   919  		h.logger.Printf("[ERR] driver.docker: error deregistering services: %v", err)
   920  	}
   921  
   922  	// Shutdown the syslog collector
   923  	if err := h.executor.Exit(); err != nil {
   924  		h.logger.Printf("[ERR] driver.docker: failed to kill the syslog collector: %v", err)
   925  	}
   926  	h.pluginClient.Kill()
   927  
   928  	// Stop the container just incase the docker daemon's wait returned
   929  	// incorrectly
   930  	if err := h.client.StopContainer(h.containerID, 0); err != nil {
   931  		_, noSuchContainer := err.(*docker.NoSuchContainer)
   932  		_, containerNotRunning := err.(*docker.ContainerNotRunning)
   933  		if !containerNotRunning && !noSuchContainer {
   934  			h.logger.Printf("[ERR] driver.docker: error stopping container: %v", err)
   935  		}
   936  	}
   937  
   938  	// Remove the container
   939  	if err := h.client.RemoveContainer(docker.RemoveContainerOptions{ID: h.containerID, Force: true}); err != nil {
   940  		h.logger.Printf("[ERR] driver.docker: error removing container: %v", err)
   941  	}
   942  
   943  	// Cleanup the image
   944  	if h.cleanupImage {
   945  		if err := h.client.RemoveImage(h.imageID); err != nil {
   946  			h.logger.Printf("[DEBUG] driver.docker: error removing image: %v", err)
   947  		}
   948  	}
   949  }