github.com/ncodes/nomad@v0.5.7-0.20170403112158-97adf4a74fb3/client/allocdir/alloc_dir.go (about)

     1  package allocdir
     2  
     3  import (
     4  	"archive/tar"
     5  	"fmt"
     6  	"io"
     7  	"io/ioutil"
     8  	"log"
     9  	"os"
    10  	"path/filepath"
    11  	"time"
    12  
    13  	"gopkg.in/tomb.v1"
    14  
    15  	"github.com/hashicorp/go-multierror"
    16  	"github.com/ncodes/nomad/nomad/structs"
    17  	"github.com/hpcloud/tail/watch"
    18  )
    19  
    20  var (
    21  	// The name of the directory that is shared across tasks in a task group.
    22  	SharedAllocName = "alloc"
    23  
    24  	// Name of the directory where logs of Tasks are written
    25  	LogDirName = "logs"
    26  
    27  	// SharedDataDir is one of the shared allocation directories. It is
    28  	// included in snapshots.
    29  	SharedDataDir = "data"
    30  
    31  	// The set of directories that exist inside eache shared alloc directory.
    32  	SharedAllocDirs = []string{LogDirName, "tmp", SharedDataDir}
    33  
    34  	// The name of the directory that exists inside each task directory
    35  	// regardless of driver.
    36  	TaskLocal = "local"
    37  
    38  	// TaskSecrets is the name of the secret directory inside each task
    39  	// directory
    40  	TaskSecrets = "secrets"
    41  
    42  	// TaskDirs is the set of directories created in each tasks directory.
    43  	TaskDirs = []string{"tmp"}
    44  )
    45  
    46  type AllocDir struct {
    47  	// AllocDir is the directory used for storing any state
    48  	// of this allocation. It will be purged on alloc destroy.
    49  	AllocDir string
    50  
    51  	// The shared directory is available to all tasks within the same task
    52  	// group.
    53  	SharedDir string
    54  
    55  	// TaskDirs is a mapping of task names to their non-shared directory.
    56  	TaskDirs map[string]*TaskDir
    57  
    58  	logger *log.Logger
    59  }
    60  
    61  // AllocFileInfo holds information about a file inside the AllocDir
    62  type AllocFileInfo struct {
    63  	Name     string
    64  	IsDir    bool
    65  	Size     int64
    66  	FileMode string
    67  	ModTime  time.Time
    68  }
    69  
    70  // AllocDirFS exposes file operations on the alloc dir
    71  type AllocDirFS interface {
    72  	List(path string) ([]*AllocFileInfo, error)
    73  	Stat(path string) (*AllocFileInfo, error)
    74  	ReadAt(path string, offset int64) (io.ReadCloser, error)
    75  	Snapshot(w io.Writer) error
    76  	BlockUntilExists(path string, t *tomb.Tomb) (chan error, error)
    77  	ChangeEvents(path string, curOffset int64, t *tomb.Tomb) (*watch.FileChanges, error)
    78  }
    79  
    80  // NewAllocDir initializes the AllocDir struct with allocDir as base path for
    81  // the allocation directory.
    82  func NewAllocDir(logger *log.Logger, allocDir string) *AllocDir {
    83  	return &AllocDir{
    84  		AllocDir:  allocDir,
    85  		SharedDir: filepath.Join(allocDir, SharedAllocName),
    86  		TaskDirs:  make(map[string]*TaskDir),
    87  		logger:    logger,
    88  	}
    89  }
    90  
    91  // NewTaskDir creates a new TaskDir and adds it to the AllocDirs TaskDirs map.
    92  func (d *AllocDir) NewTaskDir(name string) *TaskDir {
    93  	td := newTaskDir(d.logger, d.AllocDir, name)
    94  	d.TaskDirs[name] = td
    95  	return td
    96  }
    97  
    98  // Snapshot creates an archive of the files and directories in the data dir of
    99  // the allocation and the task local directories
   100  func (d *AllocDir) Snapshot(w io.Writer) error {
   101  	allocDataDir := filepath.Join(d.SharedDir, SharedDataDir)
   102  	rootPaths := []string{allocDataDir}
   103  	for _, taskdir := range d.TaskDirs {
   104  		rootPaths = append(rootPaths, taskdir.LocalDir)
   105  	}
   106  
   107  	tw := tar.NewWriter(w)
   108  	defer tw.Close()
   109  
   110  	walkFn := func(path string, fileInfo os.FileInfo, err error) error {
   111  		// Ignore if the file is a symlink
   112  		if fileInfo.Mode() == os.ModeSymlink {
   113  			return nil
   114  		}
   115  
   116  		// Include the path of the file name relative to the alloc dir
   117  		// so that we can put the files in the right directories
   118  		relPath, err := filepath.Rel(d.AllocDir, path)
   119  		if err != nil {
   120  			return err
   121  		}
   122  		hdr, err := tar.FileInfoHeader(fileInfo, "")
   123  		if err != nil {
   124  			return fmt.Errorf("error creating file header: %v", err)
   125  		}
   126  		hdr.Name = relPath
   127  		tw.WriteHeader(hdr)
   128  
   129  		// If it's a directory we just write the header into the tar
   130  		if fileInfo.IsDir() {
   131  			return nil
   132  		}
   133  
   134  		// Write the file into the archive
   135  		file, err := os.Open(path)
   136  		if err != nil {
   137  			return err
   138  		}
   139  		defer file.Close()
   140  
   141  		if _, err := io.Copy(tw, file); err != nil {
   142  			return err
   143  		}
   144  		return nil
   145  	}
   146  
   147  	// Walk through all the top level directories and add the files and
   148  	// directories in the archive
   149  	for _, path := range rootPaths {
   150  		if err := filepath.Walk(path, walkFn); err != nil {
   151  			return err
   152  		}
   153  	}
   154  
   155  	return nil
   156  }
   157  
   158  // Move other alloc directory's shared path and local dir to this alloc dir.
   159  func (d *AllocDir) Move(other *AllocDir, tasks []*structs.Task) error {
   160  	// Move the data directory
   161  	otherDataDir := filepath.Join(other.SharedDir, SharedDataDir)
   162  	dataDir := filepath.Join(d.SharedDir, SharedDataDir)
   163  	if fileInfo, err := os.Stat(otherDataDir); fileInfo != nil && err == nil {
   164  		os.Remove(dataDir) // remove an empty data dir if it exists
   165  		if err := os.Rename(otherDataDir, dataDir); err != nil {
   166  			return fmt.Errorf("error moving data dir: %v", err)
   167  		}
   168  	}
   169  
   170  	// Move the task directories
   171  	for _, task := range tasks {
   172  		otherTaskDir := filepath.Join(other.AllocDir, task.Name)
   173  		otherTaskLocal := filepath.Join(otherTaskDir, TaskLocal)
   174  
   175  		fileInfo, err := os.Stat(otherTaskLocal)
   176  		if fileInfo != nil && err == nil {
   177  			// TaskDirs haven't been built yet, so create it
   178  			newTaskDir := filepath.Join(d.AllocDir, task.Name)
   179  			if err := os.MkdirAll(newTaskDir, 0777); err != nil {
   180  				return fmt.Errorf("error creating task %q dir: %v", task.Name, err)
   181  			}
   182  			localDir := filepath.Join(newTaskDir, TaskLocal)
   183  			os.Remove(localDir) // remove an empty local dir if it exists
   184  			if err := os.Rename(otherTaskLocal, localDir); err != nil {
   185  				return fmt.Errorf("error moving task %q local dir: %v", task.Name, err)
   186  			}
   187  		}
   188  	}
   189  
   190  	return nil
   191  }
   192  
   193  // Tears down previously build directory structure.
   194  func (d *AllocDir) Destroy() error {
   195  
   196  	// Unmount all mounted shared alloc dirs.
   197  	var mErr multierror.Error
   198  	if err := d.UnmountAll(); err != nil {
   199  		mErr.Errors = append(mErr.Errors, err)
   200  	}
   201  
   202  	if err := os.RemoveAll(d.AllocDir); err != nil {
   203  		mErr.Errors = append(mErr.Errors, fmt.Errorf("failed to remove alloc dir %q: %v", d.AllocDir, err))
   204  	}
   205  
   206  	return mErr.ErrorOrNil()
   207  }
   208  
   209  // UnmountAll linked/mounted directories in task dirs.
   210  func (d *AllocDir) UnmountAll() error {
   211  	var mErr multierror.Error
   212  	for _, dir := range d.TaskDirs {
   213  		// Check if the directory has the shared alloc mounted.
   214  		if pathExists(dir.SharedTaskDir) {
   215  			if err := unlinkDir(dir.SharedTaskDir); err != nil {
   216  				mErr.Errors = append(mErr.Errors,
   217  					fmt.Errorf("failed to unmount shared alloc dir %q: %v", dir.SharedTaskDir, err))
   218  			} else if err := os.RemoveAll(dir.SharedTaskDir); err != nil {
   219  				mErr.Errors = append(mErr.Errors,
   220  					fmt.Errorf("failed to delete shared alloc dir %q: %v", dir.SharedTaskDir, err))
   221  			}
   222  		}
   223  
   224  		if pathExists(dir.SecretsDir) {
   225  			if err := removeSecretDir(dir.SecretsDir); err != nil {
   226  				mErr.Errors = append(mErr.Errors,
   227  					fmt.Errorf("failed to remove the secret dir %q: %v", dir.SecretsDir, err))
   228  			}
   229  		}
   230  
   231  		// Unmount dev/ and proc/ have been mounted.
   232  		if err := dir.unmountSpecialDirs(); err != nil {
   233  			mErr.Errors = append(mErr.Errors, err)
   234  		}
   235  	}
   236  
   237  	return mErr.ErrorOrNil()
   238  }
   239  
   240  // Build the directory tree for an allocation.
   241  func (d *AllocDir) Build() error {
   242  	// Make the alloc directory, owned by the nomad process.
   243  	if err := os.MkdirAll(d.AllocDir, 0755); err != nil {
   244  		return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err)
   245  	}
   246  
   247  	// Make the shared directory and make it available to all user/groups.
   248  	if err := os.MkdirAll(d.SharedDir, 0777); err != nil {
   249  		return err
   250  	}
   251  
   252  	// Make the shared directory have non-root permissions.
   253  	if err := dropDirPermissions(d.SharedDir); err != nil {
   254  		return err
   255  	}
   256  
   257  	// Create shared subdirs
   258  	for _, dir := range SharedAllocDirs {
   259  		p := filepath.Join(d.SharedDir, dir)
   260  		if err := os.MkdirAll(p, 0777); err != nil {
   261  			return err
   262  		}
   263  		if err := dropDirPermissions(p); err != nil {
   264  			return err
   265  		}
   266  	}
   267  
   268  	return nil
   269  }
   270  
   271  // List returns the list of files at a path relative to the alloc dir
   272  func (d *AllocDir) List(path string) ([]*AllocFileInfo, error) {
   273  	if escapes, err := structs.PathEscapesAllocDir("", path); err != nil {
   274  		return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err)
   275  	} else if escapes {
   276  		return nil, fmt.Errorf("Path escapes the alloc directory")
   277  	}
   278  
   279  	p := filepath.Join(d.AllocDir, path)
   280  	finfos, err := ioutil.ReadDir(p)
   281  	if err != nil {
   282  		return []*AllocFileInfo{}, err
   283  	}
   284  	files := make([]*AllocFileInfo, len(finfos))
   285  	for idx, info := range finfos {
   286  		files[idx] = &AllocFileInfo{
   287  			Name:     info.Name(),
   288  			IsDir:    info.IsDir(),
   289  			Size:     info.Size(),
   290  			FileMode: info.Mode().String(),
   291  			ModTime:  info.ModTime(),
   292  		}
   293  	}
   294  	return files, err
   295  }
   296  
   297  // Stat returns information about the file at a path relative to the alloc dir
   298  func (d *AllocDir) Stat(path string) (*AllocFileInfo, error) {
   299  	if escapes, err := structs.PathEscapesAllocDir("", path); err != nil {
   300  		return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err)
   301  	} else if escapes {
   302  		return nil, fmt.Errorf("Path escapes the alloc directory")
   303  	}
   304  
   305  	p := filepath.Join(d.AllocDir, path)
   306  	info, err := os.Stat(p)
   307  	if err != nil {
   308  		return nil, err
   309  	}
   310  
   311  	return &AllocFileInfo{
   312  		Size:     info.Size(),
   313  		Name:     info.Name(),
   314  		IsDir:    info.IsDir(),
   315  		FileMode: info.Mode().String(),
   316  		ModTime:  info.ModTime(),
   317  	}, nil
   318  }
   319  
   320  // ReadAt returns a reader for a file at the path relative to the alloc dir
   321  func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) {
   322  	if escapes, err := structs.PathEscapesAllocDir("", path); err != nil {
   323  		return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err)
   324  	} else if escapes {
   325  		return nil, fmt.Errorf("Path escapes the alloc directory")
   326  	}
   327  
   328  	p := filepath.Join(d.AllocDir, path)
   329  
   330  	// Check if it is trying to read into a secret directory
   331  	for _, dir := range d.TaskDirs {
   332  		if filepath.HasPrefix(p, dir.SecretsDir) {
   333  			return nil, fmt.Errorf("Reading secret file prohibited: %s", path)
   334  		}
   335  	}
   336  
   337  	f, err := os.Open(p)
   338  	if err != nil {
   339  		return nil, err
   340  	}
   341  	if _, err := f.Seek(offset, 0); err != nil {
   342  		return nil, fmt.Errorf("can't seek to offset %q: %v", offset, err)
   343  	}
   344  	return f, nil
   345  }
   346  
   347  // BlockUntilExists blocks until the passed file relative the allocation
   348  // directory exists. The block can be cancelled with the passed tomb.
   349  func (d *AllocDir) BlockUntilExists(path string, t *tomb.Tomb) (chan error, error) {
   350  	if escapes, err := structs.PathEscapesAllocDir("", path); err != nil {
   351  		return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err)
   352  	} else if escapes {
   353  		return nil, fmt.Errorf("Path escapes the alloc directory")
   354  	}
   355  
   356  	// Get the path relative to the alloc directory
   357  	p := filepath.Join(d.AllocDir, path)
   358  	watcher := getFileWatcher(p)
   359  	returnCh := make(chan error, 1)
   360  	go func() {
   361  		returnCh <- watcher.BlockUntilExists(t)
   362  		close(returnCh)
   363  	}()
   364  	return returnCh, nil
   365  }
   366  
   367  // ChangeEvents watches for changes to the passed path relative to the
   368  // allocation directory. The offset should be the last read offset. The tomb is
   369  // used to clean up the watch.
   370  func (d *AllocDir) ChangeEvents(path string, curOffset int64, t *tomb.Tomb) (*watch.FileChanges, error) {
   371  	if escapes, err := structs.PathEscapesAllocDir("", path); err != nil {
   372  		return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err)
   373  	} else if escapes {
   374  		return nil, fmt.Errorf("Path escapes the alloc directory")
   375  	}
   376  
   377  	// Get the path relative to the alloc directory
   378  	p := filepath.Join(d.AllocDir, path)
   379  	watcher := getFileWatcher(p)
   380  	return watcher.ChangeEvents(t, curOffset)
   381  }
   382  
   383  // getFileWatcher returns a FileWatcher for the given path.
   384  func getFileWatcher(path string) watch.FileWatcher {
   385  	return watch.NewPollingFileWatcher(path)
   386  }
   387  
   388  func fileCopy(src, dst string, perm os.FileMode) error {
   389  	// Do a simple copy.
   390  	srcFile, err := os.Open(src)
   391  	if err != nil {
   392  		return fmt.Errorf("Couldn't open src file %v: %v", src, err)
   393  	}
   394  	defer srcFile.Close()
   395  
   396  	dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, perm)
   397  	if err != nil {
   398  		return fmt.Errorf("Couldn't create destination file %v: %v", dst, err)
   399  	}
   400  	defer dstFile.Close()
   401  
   402  	if _, err := io.Copy(dstFile, srcFile); err != nil {
   403  		return fmt.Errorf("Couldn't copy %v to %v: %v", src, dst, err)
   404  	}
   405  
   406  	return nil
   407  }
   408  
   409  // pathExists is a helper function to check if the path exists.
   410  func pathExists(path string) bool {
   411  	if _, err := os.Stat(path); err != nil {
   412  		if os.IsNotExist(err) {
   413  			return false
   414  		}
   415  	}
   416  	return true
   417  }
   418  
   419  // pathEmpty returns true if a path exists, is listable, and is empty. If the
   420  // path does not exist or is not listable an error is returned.
   421  func pathEmpty(path string) (bool, error) {
   422  	f, err := os.Open(path)
   423  	if err != nil {
   424  		return false, err
   425  	}
   426  	defer f.Close()
   427  	entries, err := f.Readdir(1)
   428  	if err != nil && err != io.EOF {
   429  		return false, err
   430  	}
   431  	return len(entries) == 0, nil
   432  }
   433  
   434  // createDir creates a directory structure inside the basepath. This functions
   435  // preserves the permissions of each of the subdirectories in the relative path
   436  // by looking up the permissions in the host.
   437  func createDir(basePath, relPath string) error {
   438  	filePerms, err := splitPath(relPath)
   439  	if err != nil {
   440  		return err
   441  	}
   442  
   443  	// We are going backwards since we create the root of the directory first
   444  	// and then create the entire nested structure.
   445  	for i := len(filePerms) - 1; i >= 0; i-- {
   446  		fi := filePerms[i]
   447  		destDir := filepath.Join(basePath, fi.Name)
   448  		if err := os.MkdirAll(destDir, fi.Perm); err != nil {
   449  			return err
   450  		}
   451  	}
   452  	return nil
   453  }
   454  
   455  // fileInfo holds the path and the permissions of a file
   456  type fileInfo struct {
   457  	Name string
   458  	Perm os.FileMode
   459  }
   460  
   461  // splitPath stats each subdirectory of a path. The first element of the array
   462  // is the file passed to this function, and the last element is the root of the
   463  // path.
   464  func splitPath(path string) ([]fileInfo, error) {
   465  	var mode os.FileMode
   466  	i, err := os.Stat(path)
   467  
   468  	// If the path is not present in the host then we respond with the most
   469  	// flexible permission.
   470  	if err != nil {
   471  		mode = os.ModePerm
   472  	} else {
   473  		mode = i.Mode()
   474  	}
   475  	var dirs []fileInfo
   476  	dirs = append(dirs, fileInfo{Name: path, Perm: mode})
   477  	currentDir := path
   478  	for {
   479  		dir := filepath.Dir(filepath.Clean(currentDir))
   480  		if dir == currentDir {
   481  			break
   482  		}
   483  
   484  		// We try to find the permission of the file in the host. If the path is not
   485  		// present in the host then we respond with the most flexible permission.
   486  		i, err = os.Stat(dir)
   487  		if err != nil {
   488  			mode = os.ModePerm
   489  		} else {
   490  			mode = i.Mode()
   491  		}
   492  		dirs = append(dirs, fileInfo{Name: dir, Perm: mode})
   493  		currentDir = dir
   494  	}
   495  	return dirs, nil
   496  }