github.com/ncodes/nomad@v0.5.7-0.20170403112158-97adf4a74fb3/client/allocdir/alloc_dir.go (about) 1 package allocdir 2 3 import ( 4 "archive/tar" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "log" 9 "os" 10 "path/filepath" 11 "time" 12 13 "gopkg.in/tomb.v1" 14 15 "github.com/hashicorp/go-multierror" 16 "github.com/ncodes/nomad/nomad/structs" 17 "github.com/hpcloud/tail/watch" 18 ) 19 20 var ( 21 // The name of the directory that is shared across tasks in a task group. 22 SharedAllocName = "alloc" 23 24 // Name of the directory where logs of Tasks are written 25 LogDirName = "logs" 26 27 // SharedDataDir is one of the shared allocation directories. It is 28 // included in snapshots. 29 SharedDataDir = "data" 30 31 // The set of directories that exist inside eache shared alloc directory. 32 SharedAllocDirs = []string{LogDirName, "tmp", SharedDataDir} 33 34 // The name of the directory that exists inside each task directory 35 // regardless of driver. 36 TaskLocal = "local" 37 38 // TaskSecrets is the name of the secret directory inside each task 39 // directory 40 TaskSecrets = "secrets" 41 42 // TaskDirs is the set of directories created in each tasks directory. 43 TaskDirs = []string{"tmp"} 44 ) 45 46 type AllocDir struct { 47 // AllocDir is the directory used for storing any state 48 // of this allocation. It will be purged on alloc destroy. 49 AllocDir string 50 51 // The shared directory is available to all tasks within the same task 52 // group. 53 SharedDir string 54 55 // TaskDirs is a mapping of task names to their non-shared directory. 56 TaskDirs map[string]*TaskDir 57 58 logger *log.Logger 59 } 60 61 // AllocFileInfo holds information about a file inside the AllocDir 62 type AllocFileInfo struct { 63 Name string 64 IsDir bool 65 Size int64 66 FileMode string 67 ModTime time.Time 68 } 69 70 // AllocDirFS exposes file operations on the alloc dir 71 type AllocDirFS interface { 72 List(path string) ([]*AllocFileInfo, error) 73 Stat(path string) (*AllocFileInfo, error) 74 ReadAt(path string, offset int64) (io.ReadCloser, error) 75 Snapshot(w io.Writer) error 76 BlockUntilExists(path string, t *tomb.Tomb) (chan error, error) 77 ChangeEvents(path string, curOffset int64, t *tomb.Tomb) (*watch.FileChanges, error) 78 } 79 80 // NewAllocDir initializes the AllocDir struct with allocDir as base path for 81 // the allocation directory. 82 func NewAllocDir(logger *log.Logger, allocDir string) *AllocDir { 83 return &AllocDir{ 84 AllocDir: allocDir, 85 SharedDir: filepath.Join(allocDir, SharedAllocName), 86 TaskDirs: make(map[string]*TaskDir), 87 logger: logger, 88 } 89 } 90 91 // NewTaskDir creates a new TaskDir and adds it to the AllocDirs TaskDirs map. 92 func (d *AllocDir) NewTaskDir(name string) *TaskDir { 93 td := newTaskDir(d.logger, d.AllocDir, name) 94 d.TaskDirs[name] = td 95 return td 96 } 97 98 // Snapshot creates an archive of the files and directories in the data dir of 99 // the allocation and the task local directories 100 func (d *AllocDir) Snapshot(w io.Writer) error { 101 allocDataDir := filepath.Join(d.SharedDir, SharedDataDir) 102 rootPaths := []string{allocDataDir} 103 for _, taskdir := range d.TaskDirs { 104 rootPaths = append(rootPaths, taskdir.LocalDir) 105 } 106 107 tw := tar.NewWriter(w) 108 defer tw.Close() 109 110 walkFn := func(path string, fileInfo os.FileInfo, err error) error { 111 // Ignore if the file is a symlink 112 if fileInfo.Mode() == os.ModeSymlink { 113 return nil 114 } 115 116 // Include the path of the file name relative to the alloc dir 117 // so that we can put the files in the right directories 118 relPath, err := filepath.Rel(d.AllocDir, path) 119 if err != nil { 120 return err 121 } 122 hdr, err := tar.FileInfoHeader(fileInfo, "") 123 if err != nil { 124 return fmt.Errorf("error creating file header: %v", err) 125 } 126 hdr.Name = relPath 127 tw.WriteHeader(hdr) 128 129 // If it's a directory we just write the header into the tar 130 if fileInfo.IsDir() { 131 return nil 132 } 133 134 // Write the file into the archive 135 file, err := os.Open(path) 136 if err != nil { 137 return err 138 } 139 defer file.Close() 140 141 if _, err := io.Copy(tw, file); err != nil { 142 return err 143 } 144 return nil 145 } 146 147 // Walk through all the top level directories and add the files and 148 // directories in the archive 149 for _, path := range rootPaths { 150 if err := filepath.Walk(path, walkFn); err != nil { 151 return err 152 } 153 } 154 155 return nil 156 } 157 158 // Move other alloc directory's shared path and local dir to this alloc dir. 159 func (d *AllocDir) Move(other *AllocDir, tasks []*structs.Task) error { 160 // Move the data directory 161 otherDataDir := filepath.Join(other.SharedDir, SharedDataDir) 162 dataDir := filepath.Join(d.SharedDir, SharedDataDir) 163 if fileInfo, err := os.Stat(otherDataDir); fileInfo != nil && err == nil { 164 os.Remove(dataDir) // remove an empty data dir if it exists 165 if err := os.Rename(otherDataDir, dataDir); err != nil { 166 return fmt.Errorf("error moving data dir: %v", err) 167 } 168 } 169 170 // Move the task directories 171 for _, task := range tasks { 172 otherTaskDir := filepath.Join(other.AllocDir, task.Name) 173 otherTaskLocal := filepath.Join(otherTaskDir, TaskLocal) 174 175 fileInfo, err := os.Stat(otherTaskLocal) 176 if fileInfo != nil && err == nil { 177 // TaskDirs haven't been built yet, so create it 178 newTaskDir := filepath.Join(d.AllocDir, task.Name) 179 if err := os.MkdirAll(newTaskDir, 0777); err != nil { 180 return fmt.Errorf("error creating task %q dir: %v", task.Name, err) 181 } 182 localDir := filepath.Join(newTaskDir, TaskLocal) 183 os.Remove(localDir) // remove an empty local dir if it exists 184 if err := os.Rename(otherTaskLocal, localDir); err != nil { 185 return fmt.Errorf("error moving task %q local dir: %v", task.Name, err) 186 } 187 } 188 } 189 190 return nil 191 } 192 193 // Tears down previously build directory structure. 194 func (d *AllocDir) Destroy() error { 195 196 // Unmount all mounted shared alloc dirs. 197 var mErr multierror.Error 198 if err := d.UnmountAll(); err != nil { 199 mErr.Errors = append(mErr.Errors, err) 200 } 201 202 if err := os.RemoveAll(d.AllocDir); err != nil { 203 mErr.Errors = append(mErr.Errors, fmt.Errorf("failed to remove alloc dir %q: %v", d.AllocDir, err)) 204 } 205 206 return mErr.ErrorOrNil() 207 } 208 209 // UnmountAll linked/mounted directories in task dirs. 210 func (d *AllocDir) UnmountAll() error { 211 var mErr multierror.Error 212 for _, dir := range d.TaskDirs { 213 // Check if the directory has the shared alloc mounted. 214 if pathExists(dir.SharedTaskDir) { 215 if err := unlinkDir(dir.SharedTaskDir); err != nil { 216 mErr.Errors = append(mErr.Errors, 217 fmt.Errorf("failed to unmount shared alloc dir %q: %v", dir.SharedTaskDir, err)) 218 } else if err := os.RemoveAll(dir.SharedTaskDir); err != nil { 219 mErr.Errors = append(mErr.Errors, 220 fmt.Errorf("failed to delete shared alloc dir %q: %v", dir.SharedTaskDir, err)) 221 } 222 } 223 224 if pathExists(dir.SecretsDir) { 225 if err := removeSecretDir(dir.SecretsDir); err != nil { 226 mErr.Errors = append(mErr.Errors, 227 fmt.Errorf("failed to remove the secret dir %q: %v", dir.SecretsDir, err)) 228 } 229 } 230 231 // Unmount dev/ and proc/ have been mounted. 232 if err := dir.unmountSpecialDirs(); err != nil { 233 mErr.Errors = append(mErr.Errors, err) 234 } 235 } 236 237 return mErr.ErrorOrNil() 238 } 239 240 // Build the directory tree for an allocation. 241 func (d *AllocDir) Build() error { 242 // Make the alloc directory, owned by the nomad process. 243 if err := os.MkdirAll(d.AllocDir, 0755); err != nil { 244 return fmt.Errorf("Failed to make the alloc directory %v: %v", d.AllocDir, err) 245 } 246 247 // Make the shared directory and make it available to all user/groups. 248 if err := os.MkdirAll(d.SharedDir, 0777); err != nil { 249 return err 250 } 251 252 // Make the shared directory have non-root permissions. 253 if err := dropDirPermissions(d.SharedDir); err != nil { 254 return err 255 } 256 257 // Create shared subdirs 258 for _, dir := range SharedAllocDirs { 259 p := filepath.Join(d.SharedDir, dir) 260 if err := os.MkdirAll(p, 0777); err != nil { 261 return err 262 } 263 if err := dropDirPermissions(p); err != nil { 264 return err 265 } 266 } 267 268 return nil 269 } 270 271 // List returns the list of files at a path relative to the alloc dir 272 func (d *AllocDir) List(path string) ([]*AllocFileInfo, error) { 273 if escapes, err := structs.PathEscapesAllocDir("", path); err != nil { 274 return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err) 275 } else if escapes { 276 return nil, fmt.Errorf("Path escapes the alloc directory") 277 } 278 279 p := filepath.Join(d.AllocDir, path) 280 finfos, err := ioutil.ReadDir(p) 281 if err != nil { 282 return []*AllocFileInfo{}, err 283 } 284 files := make([]*AllocFileInfo, len(finfos)) 285 for idx, info := range finfos { 286 files[idx] = &AllocFileInfo{ 287 Name: info.Name(), 288 IsDir: info.IsDir(), 289 Size: info.Size(), 290 FileMode: info.Mode().String(), 291 ModTime: info.ModTime(), 292 } 293 } 294 return files, err 295 } 296 297 // Stat returns information about the file at a path relative to the alloc dir 298 func (d *AllocDir) Stat(path string) (*AllocFileInfo, error) { 299 if escapes, err := structs.PathEscapesAllocDir("", path); err != nil { 300 return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err) 301 } else if escapes { 302 return nil, fmt.Errorf("Path escapes the alloc directory") 303 } 304 305 p := filepath.Join(d.AllocDir, path) 306 info, err := os.Stat(p) 307 if err != nil { 308 return nil, err 309 } 310 311 return &AllocFileInfo{ 312 Size: info.Size(), 313 Name: info.Name(), 314 IsDir: info.IsDir(), 315 FileMode: info.Mode().String(), 316 ModTime: info.ModTime(), 317 }, nil 318 } 319 320 // ReadAt returns a reader for a file at the path relative to the alloc dir 321 func (d *AllocDir) ReadAt(path string, offset int64) (io.ReadCloser, error) { 322 if escapes, err := structs.PathEscapesAllocDir("", path); err != nil { 323 return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err) 324 } else if escapes { 325 return nil, fmt.Errorf("Path escapes the alloc directory") 326 } 327 328 p := filepath.Join(d.AllocDir, path) 329 330 // Check if it is trying to read into a secret directory 331 for _, dir := range d.TaskDirs { 332 if filepath.HasPrefix(p, dir.SecretsDir) { 333 return nil, fmt.Errorf("Reading secret file prohibited: %s", path) 334 } 335 } 336 337 f, err := os.Open(p) 338 if err != nil { 339 return nil, err 340 } 341 if _, err := f.Seek(offset, 0); err != nil { 342 return nil, fmt.Errorf("can't seek to offset %q: %v", offset, err) 343 } 344 return f, nil 345 } 346 347 // BlockUntilExists blocks until the passed file relative the allocation 348 // directory exists. The block can be cancelled with the passed tomb. 349 func (d *AllocDir) BlockUntilExists(path string, t *tomb.Tomb) (chan error, error) { 350 if escapes, err := structs.PathEscapesAllocDir("", path); err != nil { 351 return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err) 352 } else if escapes { 353 return nil, fmt.Errorf("Path escapes the alloc directory") 354 } 355 356 // Get the path relative to the alloc directory 357 p := filepath.Join(d.AllocDir, path) 358 watcher := getFileWatcher(p) 359 returnCh := make(chan error, 1) 360 go func() { 361 returnCh <- watcher.BlockUntilExists(t) 362 close(returnCh) 363 }() 364 return returnCh, nil 365 } 366 367 // ChangeEvents watches for changes to the passed path relative to the 368 // allocation directory. The offset should be the last read offset. The tomb is 369 // used to clean up the watch. 370 func (d *AllocDir) ChangeEvents(path string, curOffset int64, t *tomb.Tomb) (*watch.FileChanges, error) { 371 if escapes, err := structs.PathEscapesAllocDir("", path); err != nil { 372 return nil, fmt.Errorf("Failed to check if path escapes alloc directory: %v", err) 373 } else if escapes { 374 return nil, fmt.Errorf("Path escapes the alloc directory") 375 } 376 377 // Get the path relative to the alloc directory 378 p := filepath.Join(d.AllocDir, path) 379 watcher := getFileWatcher(p) 380 return watcher.ChangeEvents(t, curOffset) 381 } 382 383 // getFileWatcher returns a FileWatcher for the given path. 384 func getFileWatcher(path string) watch.FileWatcher { 385 return watch.NewPollingFileWatcher(path) 386 } 387 388 func fileCopy(src, dst string, perm os.FileMode) error { 389 // Do a simple copy. 390 srcFile, err := os.Open(src) 391 if err != nil { 392 return fmt.Errorf("Couldn't open src file %v: %v", src, err) 393 } 394 defer srcFile.Close() 395 396 dstFile, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE, perm) 397 if err != nil { 398 return fmt.Errorf("Couldn't create destination file %v: %v", dst, err) 399 } 400 defer dstFile.Close() 401 402 if _, err := io.Copy(dstFile, srcFile); err != nil { 403 return fmt.Errorf("Couldn't copy %v to %v: %v", src, dst, err) 404 } 405 406 return nil 407 } 408 409 // pathExists is a helper function to check if the path exists. 410 func pathExists(path string) bool { 411 if _, err := os.Stat(path); err != nil { 412 if os.IsNotExist(err) { 413 return false 414 } 415 } 416 return true 417 } 418 419 // pathEmpty returns true if a path exists, is listable, and is empty. If the 420 // path does not exist or is not listable an error is returned. 421 func pathEmpty(path string) (bool, error) { 422 f, err := os.Open(path) 423 if err != nil { 424 return false, err 425 } 426 defer f.Close() 427 entries, err := f.Readdir(1) 428 if err != nil && err != io.EOF { 429 return false, err 430 } 431 return len(entries) == 0, nil 432 } 433 434 // createDir creates a directory structure inside the basepath. This functions 435 // preserves the permissions of each of the subdirectories in the relative path 436 // by looking up the permissions in the host. 437 func createDir(basePath, relPath string) error { 438 filePerms, err := splitPath(relPath) 439 if err != nil { 440 return err 441 } 442 443 // We are going backwards since we create the root of the directory first 444 // and then create the entire nested structure. 445 for i := len(filePerms) - 1; i >= 0; i-- { 446 fi := filePerms[i] 447 destDir := filepath.Join(basePath, fi.Name) 448 if err := os.MkdirAll(destDir, fi.Perm); err != nil { 449 return err 450 } 451 } 452 return nil 453 } 454 455 // fileInfo holds the path and the permissions of a file 456 type fileInfo struct { 457 Name string 458 Perm os.FileMode 459 } 460 461 // splitPath stats each subdirectory of a path. The first element of the array 462 // is the file passed to this function, and the last element is the root of the 463 // path. 464 func splitPath(path string) ([]fileInfo, error) { 465 var mode os.FileMode 466 i, err := os.Stat(path) 467 468 // If the path is not present in the host then we respond with the most 469 // flexible permission. 470 if err != nil { 471 mode = os.ModePerm 472 } else { 473 mode = i.Mode() 474 } 475 var dirs []fileInfo 476 dirs = append(dirs, fileInfo{Name: path, Perm: mode}) 477 currentDir := path 478 for { 479 dir := filepath.Dir(filepath.Clean(currentDir)) 480 if dir == currentDir { 481 break 482 } 483 484 // We try to find the permission of the file in the host. If the path is not 485 // present in the host then we respond with the most flexible permission. 486 i, err = os.Stat(dir) 487 if err != nil { 488 mode = os.ModePerm 489 } else { 490 mode = i.Mode() 491 } 492 dirs = append(dirs, fileInfo{Name: dir, Perm: mode}) 493 currentDir = dir 494 } 495 return dirs, nil 496 }