github.com/pachyderm/pachyderm@v1.13.4/src/server/worker/driver/driver_unix.go (about)

     1  // +build !windows
     2  
     3  package driver
     4  
     5  import (
     6  	"os"
     7  	"path/filepath"
     8  	"strings"
     9  	"syscall"
    10  
    11  	"github.com/pachyderm/pachyderm/src/client"
    12  	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
    13  	"github.com/pachyderm/pachyderm/src/server/worker/common"
    14  )
    15  
    16  // Mkfifo does not exist on Windows, so this is left unimplemented there, except for tests
    17  func createSpoutFifo(path string) error {
    18  	return syscall.Mkfifo(path, 0666)
    19  }
    20  
    21  func makeCmdCredentials(uid uint32, gid uint32) *syscall.SysProcAttr {
    22  	return &syscall.SysProcAttr{
    23  		Credential: &syscall.Credential{
    24  			Uid: uid,
    25  			Gid: gid,
    26  		},
    27  	}
    28  }
    29  
    30  // WithActiveData is implemented differently in unix vs windows because of how
    31  // symlinks work on windows. Here, we create symlinks to the scratch space
    32  // directory, then clean up before returning.
    33  func (d *driver) WithActiveData(inputs []*common.Input, dir string, cb func() error) (retErr error) {
    34  	d.activeDataMutex.Lock()
    35  	defer d.activeDataMutex.Unlock()
    36  
    37  	if err := d.linkData(inputs, dir); err != nil {
    38  		return errors.Wrap(err, "error when linking active data directory")
    39  	}
    40  	defer func() {
    41  		if !d.PipelineInfo().S3Out {
    42  			if err := d.rewriteSymlinks(dir); err != nil && retErr == nil {
    43  				retErr = errors.Wrap(err, "error when redirecting symlinks in the active data directory")
    44  			}
    45  		}
    46  		if err := d.unlinkData(inputs); err != nil && retErr == nil {
    47  			retErr = errors.Wrap(err, "error when unlinking active data directory")
    48  		}
    49  	}()
    50  
    51  	return cb()
    52  }
    53  
    54  // When deactivating a data directory, there may be active symlinks from the
    55  // output dir to an input dir. The paths used in these symlinks may be
    56  // invalidated when we deactivate the output directory, so walk the output
    57  // directory and rewrite any such links.
    58  func (d *driver) rewriteSymlinks(scratchSubdir string) error {
    59  	outputDir := filepath.Join(scratchSubdir, "out")
    60  	inputDirFields := strings.Split(filepath.Clean(d.InputDir()), string(filepath.Separator))
    61  	return filepath.Walk(outputDir, func(path string, info os.FileInfo, err error) error {
    62  		if err != nil {
    63  			return err
    64  		}
    65  
    66  		if (info.Mode() & os.ModeSymlink) == 0 {
    67  			return nil
    68  		}
    69  
    70  		target, err := os.Readlink(path)
    71  		if err != nil {
    72  			return err
    73  		}
    74  		if !filepath.IsAbs(target) {
    75  			target, err = filepath.Abs(filepath.Join(filepath.Dir(path), target))
    76  			if err != nil {
    77  				return err
    78  			}
    79  		}
    80  
    81  		// Filter out any symlinks that aren't pointing to files in the active data
    82  		targetFields := strings.Split(filepath.Clean(target), string(filepath.Separator))
    83  		for i, dirname := range inputDirFields {
    84  			if targetFields[i] != dirname {
    85  				return nil
    86  			}
    87  		}
    88  
    89  		// If it's not pointing into the scratch space, we need to change the target
    90  		if targetFields[len(inputDirFields)] != client.PPSScratchSpace {
    91  			targetFields = append([]string{scratchSubdir}, targetFields[len(inputDirFields):]...)
    92  			target = filepath.Join(targetFields...)
    93  		}
    94  
    95  		if err := os.Remove(path); err != nil {
    96  			return err
    97  		}
    98  
    99  		// Always overwrite the symlink at this point, in case it's relative or some dumb shit
   100  		return os.Symlink(filepath.Join(target), filepath.Join(path))
   101  	})
   102  }
   103  
   104  func (d *driver) linkData(inputs []*common.Input, dir string) error {
   105  	// Make sure that the previously-symlinked outputs are removed.
   106  	if err := d.unlinkData(inputs); err != nil {
   107  		return err
   108  	}
   109  
   110  	// sometimes for group inputs, this part may get run multiple times for the same file
   111  	seen := make(map[string]bool)
   112  	for _, input := range inputs {
   113  		if input.S3 {
   114  			continue // S3 data is not downloaded
   115  		}
   116  		if input.Name == "" {
   117  			return errors.New("input does not have a name")
   118  		}
   119  		if _, ok := seen[input.Name]; !ok {
   120  			seen[input.Name] = true
   121  			src := filepath.Join(dir, input.Name)
   122  			dst := filepath.Join(d.InputDir(), input.Name)
   123  			if err := os.Symlink(src, dst); err != nil {
   124  				return err
   125  			}
   126  		}
   127  	}
   128  
   129  	if d.PipelineInfo().Spout != nil && d.PipelineInfo().Spout.Marker != "" {
   130  		if err := os.Symlink(
   131  			filepath.Join(dir, d.PipelineInfo().Spout.Marker),
   132  			filepath.Join(d.InputDir(), d.PipelineInfo().Spout.Marker),
   133  		); err != nil {
   134  			return err
   135  		}
   136  	}
   137  
   138  	if !d.PipelineInfo().S3Out {
   139  		if err := os.Symlink(filepath.Join(dir, "out"), filepath.Join(d.InputDir(), "out")); err != nil {
   140  			return err
   141  		}
   142  	}
   143  
   144  	return nil
   145  }