github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/vfs/read_write.go (about)

     1  package vfs
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"io/ioutil"
     8  	"os"
     9  	"runtime"
    10  	"sync"
    11  
    12  	"github.com/pkg/errors"
    13  	"github.com/rclone/rclone/fs"
    14  	"github.com/rclone/rclone/fs/log"
    15  	"github.com/rclone/rclone/lib/file"
    16  )
    17  
    18  // RWFileHandle is a handle that can be open for read and write.
    19  //
    20  // It will be open to a temporary file which, when closed, will be
    21  // transferred to the remote.
    22  type RWFileHandle struct {
    23  	*os.File
    24  	mu          sync.Mutex
    25  	closed      bool // set if handle has been closed
    26  	file        *File
    27  	d           *Dir
    28  	opened      bool
    29  	flags       int  // open flags
    30  	writeCalled bool // if any Write() methods have been called
    31  	changed     bool // file contents was changed in any other way
    32  }
    33  
    34  // Check interfaces
    35  var (
    36  	_ io.Reader   = (*RWFileHandle)(nil)
    37  	_ io.ReaderAt = (*RWFileHandle)(nil)
    38  	_ io.Writer   = (*RWFileHandle)(nil)
    39  	_ io.WriterAt = (*RWFileHandle)(nil)
    40  	_ io.Seeker   = (*RWFileHandle)(nil)
    41  	_ io.Closer   = (*RWFileHandle)(nil)
    42  )
    43  
    44  func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) {
    45  	// if O_CREATE and O_EXCL are set and if path already exists, then return EEXIST
    46  	if flags&(os.O_CREATE|os.O_EXCL) == os.O_CREATE|os.O_EXCL && f.exists() {
    47  		return nil, EEXIST
    48  	}
    49  
    50  	fh = &RWFileHandle{
    51  		file:  f,
    52  		d:     d,
    53  		flags: flags,
    54  	}
    55  
    56  	// mark the file as open in the cache - must be done before the mkdir
    57  	fh.d.VFS().cache.Open(fh.file.Path())
    58  
    59  	// Make a place for the file
    60  	_, err = d.VFS().cache.Mkdir(fh.file.Path())
    61  	if err != nil {
    62  		fh.d.VFS().cache.Close(fh.file.Path())
    63  		return nil, errors.Wrap(err, "open RW handle failed to make cache directory")
    64  	}
    65  
    66  	rdwrMode := fh.flags & accessModeMask
    67  	if rdwrMode != os.O_RDONLY {
    68  		fh.file.addWriter(fh)
    69  	}
    70  
    71  	// truncate or create files immediately to prepare the cache
    72  	if fh.flags&os.O_TRUNC != 0 || fh.flags&(os.O_CREATE) != 0 && !f.exists() {
    73  		if err := fh.openPending(false); err != nil {
    74  			fh.file.delWriter(fh, false)
    75  			return nil, err
    76  		}
    77  	}
    78  
    79  	return fh, nil
    80  }
    81  
    82  // openPending opens the file if there is a pending open
    83  //
    84  // call with the lock held
    85  func (fh *RWFileHandle) openPending(truncate bool) (err error) {
    86  	if fh.opened {
    87  		return nil
    88  	}
    89  
    90  	fh.file.muRW.Lock()
    91  	defer fh.file.muRW.Unlock()
    92  
    93  	o := fh.file.getObject()
    94  
    95  	var fd *os.File
    96  	cacheFileOpenFlags := fh.flags
    97  	// if not truncating the file, need to read it first
    98  	if fh.flags&os.O_TRUNC == 0 && !truncate {
    99  		// If the remote object exists AND its cached file exists locally AND there are no
   100  		// other RW handles with it open, then attempt to update it.
   101  		if o != nil && fh.file.rwOpens() == 0 {
   102  			err = fh.d.VFS().cache.Check(context.TODO(), o, fh.file.Path())
   103  			if err != nil {
   104  				return errors.Wrap(err, "open RW handle failed to check cache file")
   105  			}
   106  		}
   107  
   108  		// try to open an existing cache file
   109  		fd, err = file.OpenFile(fh.file.osPath(), cacheFileOpenFlags&^os.O_CREATE, 0600)
   110  		if os.IsNotExist(err) {
   111  			// cache file does not exist, so need to fetch it if we have an object to fetch
   112  			// it from
   113  			if o != nil {
   114  				err = fh.d.VFS().cache.Fetch(context.TODO(), o, fh.file.Path())
   115  				if err != nil {
   116  					cause := errors.Cause(err)
   117  					if cause != fs.ErrorObjectNotFound && cause != fs.ErrorDirNotFound {
   118  						// return any non NotFound errors
   119  						return errors.Wrap(err, "open RW handle failed to cache file")
   120  					}
   121  					// continue here with err=fs.Error{Object,Dir}NotFound
   122  				}
   123  			}
   124  			// if err == nil, then we have cached the file successfully, otherwise err is
   125  			// indicating some kind of non existent file/directory either
   126  			// os.IsNotExist(err) or fs.Error{Object,Dir}NotFound
   127  			if err != nil {
   128  				if fh.flags&os.O_CREATE != 0 {
   129  					// if the object wasn't found AND O_CREATE is set then
   130  					// ignore error as we are about to create the file
   131  					fh.file.setSize(0)
   132  					fh.changed = true
   133  				} else {
   134  					return errors.Wrap(err, "open RW handle failed to cache file")
   135  				}
   136  			}
   137  		} else if err != nil {
   138  			return errors.Wrap(err, "cache open file failed")
   139  		} else {
   140  			fs.Debugf(fh.logPrefix(), "Opened existing cached copy with flags=%s", decodeOpenFlags(fh.flags))
   141  		}
   142  	} else {
   143  		// Set the size to 0 since we are truncating and flag we need to write it back
   144  		fh.file.setSize(0)
   145  		fh.changed = true
   146  		if fh.flags&os.O_CREATE == 0 && fh.file.exists() {
   147  			// create an empty file if it exists on the source
   148  			err = ioutil.WriteFile(fh.file.osPath(), []byte{}, 0600)
   149  			if err != nil {
   150  				return errors.Wrap(err, "cache open failed to create zero length file")
   151  			}
   152  		}
   153  		// Windows doesn't seem to deal well with O_TRUNC and
   154  		// certain access modes so truncate the file if it
   155  		// exists in these cases.
   156  		if runtime.GOOS == "windows" && fh.flags&os.O_APPEND != 0 {
   157  			cacheFileOpenFlags &^= os.O_TRUNC
   158  			_, err = os.Stat(fh.file.osPath())
   159  			if err == nil {
   160  				err = os.Truncate(fh.file.osPath(), 0)
   161  				if err != nil {
   162  					return errors.Wrap(err, "cache open failed to truncate")
   163  				}
   164  			}
   165  		}
   166  	}
   167  
   168  	if fd == nil {
   169  		fs.Debugf(fh.logPrefix(), "Opening cached copy with flags=%s", decodeOpenFlags(fh.flags))
   170  		fd, err = file.OpenFile(fh.file.osPath(), cacheFileOpenFlags, 0600)
   171  		if err != nil {
   172  			return errors.Wrap(err, "cache open file failed")
   173  		}
   174  	}
   175  	fh.File = fd
   176  	fh.opened = true
   177  	fh.file.addRWOpen()
   178  	fh.d.addObject(fh.file) // make sure the directory has this object in it now
   179  	return nil
   180  }
   181  
   182  // String converts it to printable
   183  func (fh *RWFileHandle) String() string {
   184  	if fh == nil {
   185  		return "<nil *RWFileHandle>"
   186  	}
   187  	fh.mu.Lock()
   188  	defer fh.mu.Unlock()
   189  	if fh.file == nil {
   190  		return "<nil *RWFileHandle.file>"
   191  	}
   192  	return fh.file.String() + " (rw)"
   193  }
   194  
   195  // Node returns the Node assocuated with this - satisfies Noder interface
   196  func (fh *RWFileHandle) Node() Node {
   197  	fh.mu.Lock()
   198  	defer fh.mu.Unlock()
   199  	return fh.file
   200  }
   201  
   202  // Returns whether the file needs to be written back.
   203  //
   204  // If write hasn't been called and the file hasn't been changed in any other
   205  // way we haven't modified it so we don't need to transfer it
   206  //
   207  // Must be called with fh.mu held
   208  func (fh *RWFileHandle) modified() bool {
   209  	if !fh.writeCalled && !fh.changed {
   210  		fs.Debugf(fh.logPrefix(), "not modified so not transferring")
   211  		return false
   212  	}
   213  	return true
   214  }
   215  
   216  // flushWrites flushes any pending writes to cloud storage
   217  //
   218  // Must be called with fh.muRW held
   219  func (fh *RWFileHandle) flushWrites(closeFile bool) error {
   220  	if fh.closed && !closeFile {
   221  		return nil
   222  	}
   223  
   224  	rdwrMode := fh.flags & accessModeMask
   225  	writer := rdwrMode != os.O_RDONLY
   226  
   227  	// If read only then return
   228  	if !fh.opened && rdwrMode == os.O_RDONLY {
   229  		return nil
   230  	}
   231  
   232  	isCopied := false
   233  	if writer {
   234  		isCopied = fh.file.delWriter(fh, fh.modified())
   235  		defer fh.file.finishWriterClose()
   236  	}
   237  
   238  	// If we aren't creating or truncating the file then
   239  	// we haven't modified it so don't need to transfer it
   240  	if fh.flags&(os.O_CREATE|os.O_TRUNC) != 0 {
   241  		if err := fh.openPending(false); err != nil {
   242  			return err
   243  		}
   244  	}
   245  
   246  	if writer && fh.opened {
   247  		fi, err := fh.File.Stat()
   248  		if err != nil {
   249  			fs.Errorf(fh.logPrefix(), "Failed to stat cache file: %v", err)
   250  		} else {
   251  			fh.file.setSize(fi.Size())
   252  		}
   253  	}
   254  
   255  	// Close the underlying file
   256  	if fh.opened && closeFile {
   257  		err := fh.File.Close()
   258  		if err != nil {
   259  			err = errors.Wrap(err, "failed to close cache file")
   260  			return err
   261  		}
   262  	}
   263  
   264  	if isCopied {
   265  		o, err := fh.d.VFS().cache.Store(context.TODO(), fh.file.getObject(), fh.file.Path())
   266  		if err != nil {
   267  			fs.Errorf(fh.logPrefix(), "%v", err)
   268  			return err
   269  		}
   270  		fh.file.setObject(o)
   271  		fs.Debugf(o, "transferred to remote")
   272  	}
   273  
   274  	return nil
   275  }
   276  
   277  // close the file handle returning EBADF if it has been
   278  // closed already.
   279  //
   280  // Must be called with fh.mu held
   281  //
   282  // Note that we leave the file around in the cache on error conditions
   283  // to give the user a chance to recover it.
   284  func (fh *RWFileHandle) close() (err error) {
   285  	defer log.Trace(fh.logPrefix(), "")("err=%v", &err)
   286  	fh.file.muRW.Lock()
   287  	defer fh.file.muRW.Unlock()
   288  
   289  	if fh.closed {
   290  		return ECLOSED
   291  	}
   292  	fh.closed = true
   293  	defer func() {
   294  		if fh.opened {
   295  			fh.file.delRWOpen()
   296  		}
   297  		fh.d.VFS().cache.Close(fh.file.Path())
   298  	}()
   299  
   300  	return fh.flushWrites(true)
   301  }
   302  
   303  // Close closes the file
   304  func (fh *RWFileHandle) Close() error {
   305  	fh.mu.Lock()
   306  	defer fh.mu.Unlock()
   307  	return fh.close()
   308  }
   309  
   310  // Flush is called each time the file or directory is closed.
   311  // Because there can be multiple file descriptors referring to a
   312  // single opened file, Flush can be called multiple times.
   313  func (fh *RWFileHandle) Flush() error {
   314  	fh.mu.Lock()
   315  	defer fh.mu.Unlock()
   316  	if !fh.opened {
   317  		return nil
   318  	}
   319  	if fh.closed {
   320  		fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush nothing to do")
   321  		return nil
   322  	}
   323  	// fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush")
   324  	if !fh.opened {
   325  		fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush ignoring flush on unopened handle")
   326  		return nil
   327  	}
   328  
   329  	// If Write hasn't been called then ignore the Flush - Release
   330  	// will pick it up
   331  	if !fh.writeCalled {
   332  		fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush ignoring flush on unwritten handle")
   333  		return nil
   334  	}
   335  
   336  	fh.file.muRW.Lock()
   337  	defer fh.file.muRW.Unlock()
   338  	err := fh.flushWrites(false)
   339  
   340  	if err != nil {
   341  		fs.Errorf(fh.logPrefix(), "RWFileHandle.Flush error: %v", err)
   342  	} else {
   343  		// fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush OK")
   344  	}
   345  	return err
   346  }
   347  
   348  // Release is called when we are finished with the file handle
   349  //
   350  // It isn't called directly from userspace so the error is ignored by
   351  // the kernel
   352  func (fh *RWFileHandle) Release() error {
   353  	fh.mu.Lock()
   354  	defer fh.mu.Unlock()
   355  	if fh.closed {
   356  		fs.Debugf(fh.logPrefix(), "RWFileHandle.Release nothing to do")
   357  		return nil
   358  	}
   359  	fs.Debugf(fh.logPrefix(), "RWFileHandle.Release closing")
   360  	err := fh.close()
   361  	if err != nil {
   362  		fs.Errorf(fh.logPrefix(), "RWFileHandle.Release error: %v", err)
   363  	} else {
   364  		// fs.Debugf(fh.logPrefix(), "RWFileHandle.Release OK")
   365  	}
   366  	return err
   367  }
   368  
   369  // Size returns the size of the underlying file
   370  func (fh *RWFileHandle) Size() int64 {
   371  	fh.mu.Lock()
   372  	defer fh.mu.Unlock()
   373  	if !fh.opened {
   374  		return fh.file.Size()
   375  	}
   376  	fi, err := fh.File.Stat()
   377  	if err != nil {
   378  		return 0
   379  	}
   380  	return fi.Size()
   381  }
   382  
   383  // Stat returns info about the file
   384  func (fh *RWFileHandle) Stat() (os.FileInfo, error) {
   385  	fh.mu.Lock()
   386  	defer fh.mu.Unlock()
   387  	return fh.file, nil
   388  }
   389  
   390  // readFn is a general purpose read function - pass in a closure to do
   391  // the actual read
   392  func (fh *RWFileHandle) readFn(read func() (int, error)) (n int, err error) {
   393  	fh.mu.Lock()
   394  	defer fh.mu.Unlock()
   395  	if fh.closed {
   396  		return 0, ECLOSED
   397  	}
   398  	if fh.flags&accessModeMask == os.O_WRONLY {
   399  		return 0, EBADF
   400  	}
   401  	if err = fh.openPending(false); err != nil {
   402  		return n, err
   403  	}
   404  	return read()
   405  }
   406  
   407  // Read bytes from the file
   408  func (fh *RWFileHandle) Read(b []byte) (n int, err error) {
   409  	return fh.readFn(func() (int, error) {
   410  		return fh.File.Read(b)
   411  	})
   412  }
   413  
   414  // ReadAt bytes from the file at off
   415  func (fh *RWFileHandle) ReadAt(b []byte, off int64) (n int, err error) {
   416  	return fh.readFn(func() (int, error) {
   417  		return fh.File.ReadAt(b, off)
   418  	})
   419  }
   420  
   421  // Seek to new file position
   422  func (fh *RWFileHandle) Seek(offset int64, whence int) (ret int64, err error) {
   423  	fh.mu.Lock()
   424  	defer fh.mu.Unlock()
   425  	if fh.closed {
   426  		return 0, ECLOSED
   427  	}
   428  	if !fh.opened && offset == 0 && whence != 2 {
   429  		return 0, nil
   430  	}
   431  	if err = fh.openPending(false); err != nil {
   432  		return ret, err
   433  	}
   434  	return fh.File.Seek(offset, whence)
   435  }
   436  
   437  // writeFn general purpose write call
   438  //
   439  // Pass a closure to do the actual write
   440  func (fh *RWFileHandle) writeFn(write func() error) (err error) {
   441  	fh.mu.Lock()
   442  	defer fh.mu.Unlock()
   443  	if fh.closed {
   444  		return ECLOSED
   445  	}
   446  	if fh.flags&accessModeMask == os.O_RDONLY {
   447  		return EBADF
   448  	}
   449  	if err = fh.openPending(false); err != nil {
   450  		return err
   451  	}
   452  	fh.writeCalled = true
   453  	err = write()
   454  	if err != nil {
   455  		return err
   456  	}
   457  	fi, err := fh.File.Stat()
   458  	if err != nil {
   459  		return errors.Wrap(err, "failed to stat cache file")
   460  	}
   461  	fh.file.setSize(fi.Size())
   462  	return nil
   463  }
   464  
   465  // Write bytes to the file
   466  func (fh *RWFileHandle) Write(b []byte) (n int, err error) {
   467  	err = fh.writeFn(func() error {
   468  		n, err = fh.File.Write(b)
   469  		return err
   470  	})
   471  	return n, err
   472  }
   473  
   474  // WriteAt bytes to the file at off
   475  func (fh *RWFileHandle) WriteAt(b []byte, off int64) (n int, err error) {
   476  	if fh.flags&os.O_APPEND != 0 {
   477  		// if append is set, call Write as WriteAt returns an error if append is set
   478  		return fh.Write(b)
   479  	}
   480  	err = fh.writeFn(func() error {
   481  		n, err = fh.File.WriteAt(b, off)
   482  		return err
   483  	})
   484  	return n, err
   485  }
   486  
   487  // WriteString a string to the file
   488  func (fh *RWFileHandle) WriteString(s string) (n int, err error) {
   489  	err = fh.writeFn(func() error {
   490  		n, err = fh.File.WriteString(s)
   491  		return err
   492  	})
   493  	return n, err
   494  }
   495  
   496  // Truncate file to given size
   497  func (fh *RWFileHandle) Truncate(size int64) (err error) {
   498  	fh.mu.Lock()
   499  	defer fh.mu.Unlock()
   500  	if fh.closed {
   501  		return ECLOSED
   502  	}
   503  	if err = fh.openPending(size == 0); err != nil {
   504  		return err
   505  	}
   506  	fh.changed = true
   507  	fh.file.setSize(size)
   508  	return fh.File.Truncate(size)
   509  }
   510  
   511  // Sync commits the current contents of the file to stable storage. Typically,
   512  // this means flushing the file system's in-memory copy of recently written
   513  // data to disk.
   514  func (fh *RWFileHandle) Sync() error {
   515  	fh.mu.Lock()
   516  	defer fh.mu.Unlock()
   517  	if fh.closed {
   518  		return ECLOSED
   519  	}
   520  	if !fh.opened {
   521  		return nil
   522  	}
   523  	if fh.flags&accessModeMask == os.O_RDONLY {
   524  		return nil
   525  	}
   526  	return fh.File.Sync()
   527  }
   528  
   529  func (fh *RWFileHandle) logPrefix() string {
   530  	return fmt.Sprintf("%s(%p)", fh.file.Path(), fh)
   531  }