github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/vfs/read_write.go (about) 1 package vfs 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "io/ioutil" 8 "os" 9 "runtime" 10 "sync" 11 12 "github.com/pkg/errors" 13 "github.com/rclone/rclone/fs" 14 "github.com/rclone/rclone/fs/log" 15 "github.com/rclone/rclone/lib/file" 16 ) 17 18 // RWFileHandle is a handle that can be open for read and write. 19 // 20 // It will be open to a temporary file which, when closed, will be 21 // transferred to the remote. 22 type RWFileHandle struct { 23 *os.File 24 mu sync.Mutex 25 closed bool // set if handle has been closed 26 file *File 27 d *Dir 28 opened bool 29 flags int // open flags 30 writeCalled bool // if any Write() methods have been called 31 changed bool // file contents was changed in any other way 32 } 33 34 // Check interfaces 35 var ( 36 _ io.Reader = (*RWFileHandle)(nil) 37 _ io.ReaderAt = (*RWFileHandle)(nil) 38 _ io.Writer = (*RWFileHandle)(nil) 39 _ io.WriterAt = (*RWFileHandle)(nil) 40 _ io.Seeker = (*RWFileHandle)(nil) 41 _ io.Closer = (*RWFileHandle)(nil) 42 ) 43 44 func newRWFileHandle(d *Dir, f *File, flags int) (fh *RWFileHandle, err error) { 45 // if O_CREATE and O_EXCL are set and if path already exists, then return EEXIST 46 if flags&(os.O_CREATE|os.O_EXCL) == os.O_CREATE|os.O_EXCL && f.exists() { 47 return nil, EEXIST 48 } 49 50 fh = &RWFileHandle{ 51 file: f, 52 d: d, 53 flags: flags, 54 } 55 56 // mark the file as open in the cache - must be done before the mkdir 57 fh.d.VFS().cache.Open(fh.file.Path()) 58 59 // Make a place for the file 60 _, err = d.VFS().cache.Mkdir(fh.file.Path()) 61 if err != nil { 62 fh.d.VFS().cache.Close(fh.file.Path()) 63 return nil, errors.Wrap(err, "open RW handle failed to make cache directory") 64 } 65 66 rdwrMode := fh.flags & accessModeMask 67 if rdwrMode != os.O_RDONLY { 68 fh.file.addWriter(fh) 69 } 70 71 // truncate or create files immediately to prepare the cache 72 if fh.flags&os.O_TRUNC != 0 || fh.flags&(os.O_CREATE) != 0 && !f.exists() { 73 if err := fh.openPending(false); err != nil { 74 fh.file.delWriter(fh, false) 75 return nil, err 76 } 77 } 78 79 return fh, nil 80 } 81 82 // openPending opens the file if there is a pending open 83 // 84 // call with the lock held 85 func (fh *RWFileHandle) openPending(truncate bool) (err error) { 86 if fh.opened { 87 return nil 88 } 89 90 fh.file.muRW.Lock() 91 defer fh.file.muRW.Unlock() 92 93 o := fh.file.getObject() 94 95 var fd *os.File 96 cacheFileOpenFlags := fh.flags 97 // if not truncating the file, need to read it first 98 if fh.flags&os.O_TRUNC == 0 && !truncate { 99 // If the remote object exists AND its cached file exists locally AND there are no 100 // other RW handles with it open, then attempt to update it. 101 if o != nil && fh.file.rwOpens() == 0 { 102 err = fh.d.VFS().cache.Check(context.TODO(), o, fh.file.Path()) 103 if err != nil { 104 return errors.Wrap(err, "open RW handle failed to check cache file") 105 } 106 } 107 108 // try to open an existing cache file 109 fd, err = file.OpenFile(fh.file.osPath(), cacheFileOpenFlags&^os.O_CREATE, 0600) 110 if os.IsNotExist(err) { 111 // cache file does not exist, so need to fetch it if we have an object to fetch 112 // it from 113 if o != nil { 114 err = fh.d.VFS().cache.Fetch(context.TODO(), o, fh.file.Path()) 115 if err != nil { 116 cause := errors.Cause(err) 117 if cause != fs.ErrorObjectNotFound && cause != fs.ErrorDirNotFound { 118 // return any non NotFound errors 119 return errors.Wrap(err, "open RW handle failed to cache file") 120 } 121 // continue here with err=fs.Error{Object,Dir}NotFound 122 } 123 } 124 // if err == nil, then we have cached the file successfully, otherwise err is 125 // indicating some kind of non existent file/directory either 126 // os.IsNotExist(err) or fs.Error{Object,Dir}NotFound 127 if err != nil { 128 if fh.flags&os.O_CREATE != 0 { 129 // if the object wasn't found AND O_CREATE is set then 130 // ignore error as we are about to create the file 131 fh.file.setSize(0) 132 fh.changed = true 133 } else { 134 return errors.Wrap(err, "open RW handle failed to cache file") 135 } 136 } 137 } else if err != nil { 138 return errors.Wrap(err, "cache open file failed") 139 } else { 140 fs.Debugf(fh.logPrefix(), "Opened existing cached copy with flags=%s", decodeOpenFlags(fh.flags)) 141 } 142 } else { 143 // Set the size to 0 since we are truncating and flag we need to write it back 144 fh.file.setSize(0) 145 fh.changed = true 146 if fh.flags&os.O_CREATE == 0 && fh.file.exists() { 147 // create an empty file if it exists on the source 148 err = ioutil.WriteFile(fh.file.osPath(), []byte{}, 0600) 149 if err != nil { 150 return errors.Wrap(err, "cache open failed to create zero length file") 151 } 152 } 153 // Windows doesn't seem to deal well with O_TRUNC and 154 // certain access modes so truncate the file if it 155 // exists in these cases. 156 if runtime.GOOS == "windows" && fh.flags&os.O_APPEND != 0 { 157 cacheFileOpenFlags &^= os.O_TRUNC 158 _, err = os.Stat(fh.file.osPath()) 159 if err == nil { 160 err = os.Truncate(fh.file.osPath(), 0) 161 if err != nil { 162 return errors.Wrap(err, "cache open failed to truncate") 163 } 164 } 165 } 166 } 167 168 if fd == nil { 169 fs.Debugf(fh.logPrefix(), "Opening cached copy with flags=%s", decodeOpenFlags(fh.flags)) 170 fd, err = file.OpenFile(fh.file.osPath(), cacheFileOpenFlags, 0600) 171 if err != nil { 172 return errors.Wrap(err, "cache open file failed") 173 } 174 } 175 fh.File = fd 176 fh.opened = true 177 fh.file.addRWOpen() 178 fh.d.addObject(fh.file) // make sure the directory has this object in it now 179 return nil 180 } 181 182 // String converts it to printable 183 func (fh *RWFileHandle) String() string { 184 if fh == nil { 185 return "<nil *RWFileHandle>" 186 } 187 fh.mu.Lock() 188 defer fh.mu.Unlock() 189 if fh.file == nil { 190 return "<nil *RWFileHandle.file>" 191 } 192 return fh.file.String() + " (rw)" 193 } 194 195 // Node returns the Node assocuated with this - satisfies Noder interface 196 func (fh *RWFileHandle) Node() Node { 197 fh.mu.Lock() 198 defer fh.mu.Unlock() 199 return fh.file 200 } 201 202 // Returns whether the file needs to be written back. 203 // 204 // If write hasn't been called and the file hasn't been changed in any other 205 // way we haven't modified it so we don't need to transfer it 206 // 207 // Must be called with fh.mu held 208 func (fh *RWFileHandle) modified() bool { 209 if !fh.writeCalled && !fh.changed { 210 fs.Debugf(fh.logPrefix(), "not modified so not transferring") 211 return false 212 } 213 return true 214 } 215 216 // flushWrites flushes any pending writes to cloud storage 217 // 218 // Must be called with fh.muRW held 219 func (fh *RWFileHandle) flushWrites(closeFile bool) error { 220 if fh.closed && !closeFile { 221 return nil 222 } 223 224 rdwrMode := fh.flags & accessModeMask 225 writer := rdwrMode != os.O_RDONLY 226 227 // If read only then return 228 if !fh.opened && rdwrMode == os.O_RDONLY { 229 return nil 230 } 231 232 isCopied := false 233 if writer { 234 isCopied = fh.file.delWriter(fh, fh.modified()) 235 defer fh.file.finishWriterClose() 236 } 237 238 // If we aren't creating or truncating the file then 239 // we haven't modified it so don't need to transfer it 240 if fh.flags&(os.O_CREATE|os.O_TRUNC) != 0 { 241 if err := fh.openPending(false); err != nil { 242 return err 243 } 244 } 245 246 if writer && fh.opened { 247 fi, err := fh.File.Stat() 248 if err != nil { 249 fs.Errorf(fh.logPrefix(), "Failed to stat cache file: %v", err) 250 } else { 251 fh.file.setSize(fi.Size()) 252 } 253 } 254 255 // Close the underlying file 256 if fh.opened && closeFile { 257 err := fh.File.Close() 258 if err != nil { 259 err = errors.Wrap(err, "failed to close cache file") 260 return err 261 } 262 } 263 264 if isCopied { 265 o, err := fh.d.VFS().cache.Store(context.TODO(), fh.file.getObject(), fh.file.Path()) 266 if err != nil { 267 fs.Errorf(fh.logPrefix(), "%v", err) 268 return err 269 } 270 fh.file.setObject(o) 271 fs.Debugf(o, "transferred to remote") 272 } 273 274 return nil 275 } 276 277 // close the file handle returning EBADF if it has been 278 // closed already. 279 // 280 // Must be called with fh.mu held 281 // 282 // Note that we leave the file around in the cache on error conditions 283 // to give the user a chance to recover it. 284 func (fh *RWFileHandle) close() (err error) { 285 defer log.Trace(fh.logPrefix(), "")("err=%v", &err) 286 fh.file.muRW.Lock() 287 defer fh.file.muRW.Unlock() 288 289 if fh.closed { 290 return ECLOSED 291 } 292 fh.closed = true 293 defer func() { 294 if fh.opened { 295 fh.file.delRWOpen() 296 } 297 fh.d.VFS().cache.Close(fh.file.Path()) 298 }() 299 300 return fh.flushWrites(true) 301 } 302 303 // Close closes the file 304 func (fh *RWFileHandle) Close() error { 305 fh.mu.Lock() 306 defer fh.mu.Unlock() 307 return fh.close() 308 } 309 310 // Flush is called each time the file or directory is closed. 311 // Because there can be multiple file descriptors referring to a 312 // single opened file, Flush can be called multiple times. 313 func (fh *RWFileHandle) Flush() error { 314 fh.mu.Lock() 315 defer fh.mu.Unlock() 316 if !fh.opened { 317 return nil 318 } 319 if fh.closed { 320 fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush nothing to do") 321 return nil 322 } 323 // fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush") 324 if !fh.opened { 325 fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush ignoring flush on unopened handle") 326 return nil 327 } 328 329 // If Write hasn't been called then ignore the Flush - Release 330 // will pick it up 331 if !fh.writeCalled { 332 fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush ignoring flush on unwritten handle") 333 return nil 334 } 335 336 fh.file.muRW.Lock() 337 defer fh.file.muRW.Unlock() 338 err := fh.flushWrites(false) 339 340 if err != nil { 341 fs.Errorf(fh.logPrefix(), "RWFileHandle.Flush error: %v", err) 342 } else { 343 // fs.Debugf(fh.logPrefix(), "RWFileHandle.Flush OK") 344 } 345 return err 346 } 347 348 // Release is called when we are finished with the file handle 349 // 350 // It isn't called directly from userspace so the error is ignored by 351 // the kernel 352 func (fh *RWFileHandle) Release() error { 353 fh.mu.Lock() 354 defer fh.mu.Unlock() 355 if fh.closed { 356 fs.Debugf(fh.logPrefix(), "RWFileHandle.Release nothing to do") 357 return nil 358 } 359 fs.Debugf(fh.logPrefix(), "RWFileHandle.Release closing") 360 err := fh.close() 361 if err != nil { 362 fs.Errorf(fh.logPrefix(), "RWFileHandle.Release error: %v", err) 363 } else { 364 // fs.Debugf(fh.logPrefix(), "RWFileHandle.Release OK") 365 } 366 return err 367 } 368 369 // Size returns the size of the underlying file 370 func (fh *RWFileHandle) Size() int64 { 371 fh.mu.Lock() 372 defer fh.mu.Unlock() 373 if !fh.opened { 374 return fh.file.Size() 375 } 376 fi, err := fh.File.Stat() 377 if err != nil { 378 return 0 379 } 380 return fi.Size() 381 } 382 383 // Stat returns info about the file 384 func (fh *RWFileHandle) Stat() (os.FileInfo, error) { 385 fh.mu.Lock() 386 defer fh.mu.Unlock() 387 return fh.file, nil 388 } 389 390 // readFn is a general purpose read function - pass in a closure to do 391 // the actual read 392 func (fh *RWFileHandle) readFn(read func() (int, error)) (n int, err error) { 393 fh.mu.Lock() 394 defer fh.mu.Unlock() 395 if fh.closed { 396 return 0, ECLOSED 397 } 398 if fh.flags&accessModeMask == os.O_WRONLY { 399 return 0, EBADF 400 } 401 if err = fh.openPending(false); err != nil { 402 return n, err 403 } 404 return read() 405 } 406 407 // Read bytes from the file 408 func (fh *RWFileHandle) Read(b []byte) (n int, err error) { 409 return fh.readFn(func() (int, error) { 410 return fh.File.Read(b) 411 }) 412 } 413 414 // ReadAt bytes from the file at off 415 func (fh *RWFileHandle) ReadAt(b []byte, off int64) (n int, err error) { 416 return fh.readFn(func() (int, error) { 417 return fh.File.ReadAt(b, off) 418 }) 419 } 420 421 // Seek to new file position 422 func (fh *RWFileHandle) Seek(offset int64, whence int) (ret int64, err error) { 423 fh.mu.Lock() 424 defer fh.mu.Unlock() 425 if fh.closed { 426 return 0, ECLOSED 427 } 428 if !fh.opened && offset == 0 && whence != 2 { 429 return 0, nil 430 } 431 if err = fh.openPending(false); err != nil { 432 return ret, err 433 } 434 return fh.File.Seek(offset, whence) 435 } 436 437 // writeFn general purpose write call 438 // 439 // Pass a closure to do the actual write 440 func (fh *RWFileHandle) writeFn(write func() error) (err error) { 441 fh.mu.Lock() 442 defer fh.mu.Unlock() 443 if fh.closed { 444 return ECLOSED 445 } 446 if fh.flags&accessModeMask == os.O_RDONLY { 447 return EBADF 448 } 449 if err = fh.openPending(false); err != nil { 450 return err 451 } 452 fh.writeCalled = true 453 err = write() 454 if err != nil { 455 return err 456 } 457 fi, err := fh.File.Stat() 458 if err != nil { 459 return errors.Wrap(err, "failed to stat cache file") 460 } 461 fh.file.setSize(fi.Size()) 462 return nil 463 } 464 465 // Write bytes to the file 466 func (fh *RWFileHandle) Write(b []byte) (n int, err error) { 467 err = fh.writeFn(func() error { 468 n, err = fh.File.Write(b) 469 return err 470 }) 471 return n, err 472 } 473 474 // WriteAt bytes to the file at off 475 func (fh *RWFileHandle) WriteAt(b []byte, off int64) (n int, err error) { 476 if fh.flags&os.O_APPEND != 0 { 477 // if append is set, call Write as WriteAt returns an error if append is set 478 return fh.Write(b) 479 } 480 err = fh.writeFn(func() error { 481 n, err = fh.File.WriteAt(b, off) 482 return err 483 }) 484 return n, err 485 } 486 487 // WriteString a string to the file 488 func (fh *RWFileHandle) WriteString(s string) (n int, err error) { 489 err = fh.writeFn(func() error { 490 n, err = fh.File.WriteString(s) 491 return err 492 }) 493 return n, err 494 } 495 496 // Truncate file to given size 497 func (fh *RWFileHandle) Truncate(size int64) (err error) { 498 fh.mu.Lock() 499 defer fh.mu.Unlock() 500 if fh.closed { 501 return ECLOSED 502 } 503 if err = fh.openPending(size == 0); err != nil { 504 return err 505 } 506 fh.changed = true 507 fh.file.setSize(size) 508 return fh.File.Truncate(size) 509 } 510 511 // Sync commits the current contents of the file to stable storage. Typically, 512 // this means flushing the file system's in-memory copy of recently written 513 // data to disk. 514 func (fh *RWFileHandle) Sync() error { 515 fh.mu.Lock() 516 defer fh.mu.Unlock() 517 if fh.closed { 518 return ECLOSED 519 } 520 if !fh.opened { 521 return nil 522 } 523 if fh.flags&accessModeMask == os.O_RDONLY { 524 return nil 525 } 526 return fh.File.Sync() 527 } 528 529 func (fh *RWFileHandle) logPrefix() string { 530 return fmt.Sprintf("%s(%p)", fh.file.Path(), fh) 531 }