github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/cmd/cmd.go (about) 1 // Package cmd implemnts the rclone command 2 // 3 // It is in a sub package so it's internals can be re-used elsewhere 4 package cmd 5 6 // FIXME only attach the remote flags when using a remote??? 7 // would probably mean bringing all the flags in to here? Or define some flagsets in fs... 8 9 import ( 10 "fmt" 11 "log" 12 "math/rand" 13 "os" 14 "os/exec" 15 "path" 16 "regexp" 17 "runtime" 18 "runtime/pprof" 19 "strconv" 20 "strings" 21 "sync" 22 "time" 23 24 "github.com/pkg/errors" 25 "github.com/rclone/rclone/fs" 26 "github.com/rclone/rclone/fs/accounting" 27 "github.com/rclone/rclone/fs/cache" 28 "github.com/rclone/rclone/fs/config/configflags" 29 "github.com/rclone/rclone/fs/config/flags" 30 "github.com/rclone/rclone/fs/filter" 31 "github.com/rclone/rclone/fs/filter/filterflags" 32 "github.com/rclone/rclone/fs/fserrors" 33 "github.com/rclone/rclone/fs/fspath" 34 fslog "github.com/rclone/rclone/fs/log" 35 "github.com/rclone/rclone/fs/rc/rcflags" 36 "github.com/rclone/rclone/fs/rc/rcserver" 37 "github.com/rclone/rclone/lib/atexit" 38 "github.com/spf13/cobra" 39 "github.com/spf13/pflag" 40 ) 41 42 // Globals 43 var ( 44 // Flags 45 cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file") 46 memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file") 47 statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)") 48 dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s") 49 version bool 50 retries = flags.IntP("retries", "", 3, "Retry operations this many times if they fail") 51 retriesInterval = flags.DurationP("retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)") 52 // Errors 53 errorCommandNotFound = errors.New("command not found") 54 errorUncategorized = errors.New("uncategorized error") 55 errorNotEnoughArguments = errors.New("not enough arguments") 56 errorTooManyArguments = errors.New("too many arguments") 57 ) 58 59 const ( 60 exitCodeSuccess = iota 61 exitCodeUsageError 62 exitCodeUncategorizedError 63 exitCodeDirNotFound 64 exitCodeFileNotFound 65 exitCodeRetryError 66 exitCodeNoRetryError 67 exitCodeFatalError 68 exitCodeTransferExceeded 69 exitCodeNoFilesTransferred 70 ) 71 72 // ShowVersion prints the version to stdout 73 func ShowVersion() { 74 fmt.Printf("rclone %s\n", fs.Version) 75 fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) 76 fmt.Printf("- go version: %s\n", runtime.Version()) 77 } 78 79 // NewFsFile creates an Fs from a name but may point to a file. 80 // 81 // It returns a string with the file name if points to a file 82 // otherwise "". 83 func NewFsFile(remote string) (fs.Fs, string) { 84 _, _, fsPath, err := fs.ParseRemote(remote) 85 if err != nil { 86 err = fs.CountError(err) 87 log.Fatalf("Failed to create file system for %q: %v", remote, err) 88 } 89 f, err := cache.Get(remote) 90 switch err { 91 case fs.ErrorIsFile: 92 return f, path.Base(fsPath) 93 case nil: 94 return f, "" 95 default: 96 err = fs.CountError(err) 97 log.Fatalf("Failed to create file system for %q: %v", remote, err) 98 } 99 return nil, "" 100 } 101 102 // newFsFileAddFilter creates an src Fs from a name 103 // 104 // This works the same as NewFsFile however it adds filters to the Fs 105 // to limit it to a single file if the remote pointed to a file. 106 func newFsFileAddFilter(remote string) (fs.Fs, string) { 107 f, fileName := NewFsFile(remote) 108 if fileName != "" { 109 if !filter.Active.InActive() { 110 err := errors.Errorf("Can't limit to single files when using filters: %v", remote) 111 err = fs.CountError(err) 112 log.Fatalf(err.Error()) 113 } 114 // Limit transfers to this file 115 err := filter.Active.AddFile(fileName) 116 if err != nil { 117 err = fs.CountError(err) 118 log.Fatalf("Failed to limit to single file %q: %v", remote, err) 119 } 120 } 121 return f, fileName 122 } 123 124 // NewFsSrc creates a new src fs from the arguments. 125 // 126 // The source can be a file or a directory - if a file then it will 127 // limit the Fs to a single file. 128 func NewFsSrc(args []string) fs.Fs { 129 fsrc, _ := newFsFileAddFilter(args[0]) 130 return fsrc 131 } 132 133 // newFsDir creates an Fs from a name 134 // 135 // This must point to a directory 136 func newFsDir(remote string) fs.Fs { 137 f, err := cache.Get(remote) 138 if err != nil { 139 err = fs.CountError(err) 140 log.Fatalf("Failed to create file system for %q: %v", remote, err) 141 } 142 return f 143 } 144 145 // NewFsDir creates a new Fs from the arguments 146 // 147 // The argument must point a directory 148 func NewFsDir(args []string) fs.Fs { 149 fdst := newFsDir(args[0]) 150 return fdst 151 } 152 153 // NewFsSrcDst creates a new src and dst fs from the arguments 154 func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) { 155 fsrc, _ := newFsFileAddFilter(args[0]) 156 fdst := newFsDir(args[1]) 157 return fsrc, fdst 158 } 159 160 // NewFsSrcFileDst creates a new src and dst fs from the arguments 161 // 162 // The source may be a file, in which case the source Fs and file name is returned 163 func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs) { 164 fsrc, srcFileName = NewFsFile(args[0]) 165 fdst = newFsDir(args[1]) 166 return fsrc, srcFileName, fdst 167 } 168 169 // NewFsSrcDstFiles creates a new src and dst fs from the arguments 170 // If src is a file then srcFileName and dstFileName will be non-empty 171 func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) { 172 fsrc, srcFileName = newFsFileAddFilter(args[0]) 173 // If copying a file... 174 dstRemote := args[1] 175 // If file exists then srcFileName != "", however if the file 176 // doesn't exist then we assume it is a directory... 177 if srcFileName != "" { 178 var err error 179 dstRemote, dstFileName, err = fspath.Split(dstRemote) 180 if err != nil { 181 log.Fatalf("Parsing %q failed: %v", args[1], err) 182 } 183 if dstRemote == "" { 184 dstRemote = "." 185 } 186 if dstFileName == "" { 187 log.Fatalf("%q is a directory", args[1]) 188 } 189 } 190 fdst, err := cache.Get(dstRemote) 191 switch err { 192 case fs.ErrorIsFile: 193 _ = fs.CountError(err) 194 log.Fatalf("Source doesn't exist or is a directory and destination is a file") 195 case nil: 196 default: 197 _ = fs.CountError(err) 198 log.Fatalf("Failed to create file system for destination %q: %v", dstRemote, err) 199 } 200 return 201 } 202 203 // NewFsDstFile creates a new dst fs with a destination file name from the arguments 204 func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) { 205 dstRemote, dstFileName, err := fspath.Split(args[0]) 206 if err != nil { 207 log.Fatalf("Parsing %q failed: %v", args[0], err) 208 } 209 if dstRemote == "" { 210 dstRemote = "." 211 } 212 if dstFileName == "" { 213 log.Fatalf("%q is a directory", args[0]) 214 } 215 fdst = newFsDir(dstRemote) 216 return 217 } 218 219 // ShowStats returns true if the user added a `--stats` flag to the command line. 220 // 221 // This is called by Run to override the default value of the 222 // showStats passed in. 223 func ShowStats() bool { 224 statsIntervalFlag := pflag.Lookup("stats") 225 return statsIntervalFlag != nil && statsIntervalFlag.Changed 226 } 227 228 // Run the function with stats and retries if required 229 func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) { 230 var cmdErr error 231 stopStats := func() {} 232 if !showStats && ShowStats() { 233 showStats = true 234 } 235 if fs.Config.Progress { 236 stopStats = startProgress() 237 } else if showStats { 238 stopStats = StartStats() 239 } 240 SigInfoHandler() 241 for try := 1; try <= *retries; try++ { 242 cmdErr = f() 243 cmdErr = fs.CountError(cmdErr) 244 lastErr := accounting.GlobalStats().GetLastError() 245 if cmdErr == nil { 246 cmdErr = lastErr 247 } 248 if !Retry || !accounting.GlobalStats().Errored() { 249 if try > 1 { 250 fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries) 251 } 252 break 253 } 254 if accounting.GlobalStats().HadFatalError() { 255 fs.Errorf(nil, "Fatal error received - not attempting retries") 256 break 257 } 258 if accounting.GlobalStats().Errored() && !accounting.GlobalStats().HadRetryError() { 259 fs.Errorf(nil, "Can't retry this error - not attempting retries") 260 break 261 } 262 if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() { 263 d := retryAfter.Sub(time.Now()) 264 if d > 0 { 265 fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d) 266 time.Sleep(d) 267 } 268 } 269 if lastErr != nil { 270 fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.GlobalStats().GetErrors(), lastErr) 271 } else { 272 fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, accounting.GlobalStats().GetErrors()) 273 } 274 if try < *retries { 275 accounting.GlobalStats().ResetErrors() 276 } 277 if *retriesInterval > 0 { 278 time.Sleep(*retriesInterval) 279 } 280 } 281 stopStats() 282 if showStats && (accounting.GlobalStats().Errored() || *statsInterval > 0) { 283 accounting.GlobalStats().Log() 284 } 285 fs.Debugf(nil, "%d go routines active\n", runtime.NumGoroutine()) 286 287 // dump all running go-routines 288 if fs.Config.Dump&fs.DumpGoRoutines != 0 { 289 err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) 290 if err != nil { 291 fs.Errorf(nil, "Failed to dump goroutines: %v", err) 292 } 293 } 294 295 // dump open files 296 if fs.Config.Dump&fs.DumpOpenFiles != 0 { 297 c := exec.Command("lsof", "-p", strconv.Itoa(os.Getpid())) 298 c.Stdout = os.Stdout 299 c.Stderr = os.Stderr 300 err := c.Run() 301 if err != nil { 302 fs.Errorf(nil, "Failed to list open files: %v", err) 303 } 304 } 305 306 // Log the final error message and exit 307 if cmdErr != nil { 308 nerrs := accounting.GlobalStats().GetErrors() 309 if nerrs <= 1 { 310 log.Printf("Failed to %s: %v", cmd.Name(), cmdErr) 311 } else { 312 log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, cmdErr) 313 } 314 } 315 resolveExitCode(cmdErr) 316 317 } 318 319 // CheckArgs checks there are enough arguments and prints a message if not 320 func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) { 321 if len(args) < MinArgs { 322 _ = cmd.Usage() 323 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum: you provided %d non flag arguments: %q\n", cmd.Name(), MinArgs, len(args), args) 324 resolveExitCode(errorNotEnoughArguments) 325 } else if len(args) > MaxArgs { 326 _ = cmd.Usage() 327 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum: you provided %d non flag arguments: %q\n", cmd.Name(), MaxArgs, len(args), args) 328 resolveExitCode(errorTooManyArguments) 329 } 330 } 331 332 // StartStats prints the stats every statsInterval 333 // 334 // It returns a func which should be called to stop the stats. 335 func StartStats() func() { 336 if *statsInterval <= 0 { 337 return func() {} 338 } 339 stopStats := make(chan struct{}) 340 var wg sync.WaitGroup 341 wg.Add(1) 342 go func() { 343 defer wg.Done() 344 ticker := time.NewTicker(*statsInterval) 345 for { 346 select { 347 case <-ticker.C: 348 accounting.GlobalStats().Log() 349 case <-stopStats: 350 ticker.Stop() 351 return 352 } 353 } 354 }() 355 return func() { 356 close(stopStats) 357 wg.Wait() 358 } 359 } 360 361 // initConfig is run by cobra after initialising the flags 362 func initConfig() { 363 // Start the logger 364 fslog.InitLogging() 365 366 // Finish parsing any command line flags 367 configflags.SetFlags() 368 369 // Load filters 370 err := filterflags.Reload() 371 if err != nil { 372 log.Fatalf("Failed to load filters: %v", err) 373 } 374 375 // Write the args for debug purposes 376 fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args) 377 378 // Start the remote control server if configured 379 _, err = rcserver.Start(&rcflags.Opt) 380 if err != nil { 381 log.Fatalf("Failed to start remote control: %v", err) 382 } 383 384 // Setup CPU profiling if desired 385 if *cpuProfile != "" { 386 fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile) 387 f, err := os.Create(*cpuProfile) 388 if err != nil { 389 err = fs.CountError(err) 390 log.Fatal(err) 391 } 392 err = pprof.StartCPUProfile(f) 393 if err != nil { 394 err = fs.CountError(err) 395 log.Fatal(err) 396 } 397 atexit.Register(func() { 398 pprof.StopCPUProfile() 399 }) 400 } 401 402 // Setup memory profiling if desired 403 if *memProfile != "" { 404 atexit.Register(func() { 405 fs.Infof(nil, "Saving Memory profile %q\n", *memProfile) 406 f, err := os.Create(*memProfile) 407 if err != nil { 408 err = fs.CountError(err) 409 log.Fatal(err) 410 } 411 err = pprof.WriteHeapProfile(f) 412 if err != nil { 413 err = fs.CountError(err) 414 log.Fatal(err) 415 } 416 err = f.Close() 417 if err != nil { 418 err = fs.CountError(err) 419 log.Fatal(err) 420 } 421 }) 422 } 423 424 if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); m == false { 425 fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.") 426 fs.Config.DataRateUnit = "bytes" 427 } else { 428 fs.Config.DataRateUnit = *dataRateUnit 429 } 430 } 431 432 func resolveExitCode(err error) { 433 atexit.Run() 434 if err == nil { 435 if fs.Config.ErrorOnNoTransfer { 436 if accounting.GlobalStats().GetTransfers() == 0 { 437 os.Exit(exitCodeNoFilesTransferred) 438 } 439 } 440 os.Exit(exitCodeSuccess) 441 } 442 443 _, unwrapped := fserrors.Cause(err) 444 445 switch { 446 case unwrapped == fs.ErrorDirNotFound: 447 os.Exit(exitCodeDirNotFound) 448 case unwrapped == fs.ErrorObjectNotFound: 449 os.Exit(exitCodeFileNotFound) 450 case unwrapped == errorUncategorized: 451 os.Exit(exitCodeUncategorizedError) 452 case unwrapped == accounting.ErrorMaxTransferLimitReached: 453 os.Exit(exitCodeTransferExceeded) 454 case fserrors.ShouldRetry(err): 455 os.Exit(exitCodeRetryError) 456 case fserrors.IsNoRetryError(err): 457 os.Exit(exitCodeNoRetryError) 458 case fserrors.IsFatalError(err): 459 os.Exit(exitCodeFatalError) 460 default: 461 os.Exit(exitCodeUsageError) 462 } 463 } 464 465 var backendFlags map[string]struct{} 466 467 // AddBackendFlags creates flags for all the backend options 468 func AddBackendFlags() { 469 backendFlags = map[string]struct{}{} 470 for _, fsInfo := range fs.Registry { 471 done := map[string]struct{}{} 472 for i := range fsInfo.Options { 473 opt := &fsInfo.Options[i] 474 // Skip if done already (eg with Provider options) 475 if _, doneAlready := done[opt.Name]; doneAlready { 476 continue 477 } 478 done[opt.Name] = struct{}{} 479 // Make a flag from each option 480 name := opt.FlagName(fsInfo.Prefix) 481 found := pflag.CommandLine.Lookup(name) != nil 482 if !found { 483 // Take first line of help only 484 help := strings.TrimSpace(opt.Help) 485 if nl := strings.IndexRune(help, '\n'); nl >= 0 { 486 help = help[:nl] 487 } 488 help = strings.TrimSpace(help) 489 if opt.IsPassword { 490 help += " (obscured)" 491 } 492 flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help) 493 if _, isBool := opt.Default.(bool); isBool { 494 flag.NoOptDefVal = "true" 495 } 496 // Hide on the command line if requested 497 if opt.Hide&fs.OptionHideCommandLine != 0 { 498 flag.Hidden = true 499 } 500 backendFlags[name] = struct{}{} 501 } else { 502 fs.Errorf(nil, "Not adding duplicate flag --%s", name) 503 } 504 //flag.Hidden = true 505 } 506 } 507 } 508 509 // Main runs rclone interpreting flags and commands out of os.Args 510 func Main() { 511 rand.Seed(time.Now().Unix()) 512 setupRootCommand(Root) 513 AddBackendFlags() 514 if err := Root.Execute(); err != nil { 515 log.Fatalf("Fatal error: %v", err) 516 } 517 }