github.com/artpar/rclone@v1.67.3/cmd/cmd.go (about) 1 // Package cmd implements the rclone command 2 // 3 // It is in a sub package so it's internals can be reused elsewhere 4 package cmd 5 6 // FIXME only attach the remote flags when using a remote??? 7 // would probably mean bringing all the flags in to here? Or define some flagsets in fs... 8 9 import ( 10 "context" 11 "errors" 12 "fmt" 13 "log" 14 "os" 15 "os/exec" 16 "path" 17 "regexp" 18 "runtime" 19 "runtime/pprof" 20 "strconv" 21 "strings" 22 "sync" 23 "time" 24 25 "github.com/artpar/rclone/fs" 26 "github.com/artpar/rclone/fs/accounting" 27 "github.com/artpar/rclone/fs/cache" 28 "github.com/artpar/rclone/fs/config/configfile" 29 "github.com/artpar/rclone/fs/config/configflags" 30 "github.com/artpar/rclone/fs/config/flags" 31 "github.com/artpar/rclone/fs/filter" 32 "github.com/artpar/rclone/fs/filter/filterflags" 33 "github.com/artpar/rclone/fs/fspath" 34 fslog "github.com/artpar/rclone/fs/log" 35 "github.com/artpar/rclone/fs/rc/rcflags" 36 "github.com/artpar/rclone/fs/rc/rcserver" 37 "github.com/artpar/rclone/lib/atexit" 38 "github.com/artpar/rclone/lib/buildinfo" 39 "github.com/artpar/rclone/lib/terminal" 40 "github.com/spf13/cobra" 41 "github.com/spf13/pflag" 42 ) 43 44 // Globals 45 var ( 46 // Flags 47 cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file", "Debugging") 48 memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file", "Debugging") 49 statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g. 500ms, 60s, 5m (0 to disable)", "Logging") 50 dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes' per second", "Logging") 51 version bool 52 // Errors 53 errorCommandNotFound = errors.New("command not found") 54 errorUncategorized = errors.New("uncategorized error") 55 errorNotEnoughArguments = errors.New("not enough arguments") 56 errorTooManyArguments = errors.New("too many arguments") 57 ) 58 59 // ShowVersion prints the version to stdout 60 func ShowVersion() { 61 osVersion, osKernel := buildinfo.GetOSVersion() 62 if osVersion == "" { 63 osVersion = "unknown" 64 } 65 if osKernel == "" { 66 osKernel = "unknown" 67 } 68 69 linking, tagString := buildinfo.GetLinkingAndTags() 70 71 arch := buildinfo.GetArch() 72 73 fmt.Printf("rclone %s\n", fs.Version) 74 fmt.Printf("- os/version: %s\n", osVersion) 75 fmt.Printf("- os/kernel: %s\n", osKernel) 76 fmt.Printf("- os/type: %s\n", runtime.GOOS) 77 fmt.Printf("- os/arch: %s\n", arch) 78 fmt.Printf("- go/version: %s\n", runtime.Version()) 79 fmt.Printf("- go/linking: %s\n", linking) 80 fmt.Printf("- go/tags: %s\n", tagString) 81 } 82 83 // NewFsFile creates an Fs from a name but may point to a file. 84 // 85 // It returns a string with the file name if points to a file 86 // otherwise "". 87 func NewFsFile(remote string) (fs.Fs, string) { 88 _, fsPath, err := fspath.SplitFs(remote) 89 if err != nil { 90 err = fs.CountError(err) 91 log.Printf("Failed to create file system for %q: %v", remote, err) 92 } 93 f, err := cache.Get(context.Background(), remote) 94 switch err { 95 case fs.ErrorIsFile: 96 cache.Pin(f) // pin indefinitely since it was on the CLI 97 return f, path.Base(fsPath) 98 case nil: 99 cache.Pin(f) // pin indefinitely since it was on the CLI 100 return f, "" 101 default: 102 err = fs.CountError(err) 103 log.Printf("Failed to create file system for %q: %v", remote, err) 104 } 105 return nil, "" 106 } 107 108 // newFsFileAddFilter creates an src Fs from a name 109 // 110 // This works the same as NewFsFile however it adds filters to the Fs 111 // to limit it to a single file if the remote pointed to a file. 112 func newFsFileAddFilter(remote string) (fs.Fs, string) { 113 fi := filter.GetConfig(context.Background()) 114 f, fileName := NewFsFile(remote) 115 if fileName != "" { 116 if !fi.InActive() { 117 err := fmt.Errorf("can't limit to single files when using filters: %v", remote) 118 err = fs.CountError(err) 119 log.Printf(err.Error()) 120 } 121 // Limit transfers to this file 122 err := fi.AddFile(fileName) 123 if err != nil { 124 err = fs.CountError(err) 125 log.Printf("Failed to limit to single file %q: %v", remote, err) 126 } 127 } 128 return f, fileName 129 } 130 131 // NewFsSrc creates a new src fs from the arguments. 132 // 133 // The source can be a file or a directory - if a file then it will 134 // limit the Fs to a single file. 135 func NewFsSrc(args []string) fs.Fs { 136 fsrc, _ := newFsFileAddFilter(args[0]) 137 return fsrc 138 } 139 140 // newFsDir creates an Fs from a name 141 // 142 // This must point to a directory 143 func newFsDir(remote string) fs.Fs { 144 f, err := cache.Get(context.Background(), remote) 145 if err != nil { 146 err = fs.CountError(err) 147 log.Printf("Failed to create file system for %q: %v", remote, err) 148 } 149 cache.Pin(f) // pin indefinitely since it was on the CLI 150 return f 151 } 152 153 // NewFsDir creates a new Fs from the arguments 154 // 155 // The argument must point a directory 156 func NewFsDir(args []string) fs.Fs { 157 fdst := newFsDir(args[0]) 158 return fdst 159 } 160 161 // NewFsSrcDst creates a new src and dst fs from the arguments 162 func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) { 163 fsrc, _ := newFsFileAddFilter(args[0]) 164 fdst := newFsDir(args[1]) 165 return fsrc, fdst 166 } 167 168 // NewFsSrcFileDst creates a new src and dst fs from the arguments 169 // 170 // The source may be a file, in which case the source Fs and file name is returned 171 func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs) { 172 fsrc, srcFileName = NewFsFile(args[0]) 173 fdst = newFsDir(args[1]) 174 return fsrc, srcFileName, fdst 175 } 176 177 // NewFsSrcDstFiles creates a new src and dst fs from the arguments 178 // If src is a file then srcFileName and dstFileName will be non-empty 179 func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) { 180 fsrc, srcFileName = newFsFileAddFilter(args[0]) 181 // If copying a file... 182 dstRemote := args[1] 183 // If file exists then srcFileName != "", however if the file 184 // doesn't exist then we assume it is a directory... 185 if srcFileName != "" { 186 var err error 187 dstRemote, dstFileName, err = fspath.Split(dstRemote) 188 if err != nil { 189 log.Printf("Parsing %q failed: %v", args[1], err) 190 } 191 if dstRemote == "" { 192 dstRemote = "." 193 } 194 if dstFileName == "" { 195 log.Printf("%q is a directory", args[1]) 196 } 197 } 198 fdst, err := cache.Get(context.Background(), dstRemote) 199 switch err { 200 case fs.ErrorIsFile: 201 _ = fs.CountError(err) 202 log.Printf("Source doesn't exist or is a directory and destination is a file") 203 case nil: 204 default: 205 _ = fs.CountError(err) 206 log.Printf("Failed to create file system for destination %q: %v", dstRemote, err) 207 } 208 cache.Pin(fdst) // pin indefinitely since it was on the CLI 209 return 210 } 211 212 // NewFsDstFile creates a new dst fs with a destination file name from the arguments 213 func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) { 214 dstRemote, dstFileName, err := fspath.Split(args[0]) 215 if err != nil { 216 log.Printf("Parsing %q failed: %v", args[0], err) 217 } 218 if dstRemote == "" { 219 dstRemote = "." 220 } 221 if dstFileName == "" { 222 log.Printf("%q is a directory", args[0]) 223 } 224 fdst = newFsDir(dstRemote) 225 return 226 } 227 228 // ShowStats returns true if the user added a `--stats` flag to the command line. 229 // 230 // This is called by Run to override the default value of the 231 // showStats passed in. 232 func ShowStats() bool { 233 statsIntervalFlag := pflag.Lookup("stats") 234 return statsIntervalFlag != nil && statsIntervalFlag.Changed 235 } 236 237 // Run the function with stats and retries if required 238 func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) { 239 ci := fs.GetConfig(context.Background()) 240 var cmdErr error 241 stopStats := func() {} 242 if !showStats && ShowStats() { 243 showStats = true 244 } 245 if ci.Progress { 246 stopStats = startProgress() 247 } else if showStats { 248 stopStats = StartStats() 249 } 250 SigInfoHandler() 251 for try := 1; try <= ci.Retries; try++ { 252 cmdErr = f() 253 cmdErr = fs.CountError(cmdErr) 254 lastErr := accounting.GlobalStats().GetLastError() 255 if cmdErr == nil { 256 cmdErr = lastErr 257 } 258 if !Retry || !accounting.GlobalStats().Errored() { 259 if try > 1 { 260 fs.Errorf(nil, "Attempt %d/%d succeeded", try, ci.Retries) 261 } 262 break 263 } 264 if accounting.GlobalStats().HadFatalError() { 265 fs.Errorf(nil, "Fatal error received - not attempting retries") 266 break 267 } 268 if accounting.GlobalStats().Errored() && !accounting.GlobalStats().HadRetryError() { 269 fs.Errorf(nil, "Can't retry any of the errors - not attempting retries") 270 break 271 } 272 if retryAfter := accounting.GlobalStats().RetryAfter(); !retryAfter.IsZero() { 273 d := time.Until(retryAfter) 274 if d > 0 { 275 fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d) 276 time.Sleep(d) 277 } 278 } 279 if lastErr != nil { 280 fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, ci.Retries, accounting.GlobalStats().GetErrors(), lastErr) 281 } else { 282 fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, ci.Retries, accounting.GlobalStats().GetErrors()) 283 } 284 if try < ci.Retries { 285 accounting.GlobalStats().ResetErrors() 286 } 287 if ci.RetriesInterval > 0 { 288 time.Sleep(ci.RetriesInterval) 289 } 290 } 291 stopStats() 292 if showStats && (accounting.GlobalStats().Errored() || *statsInterval > 0) { 293 accounting.GlobalStats().Log() 294 } 295 fs.Debugf(nil, "%d go routines active\n", runtime.NumGoroutine()) 296 297 if ci.Progress && ci.ProgressTerminalTitle { 298 // Clear terminal title 299 terminal.WriteTerminalTitle("") 300 } 301 302 // dump all running go-routines 303 if ci.Dump&fs.DumpGoRoutines != 0 { 304 err := pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) 305 if err != nil { 306 fs.Errorf(nil, "Failed to dump goroutines: %v", err) 307 } 308 } 309 310 // dump open files 311 if ci.Dump&fs.DumpOpenFiles != 0 { 312 c := exec.Command("lsof", "-p", strconv.Itoa(os.Getpid())) 313 c.Stdout = os.Stdout 314 c.Stderr = os.Stderr 315 err := c.Run() 316 if err != nil { 317 fs.Errorf(nil, "Failed to list open files: %v", err) 318 } 319 } 320 321 // clear cache and shutdown backends 322 cache.Clear() 323 if lastErr := accounting.GlobalStats().GetLastError(); cmdErr == nil { 324 cmdErr = lastErr 325 } 326 327 // Log the final error message and exit 328 if cmdErr != nil { 329 nerrs := accounting.GlobalStats().GetErrors() 330 if nerrs <= 1 { 331 log.Printf("Failed to %s: %v", cmd.Name(), cmdErr) 332 } else { 333 log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, cmdErr) 334 } 335 } 336 resolveExitCode(cmdErr) 337 } 338 339 // CheckArgs checks there are enough arguments and prints a message if not 340 func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) { 341 if len(args) < MinArgs { 342 _ = cmd.Usage() 343 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum: you provided %d non flag arguments: %q\n", cmd.Name(), MinArgs, len(args), args) 344 resolveExitCode(errorNotEnoughArguments) 345 } else if len(args) > MaxArgs { 346 _ = cmd.Usage() 347 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum: you provided %d non flag arguments: %q\n", cmd.Name(), MaxArgs, len(args), args) 348 resolveExitCode(errorTooManyArguments) 349 } 350 } 351 352 // StartStats prints the stats every statsInterval 353 // 354 // It returns a func which should be called to stop the stats. 355 func StartStats() func() { 356 if *statsInterval <= 0 { 357 return func() {} 358 } 359 stopStats := make(chan struct{}) 360 var wg sync.WaitGroup 361 wg.Add(1) 362 go func() { 363 defer wg.Done() 364 ticker := time.NewTicker(*statsInterval) 365 for { 366 select { 367 case <-ticker.C: 368 accounting.GlobalStats().Log() 369 case <-stopStats: 370 ticker.Stop() 371 return 372 } 373 } 374 }() 375 return func() { 376 close(stopStats) 377 wg.Wait() 378 } 379 } 380 381 // initConfig is run by cobra after initialising the flags 382 func initConfig() { 383 ctx := context.Background() 384 ci := fs.GetConfig(ctx) 385 386 // Start the logger 387 fslog.InitLogging() 388 389 // Finish parsing any command line flags 390 configflags.SetFlags(ci) 391 392 // Load the config 393 configfile.Install() 394 395 // Start accounting 396 accounting.Start(ctx) 397 398 // Configure console 399 if ci.NoConsole { 400 // Hide the console window 401 terminal.HideConsole() 402 } else { 403 // Enable color support on stdout if possible. 404 // This enables virtual terminal processing on Windows 10, 405 // adding native support for ANSI/VT100 escape sequences. 406 terminal.EnableColorsStdout() 407 } 408 409 // Load filters 410 err := filterflags.Reload(ctx) 411 if err != nil { 412 log.Printf("Failed to load filters: %v", err) 413 } 414 415 // Write the args for debug purposes 416 fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args) 417 418 // Inform user about systemd log support now that we have a logger 419 if fslog.Opt.LogSystemdSupport { 420 fs.Debugf("rclone", "systemd logging support activated") 421 } 422 423 // Start the remote control server if configured 424 _, err = rcserver.Start(context.Background(), &rcflags.Opt) 425 if err != nil { 426 log.Printf("Failed to start remote control: %v", err) 427 } 428 429 // Setup CPU profiling if desired 430 if *cpuProfile != "" { 431 fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile) 432 f, err := os.Create(*cpuProfile) 433 if err != nil { 434 err = fs.CountError(err) 435 log.Println(err) 436 } 437 err = pprof.StartCPUProfile(f) 438 if err != nil { 439 err = fs.CountError(err) 440 log.Println(err) 441 } 442 atexit.Register(func() { 443 pprof.StopCPUProfile() 444 err := f.Close() 445 if err != nil { 446 err = fs.CountError(err) 447 log.Fatal(err) 448 } 449 }) 450 } 451 452 // Setup memory profiling if desired 453 if *memProfile != "" { 454 atexit.Register(func() { 455 fs.Infof(nil, "Saving Memory profile %q\n", *memProfile) 456 f, err := os.Create(*memProfile) 457 if err != nil { 458 err = fs.CountError(err) 459 log.Println(err) 460 } 461 err = pprof.WriteHeapProfile(f) 462 if err != nil { 463 err = fs.CountError(err) 464 log.Println(err) 465 } 466 err = f.Close() 467 if err != nil { 468 err = fs.CountError(err) 469 log.Println(err) 470 } 471 }) 472 } 473 474 if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); !m { 475 fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.") 476 ci.DataRateUnit = "bytes" 477 } else { 478 ci.DataRateUnit = *dataRateUnit 479 } 480 } 481 482 func resolveExitCode(err error) { 483 //ci := fs.GetConfig(context.Background()) 484 atexit.Run() 485 //if err == nil { 486 // if ci.ErrorOnNoTransfer { 487 // if accounting.GlobalStats().GetTransfers() == 0 { 488 // os.Exit(exitcode.NoFilesTransferred) 489 // } 490 // } 491 // os.Exit(exitcode.Success) 492 //} 493 // 494 //switch { 495 //case errors.Is(err, fs.ErrorDirNotFound): 496 // os.Exit(exitcode.DirNotFound) 497 //case errors.Is(err, fs.ErrorObjectNotFound): 498 // os.Exit(exitcode.FileNotFound) 499 //case errors.Is(err, errorUncategorized): 500 // os.Exit(exitcode.UncategorizedError) 501 //case errors.Is(err, accounting.ErrorMaxTransferLimitReached): 502 // os.Exit(exitcode.TransferExceeded) 503 //case errors.Is(err, fssync.ErrorMaxDurationReached): 504 // os.Exit(exitcode.DurationExceeded) 505 //case fserrors.ShouldRetry(err): 506 // os.Exit(exitcode.RetryError) 507 //case fserrors.IsNoRetryError(err), fserrors.IsNoLowLevelRetryError(err): 508 // os.Exit(exitcode.NoRetryError) 509 //case fserrors.IsFatalError(err): 510 // os.Exit(exitcode.FatalError) 511 //default: 512 // os.Exit(exitcode.UsageError) 513 //} 514 } 515 516 var backendFlags map[string]struct{} 517 518 // AddBackendFlags creates flags for all the backend options 519 func AddBackendFlags() { 520 backendFlags = map[string]struct{}{} 521 for _, fsInfo := range fs.Registry { 522 done := map[string]struct{}{} 523 for i := range fsInfo.Options { 524 opt := &fsInfo.Options[i] 525 // Skip if done already (e.g. with Provider options) 526 if _, doneAlready := done[opt.Name]; doneAlready { 527 continue 528 } 529 done[opt.Name] = struct{}{} 530 // Make a flag from each option 531 name := opt.FlagName(fsInfo.Prefix) 532 found := pflag.CommandLine.Lookup(name) != nil 533 if !found { 534 // Take first line of help only 535 help := strings.TrimSpace(opt.Help) 536 if nl := strings.IndexRune(help, '\n'); nl >= 0 { 537 help = help[:nl] 538 } 539 help = strings.TrimRight(strings.TrimSpace(help), ".!?") 540 if opt.IsPassword { 541 help += " (obscured)" 542 } 543 flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help) 544 flags.SetDefaultFromEnv(pflag.CommandLine, name) 545 if _, isBool := opt.Default.(bool); isBool { 546 flag.NoOptDefVal = "true" 547 } 548 // Hide on the command line if requested 549 if opt.Hide&fs.OptionHideCommandLine != 0 { 550 flag.Hidden = true 551 } 552 backendFlags[name] = struct{}{} 553 } else { 554 fs.Errorf(nil, "Not adding duplicate flag --%s", name) 555 } 556 // flag.Hidden = true 557 } 558 } 559 } 560 561 // Main runs rclone interpreting flags and commands out of os.Args 562 func Main() { 563 setupRootCommand(Root) 564 AddBackendFlags() 565 if err := Root.Execute(); err != nil { 566 if strings.HasPrefix(err.Error(), "unknown command") && selfupdateEnabled { 567 Root.PrintErrf("You could use '%s selfupdate' to get latest features.\n\n", Root.CommandPath()) 568 } 569 log.Printf("Fatal error: %v", err) 570 } 571 }