github.com/ncw/rclone@v1.48.1-0.20190724201158-a35aa1360e3e/cmd/cmd.go (about) 1 // Package cmd implemnts the rclone command 2 // 3 // It is in a sub package so it's internals can be re-used elsewhere 4 package cmd 5 6 // FIXME only attach the remote flags when using a remote??? 7 // would probably mean bringing all the flags in to here? Or define some flagsets in fs... 8 9 import ( 10 "fmt" 11 "log" 12 "os" 13 "os/exec" 14 "path" 15 "regexp" 16 "runtime" 17 "runtime/pprof" 18 "strconv" 19 "strings" 20 "sync" 21 "time" 22 23 "github.com/ncw/rclone/fs" 24 "github.com/ncw/rclone/fs/accounting" 25 "github.com/ncw/rclone/fs/cache" 26 "github.com/ncw/rclone/fs/config/configflags" 27 "github.com/ncw/rclone/fs/config/flags" 28 "github.com/ncw/rclone/fs/filter" 29 "github.com/ncw/rclone/fs/filter/filterflags" 30 "github.com/ncw/rclone/fs/fserrors" 31 "github.com/ncw/rclone/fs/fspath" 32 fslog "github.com/ncw/rclone/fs/log" 33 "github.com/ncw/rclone/fs/rc/rcflags" 34 "github.com/ncw/rclone/fs/rc/rcserver" 35 "github.com/ncw/rclone/lib/atexit" 36 "github.com/pkg/errors" 37 "github.com/spf13/cobra" 38 "github.com/spf13/pflag" 39 ) 40 41 // Globals 42 var ( 43 // Flags 44 cpuProfile = flags.StringP("cpuprofile", "", "", "Write cpu profile to file") 45 memProfile = flags.StringP("memprofile", "", "", "Write memory profile to file") 46 statsInterval = flags.DurationP("stats", "", time.Minute*1, "Interval between printing stats, e.g 500ms, 60s, 5m. (0 to disable)") 47 dataRateUnit = flags.StringP("stats-unit", "", "bytes", "Show data rate in stats as either 'bits' or 'bytes'/s") 48 version bool 49 retries = flags.IntP("retries", "", 3, "Retry operations this many times if they fail") 50 retriesInterval = flags.DurationP("retries-sleep", "", 0, "Interval between retrying operations if they fail, e.g 500ms, 60s, 5m. (0 to disable)") 51 // Errors 52 errorCommandNotFound = errors.New("command not found") 53 errorUncategorized = errors.New("uncategorized error") 54 errorNotEnoughArguments = errors.New("not enough arguments") 55 errorTooManyArguments = errors.New("too many arguments") 56 ) 57 58 const ( 59 exitCodeSuccess = iota 60 exitCodeUsageError 61 exitCodeUncategorizedError 62 exitCodeDirNotFound 63 exitCodeFileNotFound 64 exitCodeRetryError 65 exitCodeNoRetryError 66 exitCodeFatalError 67 exitCodeTransferExceeded 68 ) 69 70 // ShowVersion prints the version to stdout 71 func ShowVersion() { 72 fmt.Printf("rclone %s\n", fs.Version) 73 fmt.Printf("- os/arch: %s/%s\n", runtime.GOOS, runtime.GOARCH) 74 fmt.Printf("- go version: %s\n", runtime.Version()) 75 } 76 77 // NewFsFile creates a Fs from a name but may point to a file. 78 // 79 // It returns a string with the file name if points to a file 80 // otherwise "". 81 func NewFsFile(remote string) (fs.Fs, string) { 82 _, _, fsPath, err := fs.ParseRemote(remote) 83 if err != nil { 84 fs.CountError(err) 85 log.Fatalf("Failed to create file system for %q: %v", remote, err) 86 } 87 f, err := cache.Get(remote) 88 switch err { 89 case fs.ErrorIsFile: 90 return f, path.Base(fsPath) 91 case nil: 92 return f, "" 93 default: 94 fs.CountError(err) 95 log.Fatalf("Failed to create file system for %q: %v", remote, err) 96 } 97 return nil, "" 98 } 99 100 // newFsFileAddFilter creates a src Fs from a name 101 // 102 // This works the same as NewFsFile however it adds filters to the Fs 103 // to limit it to a single file if the remote pointed to a file. 104 func newFsFileAddFilter(remote string) (fs.Fs, string) { 105 f, fileName := NewFsFile(remote) 106 if fileName != "" { 107 if !filter.Active.InActive() { 108 err := errors.Errorf("Can't limit to single files when using filters: %v", remote) 109 fs.CountError(err) 110 log.Fatalf(err.Error()) 111 } 112 // Limit transfers to this file 113 err := filter.Active.AddFile(fileName) 114 if err != nil { 115 fs.CountError(err) 116 log.Fatalf("Failed to limit to single file %q: %v", remote, err) 117 } 118 } 119 return f, fileName 120 } 121 122 // NewFsSrc creates a new src fs from the arguments. 123 // 124 // The source can be a file or a directory - if a file then it will 125 // limit the Fs to a single file. 126 func NewFsSrc(args []string) fs.Fs { 127 fsrc, _ := newFsFileAddFilter(args[0]) 128 return fsrc 129 } 130 131 // newFsDir creates an Fs from a name 132 // 133 // This must point to a directory 134 func newFsDir(remote string) fs.Fs { 135 f, err := cache.Get(remote) 136 if err != nil { 137 fs.CountError(err) 138 log.Fatalf("Failed to create file system for %q: %v", remote, err) 139 } 140 return f 141 } 142 143 // NewFsDir creates a new Fs from the arguments 144 // 145 // The argument must point a directory 146 func NewFsDir(args []string) fs.Fs { 147 fdst := newFsDir(args[0]) 148 return fdst 149 } 150 151 // NewFsSrcDst creates a new src and dst fs from the arguments 152 func NewFsSrcDst(args []string) (fs.Fs, fs.Fs) { 153 fsrc, _ := newFsFileAddFilter(args[0]) 154 fdst := newFsDir(args[1]) 155 return fsrc, fdst 156 } 157 158 // NewFsSrcFileDst creates a new src and dst fs from the arguments 159 // 160 // The source may be a file, in which case the source Fs and file name is returned 161 func NewFsSrcFileDst(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs) { 162 fsrc, srcFileName = NewFsFile(args[0]) 163 fdst = newFsDir(args[1]) 164 return fsrc, srcFileName, fdst 165 } 166 167 // NewFsSrcDstFiles creates a new src and dst fs from the arguments 168 // If src is a file then srcFileName and dstFileName will be non-empty 169 func NewFsSrcDstFiles(args []string) (fsrc fs.Fs, srcFileName string, fdst fs.Fs, dstFileName string) { 170 fsrc, srcFileName = newFsFileAddFilter(args[0]) 171 // If copying a file... 172 dstRemote := args[1] 173 // If file exists then srcFileName != "", however if the file 174 // doesn't exist then we assume it is a directory... 175 if srcFileName != "" { 176 dstRemote, dstFileName = fspath.Split(dstRemote) 177 if dstRemote == "" { 178 dstRemote = "." 179 } 180 if dstFileName == "" { 181 log.Fatalf("%q is a directory", args[1]) 182 } 183 } 184 fdst, err := cache.Get(dstRemote) 185 switch err { 186 case fs.ErrorIsFile: 187 fs.CountError(err) 188 log.Fatalf("Source doesn't exist or is a directory and destination is a file") 189 case nil: 190 default: 191 fs.CountError(err) 192 log.Fatalf("Failed to create file system for destination %q: %v", dstRemote, err) 193 } 194 return 195 } 196 197 // NewFsDstFile creates a new dst fs with a destination file name from the arguments 198 func NewFsDstFile(args []string) (fdst fs.Fs, dstFileName string) { 199 dstRemote, dstFileName := fspath.Split(args[0]) 200 if dstRemote == "" { 201 dstRemote = "." 202 } 203 if dstFileName == "" { 204 log.Fatalf("%q is a directory", args[0]) 205 } 206 fdst = newFsDir(dstRemote) 207 return 208 } 209 210 // ShowStats returns true if the user added a `--stats` flag to the command line. 211 // 212 // This is called by Run to override the default value of the 213 // showStats passed in. 214 func ShowStats() bool { 215 statsIntervalFlag := pflag.Lookup("stats") 216 return statsIntervalFlag != nil && statsIntervalFlag.Changed 217 } 218 219 // Run the function with stats and retries if required 220 func Run(Retry bool, showStats bool, cmd *cobra.Command, f func() error) { 221 var err error 222 stopStats := func() {} 223 if !showStats && ShowStats() { 224 showStats = true 225 } 226 if fs.Config.Progress { 227 stopStats = startProgress() 228 } else if showStats { 229 stopStats = StartStats() 230 } 231 SigInfoHandler() 232 for try := 1; try <= *retries; try++ { 233 err = f() 234 fs.CountError(err) 235 lastErr := accounting.Stats.GetLastError() 236 if err == nil { 237 err = lastErr 238 } 239 if !Retry || !accounting.Stats.Errored() { 240 if try > 1 { 241 fs.Errorf(nil, "Attempt %d/%d succeeded", try, *retries) 242 } 243 break 244 } 245 if accounting.Stats.HadFatalError() { 246 fs.Errorf(nil, "Fatal error received - not attempting retries") 247 break 248 } 249 if accounting.Stats.Errored() && !accounting.Stats.HadRetryError() { 250 fs.Errorf(nil, "Can't retry this error - not attempting retries") 251 break 252 } 253 if retryAfter := accounting.Stats.RetryAfter(); !retryAfter.IsZero() { 254 d := retryAfter.Sub(time.Now()) 255 if d > 0 { 256 fs.Logf(nil, "Received retry after error - sleeping until %s (%v)", retryAfter.Format(time.RFC3339Nano), d) 257 time.Sleep(d) 258 } 259 } 260 if lastErr != nil { 261 fs.Errorf(nil, "Attempt %d/%d failed with %d errors and: %v", try, *retries, accounting.Stats.GetErrors(), lastErr) 262 } else { 263 fs.Errorf(nil, "Attempt %d/%d failed with %d errors", try, *retries, accounting.Stats.GetErrors()) 264 } 265 if try < *retries { 266 accounting.Stats.ResetErrors() 267 } 268 if *retriesInterval > 0 { 269 time.Sleep(*retriesInterval) 270 } 271 } 272 stopStats() 273 if err != nil { 274 nerrs := accounting.Stats.GetErrors() 275 if nerrs <= 1 { 276 log.Printf("Failed to %s: %v", cmd.Name(), err) 277 } else { 278 log.Printf("Failed to %s with %d errors: last error was: %v", cmd.Name(), nerrs, err) 279 } 280 resolveExitCode(err) 281 } 282 if showStats && (accounting.Stats.Errored() || *statsInterval > 0) { 283 accounting.Stats.Log() 284 } 285 fs.Debugf(nil, "%d go routines active\n", runtime.NumGoroutine()) 286 287 // dump all running go-routines 288 if fs.Config.Dump&fs.DumpGoRoutines != 0 { 289 err = pprof.Lookup("goroutine").WriteTo(os.Stdout, 1) 290 if err != nil { 291 fs.Errorf(nil, "Failed to dump goroutines: %v", err) 292 } 293 } 294 295 // dump open files 296 if fs.Config.Dump&fs.DumpOpenFiles != 0 { 297 c := exec.Command("lsof", "-p", strconv.Itoa(os.Getpid())) 298 c.Stdout = os.Stdout 299 c.Stderr = os.Stderr 300 err = c.Run() 301 if err != nil { 302 fs.Errorf(nil, "Failed to list open files: %v", err) 303 } 304 } 305 306 if accounting.Stats.Errored() { 307 resolveExitCode(accounting.Stats.GetLastError()) 308 } 309 } 310 311 // CheckArgs checks there are enough arguments and prints a message if not 312 func CheckArgs(MinArgs, MaxArgs int, cmd *cobra.Command, args []string) { 313 if len(args) < MinArgs { 314 _ = cmd.Usage() 315 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments minimum: you provided %d non flag arguments: %q\n", cmd.Name(), MinArgs, len(args), args) 316 resolveExitCode(errorNotEnoughArguments) 317 } else if len(args) > MaxArgs { 318 _ = cmd.Usage() 319 _, _ = fmt.Fprintf(os.Stderr, "Command %s needs %d arguments maximum: you provided %d non flag arguments: %q\n", cmd.Name(), MaxArgs, len(args), args) 320 resolveExitCode(errorTooManyArguments) 321 } 322 } 323 324 // StartStats prints the stats every statsInterval 325 // 326 // It returns a func which should be called to stop the stats. 327 func StartStats() func() { 328 if *statsInterval <= 0 { 329 return func() {} 330 } 331 stopStats := make(chan struct{}) 332 var wg sync.WaitGroup 333 wg.Add(1) 334 go func() { 335 defer wg.Done() 336 ticker := time.NewTicker(*statsInterval) 337 for { 338 select { 339 case <-ticker.C: 340 accounting.Stats.Log() 341 case <-stopStats: 342 ticker.Stop() 343 return 344 } 345 } 346 }() 347 return func() { 348 close(stopStats) 349 wg.Wait() 350 } 351 } 352 353 // initConfig is run by cobra after initialising the flags 354 func initConfig() { 355 // Start the logger 356 fslog.InitLogging() 357 358 // Finish parsing any command line flags 359 configflags.SetFlags() 360 361 // Load filters 362 err := filterflags.Reload() 363 if err != nil { 364 log.Fatalf("Failed to load filters: %v", err) 365 } 366 367 // Write the args for debug purposes 368 fs.Debugf("rclone", "Version %q starting with parameters %q", fs.Version, os.Args) 369 370 // Start the remote control server if configured 371 _, err = rcserver.Start(&rcflags.Opt) 372 if err != nil { 373 log.Fatalf("Failed to start remote control: %v", err) 374 } 375 376 // Setup CPU profiling if desired 377 if *cpuProfile != "" { 378 fs.Infof(nil, "Creating CPU profile %q\n", *cpuProfile) 379 f, err := os.Create(*cpuProfile) 380 if err != nil { 381 fs.CountError(err) 382 log.Fatal(err) 383 } 384 err = pprof.StartCPUProfile(f) 385 if err != nil { 386 fs.CountError(err) 387 log.Fatal(err) 388 } 389 atexit.Register(func() { 390 pprof.StopCPUProfile() 391 }) 392 } 393 394 // Setup memory profiling if desired 395 if *memProfile != "" { 396 atexit.Register(func() { 397 fs.Infof(nil, "Saving Memory profile %q\n", *memProfile) 398 f, err := os.Create(*memProfile) 399 if err != nil { 400 fs.CountError(err) 401 log.Fatal(err) 402 } 403 err = pprof.WriteHeapProfile(f) 404 if err != nil { 405 fs.CountError(err) 406 log.Fatal(err) 407 } 408 err = f.Close() 409 if err != nil { 410 fs.CountError(err) 411 log.Fatal(err) 412 } 413 }) 414 } 415 416 if m, _ := regexp.MatchString("^(bits|bytes)$", *dataRateUnit); m == false { 417 fs.Errorf(nil, "Invalid unit passed to --stats-unit. Defaulting to bytes.") 418 fs.Config.DataRateUnit = "bytes" 419 } else { 420 fs.Config.DataRateUnit = *dataRateUnit 421 } 422 } 423 424 func resolveExitCode(err error) { 425 atexit.Run() 426 if err == nil { 427 os.Exit(exitCodeSuccess) 428 } 429 430 _, unwrapped := fserrors.Cause(err) 431 432 switch { 433 case unwrapped == fs.ErrorDirNotFound: 434 os.Exit(exitCodeDirNotFound) 435 case unwrapped == fs.ErrorObjectNotFound: 436 os.Exit(exitCodeFileNotFound) 437 case unwrapped == errorUncategorized: 438 os.Exit(exitCodeUncategorizedError) 439 case unwrapped == accounting.ErrorMaxTransferLimitReached: 440 os.Exit(exitCodeTransferExceeded) 441 case fserrors.ShouldRetry(err): 442 os.Exit(exitCodeRetryError) 443 case fserrors.IsNoRetryError(err): 444 os.Exit(exitCodeNoRetryError) 445 case fserrors.IsFatalError(err): 446 os.Exit(exitCodeFatalError) 447 default: 448 os.Exit(exitCodeUsageError) 449 } 450 } 451 452 var backendFlags map[string]struct{} 453 454 // AddBackendFlags creates flags for all the backend options 455 func AddBackendFlags() { 456 backendFlags = map[string]struct{}{} 457 for _, fsInfo := range fs.Registry { 458 done := map[string]struct{}{} 459 for i := range fsInfo.Options { 460 opt := &fsInfo.Options[i] 461 // Skip if done already (eg with Provider options) 462 if _, doneAlready := done[opt.Name]; doneAlready { 463 continue 464 } 465 done[opt.Name] = struct{}{} 466 // Make a flag from each option 467 name := opt.FlagName(fsInfo.Prefix) 468 found := pflag.CommandLine.Lookup(name) != nil 469 if !found { 470 // Take first line of help only 471 help := strings.TrimSpace(opt.Help) 472 if nl := strings.IndexRune(help, '\n'); nl >= 0 { 473 help = help[:nl] 474 } 475 help = strings.TrimSpace(help) 476 flag := pflag.CommandLine.VarPF(opt, name, opt.ShortOpt, help) 477 if _, isBool := opt.Default.(bool); isBool { 478 flag.NoOptDefVal = "true" 479 } 480 // Hide on the command line if requested 481 if opt.Hide&fs.OptionHideCommandLine != 0 { 482 flag.Hidden = true 483 } 484 backendFlags[name] = struct{}{} 485 } else { 486 fs.Errorf(nil, "Not adding duplicate flag --%s", name) 487 } 488 //flag.Hidden = true 489 } 490 } 491 } 492 493 // Main runs rclone interpreting flags and commands out of os.Args 494 func Main() { 495 setupRootCommand(Root) 496 AddBackendFlags() 497 if err := Root.Execute(); err != nil { 498 log.Fatalf("Fatal error: %v", err) 499 } 500 }