github.com/10XDev/rclone@v1.52.3-0.20200626220027-16af9ab76b2a/fs/config/configflags/configflags.go (about) 1 // Package configflags defines the flags used by rclone. It is 2 // decoupled into a separate package so it can be replaced. 3 package configflags 4 5 // Options set by command line flags 6 import ( 7 "log" 8 "net" 9 "path/filepath" 10 "strings" 11 12 "github.com/rclone/rclone/fs" 13 "github.com/rclone/rclone/fs/config" 14 "github.com/rclone/rclone/fs/config/flags" 15 fsLog "github.com/rclone/rclone/fs/log" 16 "github.com/rclone/rclone/fs/rc" 17 "github.com/sirupsen/logrus" 18 "github.com/spf13/pflag" 19 ) 20 21 var ( 22 // these will get interpreted into fs.Config via SetFlags() below 23 verbose int 24 quiet bool 25 dumpHeaders bool 26 dumpBodies bool 27 deleteBefore bool 28 deleteDuring bool 29 deleteAfter bool 30 bindAddr string 31 disableFeatures string 32 uploadHeaders []string 33 downloadHeaders []string 34 headers []string 35 ) 36 37 // AddFlags adds the non filing system specific flags to the command 38 func AddFlags(flagSet *pflag.FlagSet) { 39 rc.AddOption("main", fs.Config) 40 // NB defaults which aren't the zero for the type should be set in fs/config.go NewConfig 41 flags.CountVarP(flagSet, &verbose, "verbose", "v", "Print lots more stuff (repeat for more)") 42 flags.BoolVarP(flagSet, &quiet, "quiet", "q", false, "Print as little stuff as possible") 43 flags.DurationVarP(flagSet, &fs.Config.ModifyWindow, "modify-window", "", fs.Config.ModifyWindow, "Max time diff to be considered the same") 44 flags.IntVarP(flagSet, &fs.Config.Checkers, "checkers", "", fs.Config.Checkers, "Number of checkers to run in parallel.") 45 flags.IntVarP(flagSet, &fs.Config.Transfers, "transfers", "", fs.Config.Transfers, "Number of file transfers to run in parallel.") 46 flags.StringVarP(flagSet, &config.ConfigPath, "config", "", config.ConfigPath, "Config file.") 47 flags.StringVarP(flagSet, &config.CacheDir, "cache-dir", "", config.CacheDir, "Directory rclone will use for caching.") 48 flags.BoolVarP(flagSet, &fs.Config.CheckSum, "checksum", "c", fs.Config.CheckSum, "Skip based on checksum (if available) & size, not mod-time & size") 49 flags.BoolVarP(flagSet, &fs.Config.SizeOnly, "size-only", "", fs.Config.SizeOnly, "Skip based on size only, not mod-time or checksum") 50 flags.BoolVarP(flagSet, &fs.Config.IgnoreTimes, "ignore-times", "I", fs.Config.IgnoreTimes, "Don't skip files that match size and time - transfer all files") 51 flags.BoolVarP(flagSet, &fs.Config.IgnoreExisting, "ignore-existing", "", fs.Config.IgnoreExisting, "Skip all files that exist on destination") 52 flags.BoolVarP(flagSet, &fs.Config.IgnoreErrors, "ignore-errors", "", fs.Config.IgnoreErrors, "delete even if there are I/O errors") 53 flags.BoolVarP(flagSet, &fs.Config.DryRun, "dry-run", "n", fs.Config.DryRun, "Do a trial run with no permanent changes") 54 flags.DurationVarP(flagSet, &fs.Config.ConnectTimeout, "contimeout", "", fs.Config.ConnectTimeout, "Connect timeout") 55 flags.DurationVarP(flagSet, &fs.Config.Timeout, "timeout", "", fs.Config.Timeout, "IO idle timeout") 56 flags.DurationVarP(flagSet, &fs.Config.ExpectContinueTimeout, "expect-continue-timeout", "", fs.Config.ExpectContinueTimeout, "Timeout when using expect / 100-continue in HTTP") 57 flags.BoolVarP(flagSet, &dumpHeaders, "dump-headers", "", false, "Dump HTTP headers - may contain sensitive info") 58 flags.BoolVarP(flagSet, &dumpBodies, "dump-bodies", "", false, "Dump HTTP headers and bodies - may contain sensitive info") 59 flags.BoolVarP(flagSet, &fs.Config.InsecureSkipVerify, "no-check-certificate", "", fs.Config.InsecureSkipVerify, "Do not verify the server SSL certificate. Insecure.") 60 flags.BoolVarP(flagSet, &fs.Config.AskPassword, "ask-password", "", fs.Config.AskPassword, "Allow prompt for password for encrypted configuration.") 61 flags.FVarP(flagSet, &fs.Config.PasswordCommand, "password-command", "", "Command for supplying password for encrypted configuration.") 62 flags.BoolVarP(flagSet, &deleteBefore, "delete-before", "", false, "When synchronizing, delete files on destination before transferring") 63 flags.BoolVarP(flagSet, &deleteDuring, "delete-during", "", false, "When synchronizing, delete files during transfer") 64 flags.BoolVarP(flagSet, &deleteAfter, "delete-after", "", false, "When synchronizing, delete files on destination after transferring (default)") 65 flags.Int64VarP(flagSet, &fs.Config.MaxDelete, "max-delete", "", -1, "When synchronizing, limit the number of deletes") 66 flags.BoolVarP(flagSet, &fs.Config.TrackRenames, "track-renames", "", fs.Config.TrackRenames, "When synchronizing, track file renames and do a server side move if possible") 67 flags.StringVarP(flagSet, &fs.Config.TrackRenamesStrategy, "track-renames-strategy", "", fs.Config.TrackRenamesStrategy, "Strategies to use when synchronizing using track-renames hash|modtime") 68 flags.IntVarP(flagSet, &fs.Config.LowLevelRetries, "low-level-retries", "", fs.Config.LowLevelRetries, "Number of low level retries to do.") 69 flags.BoolVarP(flagSet, &fs.Config.UpdateOlder, "update", "u", fs.Config.UpdateOlder, "Skip files that are newer on the destination.") 70 flags.BoolVarP(flagSet, &fs.Config.UseServerModTime, "use-server-modtime", "", fs.Config.UseServerModTime, "Use server modified time instead of object metadata") 71 flags.BoolVarP(flagSet, &fs.Config.NoGzip, "no-gzip-encoding", "", fs.Config.NoGzip, "Don't set Accept-Encoding: gzip.") 72 flags.IntVarP(flagSet, &fs.Config.MaxDepth, "max-depth", "", fs.Config.MaxDepth, "If set limits the recursion depth to this.") 73 flags.BoolVarP(flagSet, &fs.Config.IgnoreSize, "ignore-size", "", false, "Ignore size when skipping use mod-time or checksum.") 74 flags.BoolVarP(flagSet, &fs.Config.IgnoreChecksum, "ignore-checksum", "", fs.Config.IgnoreChecksum, "Skip post copy check of checksums.") 75 flags.BoolVarP(flagSet, &fs.Config.IgnoreCaseSync, "ignore-case-sync", "", fs.Config.IgnoreCaseSync, "Ignore case when synchronizing") 76 flags.BoolVarP(flagSet, &fs.Config.NoTraverse, "no-traverse", "", fs.Config.NoTraverse, "Don't traverse destination file system on copy.") 77 flags.BoolVarP(flagSet, &fs.Config.CheckFirst, "check-first", "", fs.Config.CheckFirst, "Do all the checks before starting transfers.") 78 flags.BoolVarP(flagSet, &fs.Config.NoCheckDest, "no-check-dest", "", fs.Config.NoCheckDest, "Don't check the destination, copy regardless.") 79 flags.BoolVarP(flagSet, &fs.Config.NoUnicodeNormalization, "no-unicode-normalization", "", fs.Config.NoUnicodeNormalization, "Don't normalize unicode characters in filenames.") 80 flags.BoolVarP(flagSet, &fs.Config.NoUpdateModTime, "no-update-modtime", "", fs.Config.NoUpdateModTime, "Don't update destination mod-time if files identical.") 81 flags.StringVarP(flagSet, &fs.Config.CompareDest, "compare-dest", "", fs.Config.CompareDest, "Include additional server-side path during comparison.") 82 flags.StringVarP(flagSet, &fs.Config.CopyDest, "copy-dest", "", fs.Config.CopyDest, "Implies --compare-dest but also copies files from path into destination.") 83 flags.StringVarP(flagSet, &fs.Config.BackupDir, "backup-dir", "", fs.Config.BackupDir, "Make backups into hierarchy based in DIR.") 84 flags.StringVarP(flagSet, &fs.Config.Suffix, "suffix", "", fs.Config.Suffix, "Suffix to add to changed files.") 85 flags.BoolVarP(flagSet, &fs.Config.SuffixKeepExtension, "suffix-keep-extension", "", fs.Config.SuffixKeepExtension, "Preserve the extension when using --suffix.") 86 flags.BoolVarP(flagSet, &fs.Config.UseListR, "fast-list", "", fs.Config.UseListR, "Use recursive list if available. Uses more memory but fewer transactions.") 87 flags.Float64VarP(flagSet, &fs.Config.TPSLimit, "tpslimit", "", fs.Config.TPSLimit, "Limit HTTP transactions per second to this.") 88 flags.IntVarP(flagSet, &fs.Config.TPSLimitBurst, "tpslimit-burst", "", fs.Config.TPSLimitBurst, "Max burst of transactions for --tpslimit.") 89 flags.StringVarP(flagSet, &bindAddr, "bind", "", "", "Local address to bind to for outgoing connections, IPv4, IPv6 or name.") 90 flags.StringVarP(flagSet, &disableFeatures, "disable", "", "", "Disable a comma separated list of features. Use help to see a list.") 91 flags.StringVarP(flagSet, &fs.Config.UserAgent, "user-agent", "", fs.Config.UserAgent, "Set the user-agent to a specified string. The default is rclone/ version") 92 flags.BoolVarP(flagSet, &fs.Config.Immutable, "immutable", "", fs.Config.Immutable, "Do not modify files. Fail if existing files have been modified.") 93 flags.BoolVarP(flagSet, &fs.Config.AutoConfirm, "auto-confirm", "", fs.Config.AutoConfirm, "If enabled, do not request console confirmation.") 94 flags.IntVarP(flagSet, &fs.Config.StatsFileNameLength, "stats-file-name-length", "", fs.Config.StatsFileNameLength, "Max file name length in stats. 0 for no limit") 95 flags.FVarP(flagSet, &fs.Config.LogLevel, "log-level", "", "Log level DEBUG|INFO|NOTICE|ERROR") 96 flags.FVarP(flagSet, &fs.Config.StatsLogLevel, "stats-log-level", "", "Log level to show --stats output DEBUG|INFO|NOTICE|ERROR") 97 flags.FVarP(flagSet, &fs.Config.BwLimit, "bwlimit", "", "Bandwidth limit in kBytes/s, or use suffix b|k|M|G or a full timetable.") 98 flags.FVarP(flagSet, &fs.Config.BufferSize, "buffer-size", "", "In memory buffer size when reading files for each --transfer.") 99 flags.FVarP(flagSet, &fs.Config.StreamingUploadCutoff, "streaming-upload-cutoff", "", "Cutoff for switching to chunked upload if file size is unknown. Upload starts after reaching cutoff or when file ends.") 100 flags.FVarP(flagSet, &fs.Config.Dump, "dump", "", "List of items to dump from: "+fs.DumpFlagsList) 101 flags.FVarP(flagSet, &fs.Config.MaxTransfer, "max-transfer", "", "Maximum size of data to transfer.") 102 flags.DurationVarP(flagSet, &fs.Config.MaxDuration, "max-duration", "", 0, "Maximum duration rclone will transfer data for.") 103 flags.FVarP(flagSet, &fs.Config.CutoffMode, "cutoff-mode", "", "Mode to stop transfers when reaching the max transfer limit HARD|SOFT|CAUTIOUS") 104 flags.IntVarP(flagSet, &fs.Config.MaxBacklog, "max-backlog", "", fs.Config.MaxBacklog, "Maximum number of objects in sync or check backlog.") 105 flags.IntVarP(flagSet, &fs.Config.MaxStatsGroups, "max-stats-groups", "", fs.Config.MaxStatsGroups, "Maximum number of stats groups to keep in memory. On max oldest is discarded.") 106 flags.BoolVarP(flagSet, &fs.Config.StatsOneLine, "stats-one-line", "", fs.Config.StatsOneLine, "Make the stats fit on one line.") 107 flags.BoolVarP(flagSet, &fs.Config.StatsOneLineDate, "stats-one-line-date", "", fs.Config.StatsOneLineDate, "Enables --stats-one-line and add current date/time prefix.") 108 flags.StringVarP(flagSet, &fs.Config.StatsOneLineDateFormat, "stats-one-line-date-format", "", fs.Config.StatsOneLineDateFormat, "Enables --stats-one-line-date and uses custom formatted date. Enclose date string in double quotes (\"). See https://golang.org/pkg/time/#Time.Format") 109 flags.BoolVarP(flagSet, &fs.Config.ErrorOnNoTransfer, "error-on-no-transfer", "", fs.Config.ErrorOnNoTransfer, "Sets exit code 9 if no files are transferred, useful in scripts") 110 flags.BoolVarP(flagSet, &fs.Config.Progress, "progress", "P", fs.Config.Progress, "Show progress during transfer.") 111 flags.BoolVarP(flagSet, &fs.Config.Cookie, "use-cookies", "", fs.Config.Cookie, "Enable session cookiejar.") 112 flags.BoolVarP(flagSet, &fs.Config.UseMmap, "use-mmap", "", fs.Config.UseMmap, "Use mmap allocator (see docs).") 113 flags.StringVarP(flagSet, &fs.Config.CaCert, "ca-cert", "", fs.Config.CaCert, "CA certificate used to verify servers") 114 flags.StringVarP(flagSet, &fs.Config.ClientCert, "client-cert", "", fs.Config.ClientCert, "Client SSL certificate (PEM) for mutual TLS auth") 115 flags.StringVarP(flagSet, &fs.Config.ClientKey, "client-key", "", fs.Config.ClientKey, "Client SSL private key (PEM) for mutual TLS auth") 116 flags.FVarP(flagSet, &fs.Config.MultiThreadCutoff, "multi-thread-cutoff", "", "Use multi-thread downloads for files above this size.") 117 flags.IntVarP(flagSet, &fs.Config.MultiThreadStreams, "multi-thread-streams", "", fs.Config.MultiThreadStreams, "Max number of streams to use for multi-thread downloads.") 118 flags.BoolVarP(flagSet, &fs.Config.UseJSONLog, "use-json-log", "", fs.Config.UseJSONLog, "Use json log format.") 119 flags.StringVarP(flagSet, &fs.Config.OrderBy, "order-by", "", fs.Config.OrderBy, "Instructions on how to order the transfers, eg 'size,descending'") 120 flags.StringArrayVarP(flagSet, &uploadHeaders, "header-upload", "", nil, "Set HTTP header for upload transactions") 121 flags.StringArrayVarP(flagSet, &downloadHeaders, "header-download", "", nil, "Set HTTP header for download transactions") 122 flags.StringArrayVarP(flagSet, &headers, "header", "", nil, "Set HTTP header for all transactions") 123 } 124 125 // ParseHeaders converts the strings passed in via the header flags into HTTPOptions 126 func ParseHeaders(headers []string) []*fs.HTTPOption { 127 opts := []*fs.HTTPOption{} 128 for _, header := range headers { 129 parts := strings.SplitN(header, ":", 2) 130 if len(parts) == 1 { 131 log.Fatalf("Failed to parse '%s' as an HTTP header. Expecting a string like: 'Content-Encoding: gzip'", header) 132 } 133 option := &fs.HTTPOption{ 134 Key: strings.TrimSpace(parts[0]), 135 Value: strings.TrimSpace(parts[1]), 136 } 137 opts = append(opts, option) 138 } 139 return opts 140 } 141 142 // SetFlags converts any flags into config which weren't straight forward 143 func SetFlags() { 144 if verbose >= 2 { 145 fs.Config.LogLevel = fs.LogLevelDebug 146 } else if verbose >= 1 { 147 fs.Config.LogLevel = fs.LogLevelInfo 148 } 149 if quiet { 150 if verbose > 0 { 151 log.Fatalf("Can't set -v and -q") 152 } 153 fs.Config.LogLevel = fs.LogLevelError 154 } 155 logLevelFlag := pflag.Lookup("log-level") 156 if logLevelFlag != nil && logLevelFlag.Changed { 157 if verbose > 0 { 158 log.Fatalf("Can't set -v and --log-level") 159 } 160 if quiet { 161 log.Fatalf("Can't set -q and --log-level") 162 } 163 } 164 if fs.Config.UseJSONLog { 165 logrus.AddHook(fsLog.NewCallerHook()) 166 logrus.SetFormatter(&logrus.JSONFormatter{ 167 TimestampFormat: "2006-01-02T15:04:05.999999-07:00", 168 }) 169 logrus.SetLevel(logrus.DebugLevel) 170 switch fs.Config.LogLevel { 171 case fs.LogLevelEmergency, fs.LogLevelAlert: 172 logrus.SetLevel(logrus.PanicLevel) 173 case fs.LogLevelCritical: 174 logrus.SetLevel(logrus.FatalLevel) 175 case fs.LogLevelError: 176 logrus.SetLevel(logrus.ErrorLevel) 177 case fs.LogLevelWarning, fs.LogLevelNotice: 178 logrus.SetLevel(logrus.WarnLevel) 179 case fs.LogLevelInfo: 180 logrus.SetLevel(logrus.InfoLevel) 181 case fs.LogLevelDebug: 182 logrus.SetLevel(logrus.DebugLevel) 183 } 184 } 185 186 if dumpHeaders { 187 fs.Config.Dump |= fs.DumpHeaders 188 fs.Logf(nil, "--dump-headers is obsolete - please use --dump headers instead") 189 } 190 if dumpBodies { 191 fs.Config.Dump |= fs.DumpBodies 192 fs.Logf(nil, "--dump-bodies is obsolete - please use --dump bodies instead") 193 } 194 195 switch { 196 case deleteBefore && (deleteDuring || deleteAfter), 197 deleteDuring && deleteAfter: 198 log.Fatalf(`Only one of --delete-before, --delete-during or --delete-after can be used.`) 199 case deleteBefore: 200 fs.Config.DeleteMode = fs.DeleteModeBefore 201 case deleteDuring: 202 fs.Config.DeleteMode = fs.DeleteModeDuring 203 case deleteAfter: 204 fs.Config.DeleteMode = fs.DeleteModeAfter 205 default: 206 fs.Config.DeleteMode = fs.DeleteModeDefault 207 } 208 209 if fs.Config.CompareDest != "" && fs.Config.CopyDest != "" { 210 log.Fatalf(`Can't use --compare-dest with --copy-dest.`) 211 } 212 213 switch { 214 case len(fs.Config.StatsOneLineDateFormat) > 0: 215 fs.Config.StatsOneLineDate = true 216 fs.Config.StatsOneLine = true 217 case fs.Config.StatsOneLineDate: 218 fs.Config.StatsOneLineDateFormat = "2006/01/02 15:04:05 - " 219 fs.Config.StatsOneLine = true 220 } 221 222 if bindAddr != "" { 223 addrs, err := net.LookupIP(bindAddr) 224 if err != nil { 225 log.Fatalf("--bind: Failed to parse %q as IP address: %v", bindAddr, err) 226 } 227 if len(addrs) != 1 { 228 log.Fatalf("--bind: Expecting 1 IP address for %q but got %d", bindAddr, len(addrs)) 229 } 230 fs.Config.BindAddr = addrs[0] 231 } 232 233 if disableFeatures != "" { 234 if disableFeatures == "help" { 235 log.Fatalf("Possible backend features are: %s\n", strings.Join(new(fs.Features).List(), ", ")) 236 } 237 fs.Config.DisableFeatures = strings.Split(disableFeatures, ",") 238 } 239 240 if len(uploadHeaders) != 0 { 241 fs.Config.UploadHeaders = ParseHeaders(uploadHeaders) 242 } 243 if len(downloadHeaders) != 0 { 244 fs.Config.DownloadHeaders = ParseHeaders(downloadHeaders) 245 } 246 if len(headers) != 0 { 247 fs.Config.Headers = ParseHeaders(headers) 248 } 249 250 // Make the config file absolute 251 configPath, err := filepath.Abs(config.ConfigPath) 252 if err == nil { 253 config.ConfigPath = configPath 254 } 255 256 // Set whether multi-thread-streams was set 257 multiThreadStreamsFlag := pflag.Lookup("multi-thread-streams") 258 fs.Config.MultiThreadSet = multiThreadStreamsFlag != nil && multiThreadStreamsFlag.Changed 259 260 }