github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/fs/config.go (about) 1 package fs 2 3 import ( 4 "context" 5 "errors" 6 "net" 7 "os" 8 "strconv" 9 "strings" 10 "time" 11 ) 12 13 // Global 14 var ( 15 // globalConfig for rclone 16 globalConfig = NewConfig() 17 18 // Read a value from the config file 19 // 20 // This is a function pointer to decouple the config 21 // implementation from the fs 22 ConfigFileGet = func(section, key string) (string, bool) { return "", false } 23 24 // Set a value into the config file and persist it 25 // 26 // This is a function pointer to decouple the config 27 // implementation from the fs 28 ConfigFileSet = func(section, key, value string) (err error) { 29 return errors.New("no config file set handler") 30 } 31 32 // Check if the config file has the named section 33 // 34 // This is a function pointer to decouple the config 35 // implementation from the fs 36 ConfigFileHasSection = func(section string) bool { return false } 37 38 // CountError counts an error. If any errors have been 39 // counted then rclone will exit with a non zero error code. 40 // 41 // This is a function pointer to decouple the config 42 // implementation from the fs 43 CountError = func(err error) error { return err } 44 45 // ConfigProvider is the config key used for provider options 46 ConfigProvider = "provider" 47 48 // ConfigEdit is the config key used to show we wish to edit existing entries 49 ConfigEdit = "config_fs_edit" 50 ) 51 52 // ConfigInfo is filesystem config options 53 type ConfigInfo struct { 54 LogLevel LogLevel 55 StatsLogLevel LogLevel 56 UseJSONLog bool 57 DryRun bool 58 Interactive bool 59 CheckSum bool 60 SizeOnly bool 61 IgnoreTimes bool 62 IgnoreExisting bool 63 IgnoreErrors bool 64 ModifyWindow time.Duration 65 Checkers int 66 Transfers int 67 ConnectTimeout time.Duration // Connect timeout 68 Timeout time.Duration // Data channel timeout 69 ExpectContinueTimeout time.Duration 70 Dump DumpFlags 71 InsecureSkipVerify bool // Skip server certificate verification 72 DeleteMode DeleteMode 73 MaxDelete int64 74 MaxDeleteSize SizeSuffix 75 TrackRenames bool // Track file renames. 76 TrackRenamesStrategy string // Comma separated list of strategies used to track renames 77 Retries int // High-level retries 78 RetriesInterval time.Duration // --retries-sleep 79 LowLevelRetries int 80 UpdateOlder bool // Skip files that are newer on the destination 81 NoGzip bool // Disable compression 82 MaxDepth int 83 IgnoreSize bool 84 IgnoreChecksum bool 85 IgnoreCaseSync bool 86 FixCase bool 87 NoTraverse bool 88 CheckFirst bool 89 NoCheckDest bool 90 NoUnicodeNormalization bool 91 NoUpdateModTime bool 92 NoUpdateDirModTime bool 93 DataRateUnit string 94 CompareDest []string 95 CopyDest []string 96 BackupDir string 97 Suffix string 98 SuffixKeepExtension bool 99 UseListR bool 100 BufferSize SizeSuffix 101 BwLimit BwTimetable 102 BwLimitFile BwTimetable 103 TPSLimit float64 104 TPSLimitBurst int 105 BindAddr net.IP 106 DisableFeatures []string 107 UserAgent string 108 Immutable bool 109 AutoConfirm bool 110 StreamingUploadCutoff SizeSuffix 111 StatsFileNameLength int 112 AskPassword bool 113 PasswordCommand SpaceSepList 114 UseServerModTime bool 115 MaxTransfer SizeSuffix 116 MaxDuration time.Duration 117 CutoffMode CutoffMode 118 MaxBacklog int 119 MaxStatsGroups int 120 StatsOneLine bool 121 StatsOneLineDate bool // If we want a date prefix at all 122 StatsOneLineDateFormat string // If we want to customize the prefix 123 ErrorOnNoTransfer bool // Set appropriate exit code if no files transferred 124 Progress bool 125 ProgressTerminalTitle bool 126 Cookie bool 127 UseMmap bool 128 CaCert []string // Client Side CA 129 ClientCert string // Client Side Cert 130 ClientKey string // Client Side Key 131 MultiThreadCutoff SizeSuffix 132 MultiThreadStreams int 133 MultiThreadSet bool // whether MultiThreadStreams was set (set in fs/config/configflags) 134 MultiThreadChunkSize SizeSuffix // Chunk size for multi-thread downloads / uploads, if not set by filesystem 135 MultiThreadWriteBufferSize SizeSuffix 136 OrderBy string // instructions on how to order the transfer 137 UploadHeaders []*HTTPOption 138 DownloadHeaders []*HTTPOption 139 Headers []*HTTPOption 140 MetadataSet Metadata // extra metadata to write when uploading 141 RefreshTimes bool 142 NoConsole bool 143 TrafficClass uint8 144 FsCacheExpireDuration time.Duration 145 FsCacheExpireInterval time.Duration 146 DisableHTTP2 bool 147 HumanReadable bool 148 KvLockTime time.Duration // maximum time to keep key-value database locked by process 149 DisableHTTPKeepAlives bool 150 Metadata bool 151 ServerSideAcrossConfigs bool 152 TerminalColorMode TerminalColorMode 153 DefaultTime Time // time that directories with no time should display 154 Inplace bool // Download directly to destination file instead of atomic download to temp/rename 155 PartialSuffix string 156 MetadataMapper SpaceSepList 157 } 158 159 // NewConfig creates a new config with everything set to the default 160 // value. These are the ultimate defaults and are overridden by the 161 // config module. 162 func NewConfig() *ConfigInfo { 163 c := new(ConfigInfo) 164 165 // Set any values which aren't the zero for the type 166 c.LogLevel = LogLevelNotice 167 c.StatsLogLevel = LogLevelInfo 168 c.ModifyWindow = time.Nanosecond 169 c.Checkers = 8 170 c.Transfers = 4 171 c.ConnectTimeout = 60 * time.Second 172 c.Timeout = 5 * 60 * time.Second 173 c.ExpectContinueTimeout = 1 * time.Second 174 c.DeleteMode = DeleteModeDefault 175 c.MaxDelete = -1 176 c.MaxDeleteSize = SizeSuffix(-1) 177 c.Retries = 3 178 c.LowLevelRetries = 10 179 c.MaxDepth = -1 180 c.DataRateUnit = "bytes" 181 c.BufferSize = SizeSuffix(16 << 20) 182 c.UserAgent = "rclone/" + Version 183 c.StreamingUploadCutoff = SizeSuffix(100 * 1024) 184 c.MaxStatsGroups = 1000 185 c.StatsFileNameLength = 45 186 c.AskPassword = true 187 c.TPSLimitBurst = 1 188 c.MaxTransfer = -1 189 c.MaxBacklog = 10000 190 // We do not want to set the default here. We use this variable being empty as part of the fall-through of options. 191 // c.StatsOneLineDateFormat = "2006/01/02 15:04:05 - " 192 c.MultiThreadCutoff = SizeSuffix(256 * 1024 * 1024) 193 c.MultiThreadStreams = 4 194 c.MultiThreadChunkSize = SizeSuffix(64 * 1024 * 1024) 195 c.MultiThreadWriteBufferSize = SizeSuffix(128 * 1024) 196 197 c.TrackRenamesStrategy = "hash" 198 c.FsCacheExpireDuration = 300 * time.Second 199 c.FsCacheExpireInterval = 60 * time.Second 200 c.KvLockTime = 1 * time.Second 201 c.DefaultTime = Time(time.Date(2000, 1, 1, 0, 0, 0, 0, time.UTC)) 202 c.PartialSuffix = ".partial" 203 204 // Perform a simple check for debug flags to enable debug logging during the flag initialization 205 for argIndex, arg := range os.Args { 206 if strings.HasPrefix(arg, "-vv") && strings.TrimRight(arg, "v") == "-" { 207 c.LogLevel = LogLevelDebug 208 } 209 if arg == "--log-level=DEBUG" || (arg == "--log-level" && len(os.Args) > argIndex+1 && os.Args[argIndex+1] == "DEBUG") { 210 c.LogLevel = LogLevelDebug 211 } 212 if strings.HasPrefix(arg, "--verbose=") { 213 if level, err := strconv.Atoi(arg[10:]); err == nil && level >= 2 { 214 c.LogLevel = LogLevelDebug 215 } 216 } 217 } 218 envValue, found := os.LookupEnv("RCLONE_LOG_LEVEL") 219 if found && envValue == "DEBUG" { 220 c.LogLevel = LogLevelDebug 221 } 222 223 return c 224 } 225 226 // TimeoutOrInfinite returns ci.Timeout if > 0 or infinite otherwise 227 func (c *ConfigInfo) TimeoutOrInfinite() time.Duration { 228 if c.Timeout > 0 { 229 return c.Timeout 230 } 231 return ModTimeNotSupported 232 } 233 234 type configContextKeyType struct{} 235 236 // Context key for config 237 var configContextKey = configContextKeyType{} 238 239 // GetConfig returns the global or context sensitive context 240 func GetConfig(ctx context.Context) *ConfigInfo { 241 if ctx == nil { 242 return globalConfig 243 } 244 c := ctx.Value(configContextKey) 245 if c == nil { 246 return globalConfig 247 } 248 return c.(*ConfigInfo) 249 } 250 251 // CopyConfig copies the global config (if any) from srcCtx into 252 // dstCtx returning the new context. 253 func CopyConfig(dstCtx, srcCtx context.Context) context.Context { 254 if srcCtx == nil { 255 return dstCtx 256 } 257 c := srcCtx.Value(configContextKey) 258 if c == nil { 259 return dstCtx 260 } 261 return context.WithValue(dstCtx, configContextKey, c) 262 } 263 264 // AddConfig returns a mutable config structure based on a shallow 265 // copy of that found in ctx and returns a new context with that added 266 // to it. 267 func AddConfig(ctx context.Context) (context.Context, *ConfigInfo) { 268 c := GetConfig(ctx) 269 cCopy := new(ConfigInfo) 270 *cCopy = *c 271 newCtx := context.WithValue(ctx, configContextKey, cCopy) 272 return newCtx, cCopy 273 } 274 275 // ConfigToEnv converts a config section and name, e.g. ("my-remote", 276 // "ignore-size") into an environment name 277 // "RCLONE_CONFIG_MY-REMOTE_IGNORE_SIZE" 278 func ConfigToEnv(section, name string) string { 279 return "RCLONE_CONFIG_" + strings.ToUpper(section+"_"+strings.ReplaceAll(name, "-", "_")) 280 } 281 282 // OptionToEnv converts an option name, e.g. "ignore-size" into an 283 // environment name "RCLONE_IGNORE_SIZE" 284 func OptionToEnv(name string) string { 285 return "RCLONE_" + strings.ToUpper(strings.ReplaceAll(name, "-", "_")) 286 }