github.com/divyam234/rclone@v1.64.1/fs/accounting/stats_groups.go (about) 1 package accounting 2 3 import ( 4 "context" 5 "fmt" 6 "sync" 7 8 "github.com/divyam234/rclone/fs/rc" 9 10 "github.com/divyam234/rclone/fs" 11 ) 12 13 const globalStats = "global_stats" 14 15 var groups *statsGroups 16 17 func init() { 18 // Init stats container 19 groups = newStatsGroups() 20 21 // Set the function pointer up in fs 22 fs.CountError = GlobalStats().Error 23 } 24 25 func rcListStats(ctx context.Context, in rc.Params) (rc.Params, error) { 26 out := make(rc.Params) 27 28 out["groups"] = groups.names() 29 30 return out, nil 31 } 32 33 func init() { 34 rc.Add(rc.Call{ 35 Path: "core/group-list", 36 Fn: rcListStats, 37 Title: "Returns list of stats.", 38 Help: ` 39 This returns list of stats groups currently in memory. 40 41 Returns the following values: 42 ` + "```" + ` 43 { 44 "groups": an array of group names: 45 [ 46 "group1", 47 "group2", 48 ... 49 ] 50 } 51 ` + "```" + ` 52 `, 53 }) 54 } 55 56 func rcRemoteStats(ctx context.Context, in rc.Params) (rc.Params, error) { 57 // Check to see if we should filter by group. 58 group, err := in.GetString("group") 59 if rc.NotErrParamNotFound(err) { 60 return rc.Params{}, err 61 } 62 if group != "" { 63 return StatsGroup(ctx, group).RemoteStats() 64 } 65 66 return groups.sum(ctx).RemoteStats() 67 } 68 69 func init() { 70 rc.Add(rc.Call{ 71 Path: "core/stats", 72 Fn: rcRemoteStats, 73 Title: "Returns stats about current transfers.", 74 Help: ` 75 This returns all available stats: 76 77 rclone rc core/stats 78 79 If group is not provided then summed up stats for all groups will be 80 returned. 81 82 Parameters 83 84 - group - name of the stats group (string) 85 86 Returns the following values: 87 88 ` + "```" + ` 89 { 90 "bytes": total transferred bytes since the start of the group, 91 "checks": number of files checked, 92 "deletes" : number of files deleted, 93 "elapsedTime": time in floating point seconds since rclone was started, 94 "errors": number of errors, 95 "eta": estimated time in seconds until the group completes, 96 "fatalError": boolean whether there has been at least one fatal error, 97 "lastError": last error string, 98 "renames" : number of files renamed, 99 "retryError": boolean showing whether there has been at least one non-NoRetryError, 100 "serverSideCopies": number of server side copies done, 101 "serverSideCopyBytes": number bytes server side copied, 102 "serverSideMoves": number of server side moves done, 103 "serverSideMoveBytes": number bytes server side moved, 104 "speed": average speed in bytes per second since start of the group, 105 "totalBytes": total number of bytes in the group, 106 "totalChecks": total number of checks in the group, 107 "totalTransfers": total number of transfers in the group, 108 "transferTime" : total time spent on running jobs, 109 "transfers": number of transferred files, 110 "transferring": an array of currently active file transfers: 111 [ 112 { 113 "bytes": total transferred bytes for this file, 114 "eta": estimated time in seconds until file transfer completion 115 "name": name of the file, 116 "percentage": progress of the file transfer in percent, 117 "speed": average speed over the whole transfer in bytes per second, 118 "speedAvg": current speed in bytes per second as an exponentially weighted moving average, 119 "size": size of the file in bytes 120 } 121 ], 122 "checking": an array of names of currently active file checks 123 [] 124 } 125 ` + "```" + ` 126 Values for "transferring", "checking" and "lastError" are only assigned if data is available. 127 The value for "eta" is null if an eta cannot be determined. 128 `, 129 }) 130 } 131 132 func rcTransferredStats(ctx context.Context, in rc.Params) (rc.Params, error) { 133 // Check to see if we should filter by group. 134 group, err := in.GetString("group") 135 if rc.NotErrParamNotFound(err) { 136 return rc.Params{}, err 137 } 138 139 out := make(rc.Params) 140 if group != "" { 141 out["transferred"] = StatsGroup(ctx, group).Transferred() 142 } else { 143 out["transferred"] = groups.sum(ctx).Transferred() 144 } 145 146 return out, nil 147 } 148 149 func init() { 150 rc.Add(rc.Call{ 151 Path: "core/transferred", 152 Fn: rcTransferredStats, 153 Title: "Returns stats about completed transfers.", 154 Help: ` 155 This returns stats about completed transfers: 156 157 rclone rc core/transferred 158 159 If group is not provided then completed transfers for all groups will be 160 returned. 161 162 Note only the last 100 completed transfers are returned. 163 164 Parameters 165 166 - group - name of the stats group (string) 167 168 Returns the following values: 169 ` + "```" + ` 170 { 171 "transferred": an array of completed transfers (including failed ones): 172 [ 173 { 174 "name": name of the file, 175 "size": size of the file in bytes, 176 "bytes": total transferred bytes for this file, 177 "checked": if the transfer is only checked (skipped, deleted), 178 "timestamp": integer representing millisecond unix epoch, 179 "error": string description of the error (empty if successful), 180 "jobid": id of the job that this transfer belongs to 181 } 182 ] 183 } 184 ` + "```" + ` 185 `, 186 }) 187 } 188 189 func rcResetStats(ctx context.Context, in rc.Params) (rc.Params, error) { 190 // Check to see if we should filter by group. 191 group, err := in.GetString("group") 192 if rc.NotErrParamNotFound(err) { 193 return rc.Params{}, err 194 } 195 196 if group != "" { 197 stats := groups.get(group) 198 if stats == nil { 199 return rc.Params{}, fmt.Errorf("group %q not found", group) 200 } 201 stats.ResetErrors() 202 stats.ResetCounters() 203 } else { 204 groups.reset() 205 } 206 207 return rc.Params{}, nil 208 } 209 210 func init() { 211 rc.Add(rc.Call{ 212 Path: "core/stats-reset", 213 Fn: rcResetStats, 214 Title: "Reset stats.", 215 Help: ` 216 This clears counters, errors and finished transfers for all stats or specific 217 stats group if group is provided. 218 219 Parameters 220 221 - group - name of the stats group (string) 222 `, 223 }) 224 } 225 226 func rcDeleteStats(ctx context.Context, in rc.Params) (rc.Params, error) { 227 // Group name required because we only do single group. 228 group, err := in.GetString("group") 229 if rc.NotErrParamNotFound(err) { 230 return rc.Params{}, err 231 } 232 233 if group != "" { 234 groups.delete(group) 235 } 236 237 return rc.Params{}, nil 238 } 239 240 func init() { 241 rc.Add(rc.Call{ 242 Path: "core/stats-delete", 243 Fn: rcDeleteStats, 244 Title: "Delete stats group.", 245 Help: ` 246 This deletes entire stats group. 247 248 Parameters 249 250 - group - name of the stats group (string) 251 `, 252 }) 253 } 254 255 type statsGroupCtx int64 256 257 const statsGroupKey statsGroupCtx = 1 258 259 // WithStatsGroup returns copy of the parent context with assigned group. 260 func WithStatsGroup(parent context.Context, group string) context.Context { 261 return context.WithValue(parent, statsGroupKey, group) 262 } 263 264 // StatsGroupFromContext returns group from the context if it's available. 265 // Returns false if group is empty. 266 func StatsGroupFromContext(ctx context.Context) (string, bool) { 267 statsGroup, ok := ctx.Value(statsGroupKey).(string) 268 if statsGroup == "" { 269 ok = false 270 } 271 return statsGroup, ok 272 } 273 274 // Stats gets stats by extracting group from context. 275 func Stats(ctx context.Context) *StatsInfo { 276 group, ok := StatsGroupFromContext(ctx) 277 if !ok { 278 return GlobalStats() 279 } 280 return StatsGroup(ctx, group) 281 } 282 283 // StatsGroup gets stats by group name. 284 func StatsGroup(ctx context.Context, group string) *StatsInfo { 285 stats := groups.get(group) 286 if stats == nil { 287 return NewStatsGroup(ctx, group) 288 } 289 return stats 290 } 291 292 // GlobalStats returns special stats used for global accounting. 293 func GlobalStats() *StatsInfo { 294 return StatsGroup(context.Background(), globalStats) 295 } 296 297 // NewStatsGroup creates new stats under named group. 298 func NewStatsGroup(ctx context.Context, group string) *StatsInfo { 299 stats := NewStats(ctx) 300 stats.group = group 301 groups.set(ctx, group, stats) 302 return stats 303 } 304 305 // statsGroups holds a synchronized map of stats 306 type statsGroups struct { 307 mu sync.Mutex 308 m map[string]*StatsInfo 309 order []string 310 } 311 312 // newStatsGroups makes a new statsGroups object 313 func newStatsGroups() *statsGroups { 314 return &statsGroups{ 315 m: make(map[string]*StatsInfo), 316 } 317 } 318 319 // set marks the stats as belonging to a group 320 func (sg *statsGroups) set(ctx context.Context, group string, stats *StatsInfo) { 321 sg.mu.Lock() 322 defer sg.mu.Unlock() 323 ci := fs.GetConfig(ctx) 324 325 // Limit number of groups kept in memory. 326 if len(sg.order) >= ci.MaxStatsGroups { 327 group := sg.order[0] 328 fs.LogPrintf(fs.LogLevelDebug, nil, "Max number of stats groups reached removing %s", group) 329 delete(sg.m, group) 330 r := (len(sg.order) - ci.MaxStatsGroups) + 1 331 sg.order = sg.order[r:] 332 } 333 334 // Exclude global stats from listing 335 if group != globalStats { 336 sg.order = append(sg.order, group) 337 } 338 sg.m[group] = stats 339 } 340 341 // get gets the stats for group, or nil if not found 342 func (sg *statsGroups) get(group string) *StatsInfo { 343 sg.mu.Lock() 344 defer sg.mu.Unlock() 345 stats, ok := sg.m[group] 346 if !ok { 347 return nil 348 } 349 return stats 350 } 351 352 func (sg *statsGroups) names() []string { 353 sg.mu.Lock() 354 defer sg.mu.Unlock() 355 return sg.order 356 } 357 358 // sum returns aggregate stats that contains summation of all groups. 359 func (sg *statsGroups) sum(ctx context.Context) *StatsInfo { 360 startTime := GlobalStats().startTime 361 sg.mu.Lock() 362 defer sg.mu.Unlock() 363 364 sum := NewStats(ctx) 365 for _, stats := range sg.m { 366 stats.mu.RLock() 367 { 368 sum.bytes += stats.bytes 369 sum.errors += stats.errors 370 if sum.lastError == nil && stats.lastError != nil { 371 sum.lastError = stats.lastError 372 } 373 sum.fatalError = sum.fatalError || stats.fatalError 374 sum.retryError = sum.retryError || stats.retryError 375 if stats.retryAfter.After(sum.retryAfter) { 376 // Update the retryAfter field only if it is a later date than the current one in the sum 377 sum.retryAfter = stats.retryAfter 378 } 379 sum.checks += stats.checks 380 sum.checking.merge(stats.checking) 381 sum.checkQueue += stats.checkQueue 382 sum.checkQueueSize += stats.checkQueueSize 383 sum.transfers += stats.transfers 384 sum.transferring.merge(stats.transferring) 385 sum.transferQueueSize += stats.transferQueueSize 386 sum.renames += stats.renames 387 sum.renameQueue += stats.renameQueue 388 sum.renameQueueSize += stats.renameQueueSize 389 sum.deletes += stats.deletes 390 sum.deletedDirs += stats.deletedDirs 391 sum.inProgress.merge(stats.inProgress) 392 sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...) 393 sum.oldTimeRanges = append(sum.oldTimeRanges, stats.oldTimeRanges...) 394 sum.oldDuration += stats.oldDuration 395 stats.average.mu.Lock() 396 sum.average.speed += stats.average.speed 397 stats.average.mu.Unlock() 398 } 399 stats.mu.RUnlock() 400 } 401 sum.startTime = startTime 402 return sum 403 } 404 405 func (sg *statsGroups) reset() { 406 sg.mu.Lock() 407 defer sg.mu.Unlock() 408 409 for _, stats := range sg.m { 410 stats.ResetErrors() 411 stats.ResetCounters() 412 } 413 414 sg.m = make(map[string]*StatsInfo) 415 sg.order = nil 416 } 417 418 // delete removes all references to the group. 419 func (sg *statsGroups) delete(group string) { 420 sg.mu.Lock() 421 defer sg.mu.Unlock() 422 stats := sg.m[group] 423 if stats == nil { 424 return 425 } 426 stats.ResetErrors() 427 stats.ResetCounters() 428 delete(sg.m, group) 429 430 // Remove group reference from the ordering slice. 431 tmp := sg.order[:0] 432 for _, g := range sg.order { 433 if g != group { 434 tmp = append(tmp, g) 435 } 436 } 437 sg.order = tmp 438 }