github.com/xhghs/rclone@v1.51.1-0.20200430155106-e186a28cced8/fs/accounting/stats_groups.go (about) 1 package accounting 2 3 import ( 4 "context" 5 "sync" 6 7 "github.com/rclone/rclone/fs/rc" 8 9 "github.com/rclone/rclone/fs" 10 ) 11 12 const globalStats = "global_stats" 13 14 var groups *statsGroups 15 16 func init() { 17 // Init stats container 18 groups = newStatsGroups() 19 20 // Set the function pointer up in fs 21 fs.CountError = GlobalStats().Error 22 } 23 24 func rcListStats(ctx context.Context, in rc.Params) (rc.Params, error) { 25 out := make(rc.Params) 26 27 out["groups"] = groups.names() 28 29 return out, nil 30 } 31 32 func init() { 33 rc.Add(rc.Call{ 34 Path: "core/group-list", 35 Fn: rcListStats, 36 Title: "Returns list of stats.", 37 Help: ` 38 This returns list of stats groups currently in memory. 39 40 Returns the following values: 41 ` + "```" + ` 42 { 43 "groups": an array of group names: 44 [ 45 "group1", 46 "group2", 47 ... 48 ] 49 } 50 ` + "```" + ` 51 `, 52 }) 53 } 54 55 func rcRemoteStats(ctx context.Context, in rc.Params) (rc.Params, error) { 56 // Check to see if we should filter by group. 57 group, err := in.GetString("group") 58 if rc.NotErrParamNotFound(err) { 59 return rc.Params{}, err 60 } 61 if group != "" { 62 return StatsGroup(group).RemoteStats() 63 } 64 65 return groups.sum().RemoteStats() 66 } 67 68 func init() { 69 rc.Add(rc.Call{ 70 Path: "core/stats", 71 Fn: rcRemoteStats, 72 Title: "Returns stats about current transfers.", 73 Help: ` 74 This returns all available stats: 75 76 rclone rc core/stats 77 78 If group is not provided then summed up stats for all groups will be 79 returned. 80 81 Parameters 82 83 - group - name of the stats group (string) 84 85 Returns the following values: 86 87 ` + "```" + ` 88 { 89 "speed": average speed in bytes/sec since start of the process, 90 "bytes": total transferred bytes since the start of the process, 91 "errors": number of errors, 92 "fatalError": whether there has been at least one FatalError, 93 "retryError": whether there has been at least one non-NoRetryError, 94 "checks": number of checked files, 95 "transfers": number of transferred files, 96 "deletes" : number of deleted files, 97 "elapsedTime": time in seconds since the start of the process, 98 "lastError": last occurred error, 99 "transferring": an array of currently active file transfers: 100 [ 101 { 102 "bytes": total transferred bytes for this file, 103 "eta": estimated time in seconds until file transfer completion 104 "name": name of the file, 105 "percentage": progress of the file transfer in percent, 106 "speed": speed in bytes/sec, 107 "speedAvg": speed in bytes/sec as an exponentially weighted moving average, 108 "size": size of the file in bytes 109 } 110 ], 111 "checking": an array of names of currently active file checks 112 [] 113 } 114 ` + "```" + ` 115 Values for "transferring", "checking" and "lastError" are only assigned if data is available. 116 The value for "eta" is null if an eta cannot be determined. 117 `, 118 }) 119 } 120 121 func rcTransferredStats(ctx context.Context, in rc.Params) (rc.Params, error) { 122 // Check to see if we should filter by group. 123 group, err := in.GetString("group") 124 if rc.NotErrParamNotFound(err) { 125 return rc.Params{}, err 126 } 127 128 out := make(rc.Params) 129 if group != "" { 130 out["transferred"] = StatsGroup(group).Transferred() 131 } else { 132 out["transferred"] = groups.sum().Transferred() 133 } 134 135 return out, nil 136 } 137 138 func init() { 139 rc.Add(rc.Call{ 140 Path: "core/transferred", 141 Fn: rcTransferredStats, 142 Title: "Returns stats about completed transfers.", 143 Help: ` 144 This returns stats about completed transfers: 145 146 rclone rc core/transferred 147 148 If group is not provided then completed transfers for all groups will be 149 returned. 150 151 Note only the last 100 completed transfers are returned. 152 153 Parameters 154 155 - group - name of the stats group (string) 156 157 Returns the following values: 158 ` + "```" + ` 159 { 160 "transferred": an array of completed transfers (including failed ones): 161 [ 162 { 163 "name": name of the file, 164 "size": size of the file in bytes, 165 "bytes": total transferred bytes for this file, 166 "checked": if the transfer is only checked (skipped, deleted), 167 "timestamp": integer representing millisecond unix epoch, 168 "error": string description of the error (empty if successfull), 169 "jobid": id of the job that this transfer belongs to 170 } 171 ] 172 } 173 ` + "```" + ` 174 `, 175 }) 176 } 177 178 func rcResetStats(ctx context.Context, in rc.Params) (rc.Params, error) { 179 // Check to see if we should filter by group. 180 group, err := in.GetString("group") 181 if rc.NotErrParamNotFound(err) { 182 return rc.Params{}, err 183 } 184 185 if group != "" { 186 stats := groups.get(group) 187 stats.ResetErrors() 188 stats.ResetCounters() 189 } else { 190 groups.reset() 191 } 192 193 return rc.Params{}, nil 194 } 195 196 func init() { 197 rc.Add(rc.Call{ 198 Path: "core/stats-reset", 199 Fn: rcResetStats, 200 Title: "Reset stats.", 201 Help: ` 202 This clears counters, errors and finished transfers for all stats or specific 203 stats group if group is provided. 204 205 Parameters 206 207 - group - name of the stats group (string) 208 `, 209 }) 210 } 211 212 func rcDeleteStats(ctx context.Context, in rc.Params) (rc.Params, error) { 213 // Group name required because we only do single group. 214 group, err := in.GetString("group") 215 if rc.NotErrParamNotFound(err) { 216 return rc.Params{}, err 217 } 218 219 if group != "" { 220 groups.delete(group) 221 } 222 223 return rc.Params{}, nil 224 } 225 226 func init() { 227 rc.Add(rc.Call{ 228 Path: "core/stats-delete", 229 Fn: rcDeleteStats, 230 Title: "Delete stats group.", 231 Help: ` 232 This deletes entire stats group 233 234 Parameters 235 236 - group - name of the stats group (string) 237 `, 238 }) 239 } 240 241 type statsGroupCtx int64 242 243 const statsGroupKey statsGroupCtx = 1 244 245 // WithStatsGroup returns copy of the parent context with assigned group. 246 func WithStatsGroup(parent context.Context, group string) context.Context { 247 return context.WithValue(parent, statsGroupKey, group) 248 } 249 250 // StatsGroupFromContext returns group from the context if it's available. 251 // Returns false if group is empty. 252 func StatsGroupFromContext(ctx context.Context) (string, bool) { 253 statsGroup, ok := ctx.Value(statsGroupKey).(string) 254 if statsGroup == "" { 255 ok = false 256 } 257 return statsGroup, ok 258 } 259 260 // Stats gets stats by extracting group from context. 261 func Stats(ctx context.Context) *StatsInfo { 262 group, ok := StatsGroupFromContext(ctx) 263 if !ok { 264 return GlobalStats() 265 } 266 return StatsGroup(group) 267 } 268 269 // StatsGroup gets stats by group name. 270 func StatsGroup(group string) *StatsInfo { 271 stats := groups.get(group) 272 if stats == nil { 273 return NewStatsGroup(group) 274 } 275 return stats 276 } 277 278 // GlobalStats returns special stats used for global accounting. 279 func GlobalStats() *StatsInfo { 280 return StatsGroup(globalStats) 281 } 282 283 // NewStatsGroup creates new stats under named group. 284 func NewStatsGroup(group string) *StatsInfo { 285 stats := NewStats() 286 stats.group = group 287 groups.set(group, stats) 288 return stats 289 } 290 291 // statsGroups holds a synchronized map of stats 292 type statsGroups struct { 293 mu sync.Mutex 294 m map[string]*StatsInfo 295 order []string 296 } 297 298 // newStatsGroups makes a new statsGroups object 299 func newStatsGroups() *statsGroups { 300 return &statsGroups{ 301 m: make(map[string]*StatsInfo), 302 } 303 } 304 305 // set marks the stats as belonging to a group 306 func (sg *statsGroups) set(group string, stats *StatsInfo) { 307 sg.mu.Lock() 308 defer sg.mu.Unlock() 309 310 // Limit number of groups kept in memory. 311 if len(sg.order) >= fs.Config.MaxStatsGroups { 312 group := sg.order[0] 313 //fs.LogPrintf(fs.LogLevelInfo, nil, "Max number of stats groups reached removing %s", group) 314 delete(sg.m, group) 315 r := (len(sg.order) - fs.Config.MaxStatsGroups) + 1 316 sg.order = sg.order[r:] 317 } 318 319 // Exclude global stats from listing 320 if group != globalStats { 321 sg.order = append(sg.order, group) 322 } 323 sg.m[group] = stats 324 } 325 326 // get gets the stats for group, or nil if not found 327 func (sg *statsGroups) get(group string) *StatsInfo { 328 sg.mu.Lock() 329 defer sg.mu.Unlock() 330 stats, ok := sg.m[group] 331 if !ok { 332 return nil 333 } 334 return stats 335 } 336 337 func (sg *statsGroups) names() []string { 338 sg.mu.Lock() 339 defer sg.mu.Unlock() 340 return sg.order 341 } 342 343 // sum returns aggregate stats that contains summation of all groups. 344 func (sg *statsGroups) sum() *StatsInfo { 345 sg.mu.Lock() 346 defer sg.mu.Unlock() 347 348 sum := NewStats() 349 for _, stats := range sg.m { 350 stats.mu.RLock() 351 { 352 sum.bytes += stats.bytes 353 sum.errors += stats.errors 354 sum.fatalError = sum.fatalError || stats.fatalError 355 sum.retryError = sum.retryError || stats.retryError 356 sum.checks += stats.checks 357 sum.transfers += stats.transfers 358 sum.deletes += stats.deletes 359 sum.checking.merge(stats.checking) 360 sum.transferring.merge(stats.transferring) 361 sum.inProgress.merge(stats.inProgress) 362 if sum.lastError == nil && stats.lastError != nil { 363 sum.lastError = stats.lastError 364 } 365 sum.startedTransfers = append(sum.startedTransfers, stats.startedTransfers...) 366 } 367 stats.mu.RUnlock() 368 } 369 return sum 370 } 371 372 func (sg *statsGroups) reset() { 373 sg.mu.Lock() 374 defer sg.mu.Unlock() 375 376 for _, stats := range sg.m { 377 stats.ResetErrors() 378 stats.ResetCounters() 379 } 380 381 sg.m = make(map[string]*StatsInfo) 382 sg.order = nil 383 } 384 385 // delete removes all references to the group. 386 func (sg *statsGroups) delete(group string) { 387 sg.mu.Lock() 388 defer sg.mu.Unlock() 389 stats := sg.m[group] 390 if stats == nil { 391 return 392 } 393 stats.ResetErrors() 394 stats.ResetCounters() 395 delete(sg.m, group) 396 397 // Remove group reference from the ordering slice. 398 tmp := sg.order[:0] 399 for _, g := range sg.order { 400 if g != group { 401 tmp = append(tmp, g) 402 } 403 } 404 sg.order = tmp 405 }