github.com/divyam234/rclone@v1.64.1/fs/accounting/stats_test.go (about) 1 package accounting 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "testing" 8 "time" 9 10 "github.com/divyam234/rclone/fs" 11 "github.com/divyam234/rclone/fs/fserrors" 12 "github.com/stretchr/testify/assert" 13 "github.com/stretchr/testify/require" 14 ) 15 16 func TestETA(t *testing.T) { 17 for _, test := range []struct { 18 size, total int64 19 rate float64 20 wantETA time.Duration 21 wantOK bool 22 wantString string 23 }{ 24 // Custom String Cases 25 {size: 0, total: 365 * 86400, rate: 1.0, wantETA: 365 * 86400 * time.Second, wantOK: true, wantString: "1y"}, 26 {size: 0, total: 7 * 86400, rate: 1.0, wantETA: 7 * 86400 * time.Second, wantOK: true, wantString: "1w"}, 27 {size: 0, total: 1 * 86400, rate: 1.0, wantETA: 1 * 86400 * time.Second, wantOK: true, wantString: "1d"}, 28 {size: 0, total: 1110 * 86400, rate: 1.0, wantETA: 1110 * 86400 * time.Second, wantOK: true, wantString: "3y2w1d"}, 29 {size: 0, total: 15 * 86400, rate: 1.0, wantETA: 15 * 86400 * time.Second, wantOK: true, wantString: "2w1d"}, 30 // Composite Custom String Cases 31 {size: 0, total: 1.5 * 86400, rate: 1.0, wantETA: 1.5 * 86400 * time.Second, wantOK: true, wantString: "1d12h"}, 32 {size: 0, total: 95000, rate: 1.0, wantETA: 95000 * time.Second, wantOK: true, wantString: "1d2h23m"}, // Short format, if full it would be "1d2h23m20s" 33 // Standard Duration String Cases 34 {size: 0, total: 1, rate: 2.0, wantETA: 0, wantOK: true, wantString: "0s"}, 35 {size: 0, total: 1, rate: 1.0, wantETA: time.Second, wantOK: true, wantString: "1s"}, 36 {size: 0, total: 1, rate: 0.5, wantETA: 2 * time.Second, wantOK: true, wantString: "2s"}, 37 {size: 0, total: 100, rate: 1.0, wantETA: 100 * time.Second, wantOK: true, wantString: "1m40s"}, 38 {size: 50, total: 100, rate: 1.0, wantETA: 50 * time.Second, wantOK: true, wantString: "50s"}, 39 {size: 100, total: 100, rate: 1.0, wantETA: 0 * time.Second, wantOK: true, wantString: "0s"}, 40 // No String Cases 41 {size: -1, total: 100, rate: 1.0, wantETA: 0, wantOK: false, wantString: "-"}, 42 {size: 200, total: 100, rate: 1.0, wantETA: 0, wantOK: false, wantString: "-"}, 43 {size: 10, total: -1, rate: 1.0, wantETA: 0, wantOK: false, wantString: "-"}, 44 {size: 10, total: 20, rate: 0.0, wantETA: 0, wantOK: false, wantString: "-"}, 45 {size: 10, total: 20, rate: -1.0, wantETA: 0, wantOK: false, wantString: "-"}, 46 {size: 0, total: 0, rate: 1.0, wantETA: 0, wantOK: false, wantString: "-"}, 47 // Extreme Cases 48 {size: 0, total: (1 << 63) - 1, rate: 1.0, wantETA: (time.Duration((1<<63)-1) / time.Second) * time.Second, wantOK: true, wantString: "-"}, 49 {size: 0, total: ((1 << 63) - 1) / int64(time.Second), rate: 1.0, wantETA: (time.Duration((1<<63)-1) / time.Second) * time.Second, wantOK: true, wantString: "-"}, 50 {size: 0, total: ((1<<63)-1)/int64(time.Second) - 1, rate: 1.0, wantETA: (time.Duration((1<<63)-1)/time.Second - 1) * time.Second, wantOK: true, wantString: "292y24w3d"}, // Short format, if full it would be "292y24w3d23h47m15s" 51 {size: 0, total: ((1<<63)-1)/int64(time.Second) - 1, rate: 0.1, wantETA: (time.Duration((1<<63)-1) / time.Second) * time.Second, wantOK: true, wantString: "-"}, 52 } { 53 t.Run(fmt.Sprintf("size=%d/total=%d/rate=%f", test.size, test.total, test.rate), func(t *testing.T) { 54 gotETA, gotOK := eta(test.size, test.total, test.rate) 55 assert.Equal(t, int64(test.wantETA), int64(gotETA)) 56 assert.Equal(t, test.wantOK, gotOK) 57 gotString := etaString(test.size, test.total, test.rate) 58 assert.Equal(t, test.wantString, gotString) 59 }) 60 } 61 } 62 63 func TestPercentage(t *testing.T) { 64 assert.Equal(t, percent(0, 1000), "0%") 65 assert.Equal(t, percent(1, 1000), "0%") 66 assert.Equal(t, percent(9, 1000), "1%") 67 assert.Equal(t, percent(500, 1000), "50%") 68 assert.Equal(t, percent(1000, 1000), "100%") 69 assert.Equal(t, percent(1e8, 1e9), "10%") 70 assert.Equal(t, percent(1e8, 1e9), "10%") 71 assert.Equal(t, percent(0, 0), "-") 72 assert.Equal(t, percent(100, -100), "-") 73 assert.Equal(t, percent(-100, 100), "-") 74 assert.Equal(t, percent(-100, -100), "-") 75 } 76 77 func TestStatsError(t *testing.T) { 78 ctx := context.Background() 79 s := NewStats(ctx) 80 assert.Equal(t, int64(0), s.GetErrors()) 81 assert.False(t, s.HadFatalError()) 82 assert.False(t, s.HadRetryError()) 83 assert.Equal(t, time.Time{}, s.RetryAfter()) 84 assert.Equal(t, nil, s.GetLastError()) 85 assert.False(t, s.Errored()) 86 87 t0 := time.Now() 88 t1 := t0.Add(time.Second) 89 90 _ = s.Error(nil) 91 assert.Equal(t, int64(0), s.GetErrors()) 92 assert.False(t, s.HadFatalError()) 93 assert.False(t, s.HadRetryError()) 94 assert.Equal(t, time.Time{}, s.RetryAfter()) 95 assert.Equal(t, nil, s.GetLastError()) 96 assert.False(t, s.Errored()) 97 98 _ = s.Error(io.EOF) 99 assert.Equal(t, int64(1), s.GetErrors()) 100 assert.False(t, s.HadFatalError()) 101 assert.True(t, s.HadRetryError()) 102 assert.Equal(t, time.Time{}, s.RetryAfter()) 103 assert.Equal(t, io.EOF, s.GetLastError()) 104 assert.True(t, s.Errored()) 105 106 e := fserrors.ErrorRetryAfter(t0) 107 _ = s.Error(e) 108 assert.Equal(t, int64(2), s.GetErrors()) 109 assert.False(t, s.HadFatalError()) 110 assert.True(t, s.HadRetryError()) 111 assert.Equal(t, t0, s.RetryAfter()) 112 assert.Equal(t, e, s.GetLastError()) 113 114 err := fmt.Errorf("potato: %w", fserrors.ErrorRetryAfter(t1)) 115 err = s.Error(err) 116 assert.Equal(t, int64(3), s.GetErrors()) 117 assert.False(t, s.HadFatalError()) 118 assert.True(t, s.HadRetryError()) 119 assert.Equal(t, t1, s.RetryAfter()) 120 assert.Equal(t, t1, fserrors.RetryAfterErrorTime(err)) 121 122 _ = s.Error(fserrors.FatalError(io.EOF)) 123 assert.Equal(t, int64(4), s.GetErrors()) 124 assert.True(t, s.HadFatalError()) 125 assert.True(t, s.HadRetryError()) 126 assert.Equal(t, t1, s.RetryAfter()) 127 128 s.ResetErrors() 129 assert.Equal(t, int64(0), s.GetErrors()) 130 assert.False(t, s.HadFatalError()) 131 assert.False(t, s.HadRetryError()) 132 assert.Equal(t, time.Time{}, s.RetryAfter()) 133 assert.Equal(t, nil, s.GetLastError()) 134 assert.False(t, s.Errored()) 135 136 _ = s.Error(fserrors.NoRetryError(io.EOF)) 137 assert.Equal(t, int64(1), s.GetErrors()) 138 assert.False(t, s.HadFatalError()) 139 assert.False(t, s.HadRetryError()) 140 assert.Equal(t, time.Time{}, s.RetryAfter()) 141 } 142 143 func TestStatsTotalDuration(t *testing.T) { 144 ctx := context.Background() 145 startTime := time.Now() 146 time1 := startTime.Add(-40 * time.Second) 147 time2 := time1.Add(10 * time.Second) 148 time3 := time2.Add(10 * time.Second) 149 time4 := time3.Add(10 * time.Second) 150 151 t.Run("Single completed transfer", func(t *testing.T) { 152 s := NewStats(ctx) 153 tr1 := &Transfer{ 154 startedAt: time1, 155 completedAt: time2, 156 } 157 s.AddTransfer(tr1) 158 159 s.mu.Lock() 160 total := s.totalDuration() 161 s.mu.Unlock() 162 163 assert.Equal(t, 1, len(s.startedTransfers)) 164 assert.Equal(t, 10*time.Second, total) 165 s.RemoveTransfer(tr1) 166 assert.Equal(t, 10*time.Second, total) 167 assert.Equal(t, 0, len(s.startedTransfers)) 168 }) 169 170 t.Run("Single uncompleted transfer", func(t *testing.T) { 171 s := NewStats(ctx) 172 tr1 := &Transfer{ 173 startedAt: time1, 174 } 175 s.AddTransfer(tr1) 176 177 s.mu.Lock() 178 total := s.totalDuration() 179 s.mu.Unlock() 180 181 assert.Equal(t, time.Since(time1)/time.Second, total/time.Second) 182 s.RemoveTransfer(tr1) 183 assert.Equal(t, time.Since(time1)/time.Second, total/time.Second) 184 }) 185 186 t.Run("Overlapping without ending", func(t *testing.T) { 187 s := NewStats(ctx) 188 tr1 := &Transfer{ 189 startedAt: time2, 190 completedAt: time3, 191 } 192 s.AddTransfer(tr1) 193 tr2 := &Transfer{ 194 startedAt: time2, 195 completedAt: time2.Add(time.Second), 196 } 197 s.AddTransfer(tr2) 198 tr3 := &Transfer{ 199 startedAt: time1, 200 completedAt: time3, 201 } 202 s.AddTransfer(tr3) 203 tr4 := &Transfer{ 204 startedAt: time3, 205 completedAt: time4, 206 } 207 s.AddTransfer(tr4) 208 tr5 := &Transfer{ 209 startedAt: time.Now(), 210 } 211 s.AddTransfer(tr5) 212 213 time.Sleep(time.Millisecond) 214 215 s.mu.Lock() 216 total := s.totalDuration() 217 s.mu.Unlock() 218 219 assert.Equal(t, time.Duration(30), total/time.Second) 220 s.RemoveTransfer(tr1) 221 assert.Equal(t, time.Duration(30), total/time.Second) 222 s.RemoveTransfer(tr2) 223 assert.Equal(t, time.Duration(30), total/time.Second) 224 s.RemoveTransfer(tr3) 225 assert.Equal(t, time.Duration(30), total/time.Second) 226 s.RemoveTransfer(tr4) 227 assert.Equal(t, time.Duration(30), total/time.Second) 228 }) 229 230 t.Run("Mixed completed and uncompleted transfers", func(t *testing.T) { 231 s := NewStats(ctx) 232 s.AddTransfer(&Transfer{ 233 startedAt: time1, 234 completedAt: time2, 235 }) 236 s.AddTransfer(&Transfer{ 237 startedAt: time2, 238 }) 239 s.AddTransfer(&Transfer{ 240 startedAt: time3, 241 }) 242 s.AddTransfer(&Transfer{ 243 startedAt: time3, 244 }) 245 246 s.mu.Lock() 247 total := s.totalDuration() 248 s.mu.Unlock() 249 250 assert.Equal(t, startTime.Sub(time1)/time.Second, total/time.Second) 251 }) 252 } 253 254 func TestRemoteStats(t *testing.T) { 255 ctx := context.Background() 256 startTime := time.Now() 257 time1 := startTime.Add(-40 * time.Second) 258 time2 := time1.Add(10 * time.Second) 259 260 t.Run("Single completed transfer", func(t *testing.T) { 261 s := NewStats(ctx) 262 tr1 := &Transfer{ 263 startedAt: time1, 264 completedAt: time2, 265 } 266 s.AddTransfer(tr1) 267 time.Sleep(time.Millisecond) 268 rs, err := s.RemoteStats() 269 270 require.NoError(t, err) 271 assert.Equal(t, float64(10), rs["transferTime"]) 272 assert.Greater(t, rs["elapsedTime"], float64(0)) 273 }) 274 } 275 276 // make time ranges from string description for testing 277 func makeTimeRanges(t *testing.T, in []string) timeRanges { 278 trs := make(timeRanges, len(in)) 279 for i, Range := range in { 280 var start, end int64 281 n, err := fmt.Sscanf(Range, "%d-%d", &start, &end) 282 require.NoError(t, err) 283 require.Equal(t, 2, n) 284 trs[i] = timeRange{time.Unix(start, 0), time.Unix(end, 0)} 285 } 286 return trs 287 } 288 289 func (trs timeRanges) toStrings() (out []string) { 290 out = []string{} 291 for _, tr := range trs { 292 out = append(out, fmt.Sprintf("%d-%d", tr.start.Unix(), tr.end.Unix())) 293 } 294 return out 295 } 296 297 func TestTimeRangeMerge(t *testing.T) { 298 for _, test := range []struct { 299 in []string 300 want []string 301 }{{ 302 in: []string{}, 303 want: []string{}, 304 }, { 305 in: []string{"1-2"}, 306 want: []string{"1-2"}, 307 }, { 308 in: []string{"1-4", "2-3"}, 309 want: []string{"1-4"}, 310 }, { 311 in: []string{"2-3", "1-4"}, 312 want: []string{"1-4"}, 313 }, { 314 in: []string{"1-3", "2-4"}, 315 want: []string{"1-4"}, 316 }, { 317 in: []string{"2-4", "1-3"}, 318 want: []string{"1-4"}, 319 }, { 320 in: []string{"1-2", "2-3"}, 321 want: []string{"1-3"}, 322 }, { 323 in: []string{"2-3", "1-2"}, 324 want: []string{"1-3"}, 325 }, { 326 in: []string{"1-2", "3-4"}, 327 want: []string{"1-2", "3-4"}, 328 }, { 329 in: []string{"1-3", "7-8", "4-6", "2-5", "7-8", "7-8"}, 330 want: []string{"1-6", "7-8"}, 331 }} { 332 333 in := makeTimeRanges(t, test.in) 334 in.merge() 335 336 got := in.toStrings() 337 assert.Equal(t, test.want, got) 338 } 339 } 340 341 func TestTimeRangeCull(t *testing.T) { 342 for _, test := range []struct { 343 in []string 344 cutoff int64 345 want []string 346 wantDuration time.Duration 347 }{{ 348 in: []string{}, 349 cutoff: 1, 350 want: []string{}, 351 wantDuration: 0 * time.Second, 352 }, { 353 in: []string{"1-2"}, 354 cutoff: 1, 355 want: []string{"1-2"}, 356 wantDuration: 0 * time.Second, 357 }, { 358 in: []string{"2-5", "7-9"}, 359 cutoff: 1, 360 want: []string{"2-5", "7-9"}, 361 wantDuration: 0 * time.Second, 362 }, { 363 in: []string{"2-5", "7-9"}, 364 cutoff: 4, 365 want: []string{"2-5", "7-9"}, 366 wantDuration: 0 * time.Second, 367 }, { 368 in: []string{"2-5", "7-9"}, 369 cutoff: 5, 370 want: []string{"7-9"}, 371 wantDuration: 3 * time.Second, 372 }, { 373 in: []string{"2-5", "7-9", "2-5", "2-5"}, 374 cutoff: 6, 375 want: []string{"7-9"}, 376 wantDuration: 9 * time.Second, 377 }, { 378 in: []string{"7-9", "3-3", "2-5"}, 379 cutoff: 7, 380 want: []string{"7-9"}, 381 wantDuration: 3 * time.Second, 382 }, { 383 in: []string{"2-5", "7-9"}, 384 cutoff: 8, 385 want: []string{"7-9"}, 386 wantDuration: 3 * time.Second, 387 }, { 388 in: []string{"2-5", "7-9"}, 389 cutoff: 9, 390 want: []string{}, 391 wantDuration: 5 * time.Second, 392 }, { 393 in: []string{"2-5", "7-9"}, 394 cutoff: 10, 395 want: []string{}, 396 wantDuration: 5 * time.Second, 397 }} { 398 399 in := makeTimeRanges(t, test.in) 400 cutoff := time.Unix(test.cutoff, 0) 401 gotDuration := in.cull(cutoff) 402 403 what := fmt.Sprintf("in=%q, cutoff=%d", test.in, test.cutoff) 404 got := in.toStrings() 405 assert.Equal(t, test.want, got, what) 406 assert.Equal(t, test.wantDuration, gotDuration, what) 407 } 408 } 409 410 func TestTimeRangeDuration(t *testing.T) { 411 assert.Equal(t, 0*time.Second, timeRanges{}.total()) 412 assert.Equal(t, 1*time.Second, makeTimeRanges(t, []string{"1-2"}).total()) 413 assert.Equal(t, 91*time.Second, makeTimeRanges(t, []string{"1-2", "10-100"}).total()) 414 } 415 416 func TestPruneTransfers(t *testing.T) { 417 ctx := context.Background() 418 ci := fs.GetConfig(ctx) 419 for _, test := range []struct { 420 Name string 421 Transfers int 422 Limit int 423 ExpectedStartedTransfers int 424 }{ 425 { 426 Name: "Limited number of StartedTransfers", 427 Limit: 100, 428 Transfers: 200, 429 ExpectedStartedTransfers: 100 + ci.Transfers, 430 }, 431 { 432 Name: "Unlimited number of StartedTransfers", 433 Limit: -1, 434 Transfers: 200, 435 ExpectedStartedTransfers: 200, 436 }, 437 } { 438 t.Run(test.Name, func(t *testing.T) { 439 prevLimit := MaxCompletedTransfers 440 MaxCompletedTransfers = test.Limit 441 defer func() { MaxCompletedTransfers = prevLimit }() 442 443 s := NewStats(ctx) 444 for i := int64(1); i <= int64(test.Transfers); i++ { 445 s.AddTransfer(&Transfer{ 446 startedAt: time.Unix(i, 0), 447 completedAt: time.Unix(i+1, 0), 448 }) 449 } 450 451 s.mu.Lock() 452 assert.Equal(t, time.Duration(test.Transfers)*time.Second, s.totalDuration()) 453 assert.Equal(t, test.Transfers, len(s.startedTransfers)) 454 s.mu.Unlock() 455 456 for i := 0; i < test.Transfers; i++ { 457 s.PruneTransfers() 458 } 459 460 s.mu.Lock() 461 assert.Equal(t, time.Duration(test.Transfers)*time.Second, s.totalDuration()) 462 assert.Equal(t, test.ExpectedStartedTransfers, len(s.startedTransfers)) 463 s.mu.Unlock() 464 465 }) 466 } 467 }