github.com/divyam234/rclone@v1.64.1/fs/march/march_test.go (about) 1 // Internal tests for march 2 3 package march 4 5 import ( 6 "context" 7 "errors" 8 "fmt" 9 "strings" 10 "sync" 11 "testing" 12 13 _ "github.com/divyam234/rclone/backend/local" 14 "github.com/divyam234/rclone/fs" 15 "github.com/divyam234/rclone/fs/filter" 16 "github.com/divyam234/rclone/fs/fserrors" 17 "github.com/divyam234/rclone/fstest" 18 "github.com/divyam234/rclone/fstest/mockdir" 19 "github.com/divyam234/rclone/fstest/mockobject" 20 "github.com/stretchr/testify/assert" 21 "github.com/stretchr/testify/require" 22 "golang.org/x/text/unicode/norm" 23 ) 24 25 // Some times used in the tests 26 var ( 27 t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") 28 ) 29 30 func TestMain(m *testing.M) { 31 fstest.TestMain(m) 32 } 33 34 type marchTester struct { 35 ctx context.Context // internal context for controlling go-routines 36 cancel func() // cancel the context 37 srcOnly fs.DirEntries 38 dstOnly fs.DirEntries 39 match fs.DirEntries 40 entryMutex sync.Mutex 41 errorMu sync.Mutex // Mutex covering the error variables 42 err error 43 noRetryErr error 44 fatalErr error 45 noTraverse bool 46 } 47 48 // DstOnly have an object which is in the destination only 49 func (mt *marchTester) DstOnly(dst fs.DirEntry) (recurse bool) { 50 mt.entryMutex.Lock() 51 mt.dstOnly = append(mt.dstOnly, dst) 52 mt.entryMutex.Unlock() 53 54 switch dst.(type) { 55 case fs.Object: 56 return false 57 case fs.Directory: 58 return true 59 default: 60 panic("Bad object in DirEntries") 61 } 62 } 63 64 // SrcOnly have an object which is in the source only 65 func (mt *marchTester) SrcOnly(src fs.DirEntry) (recurse bool) { 66 mt.entryMutex.Lock() 67 mt.srcOnly = append(mt.srcOnly, src) 68 mt.entryMutex.Unlock() 69 70 switch src.(type) { 71 case fs.Object: 72 return false 73 case fs.Directory: 74 return true 75 default: 76 panic("Bad object in DirEntries") 77 } 78 } 79 80 // Match is called when src and dst are present, so sync src to dst 81 func (mt *marchTester) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) { 82 mt.entryMutex.Lock() 83 mt.match = append(mt.match, src) 84 mt.entryMutex.Unlock() 85 86 switch src.(type) { 87 case fs.Object: 88 return false 89 case fs.Directory: 90 // Do the same thing to the entire contents of the directory 91 _, ok := dst.(fs.Directory) 92 if ok { 93 return true 94 } 95 // FIXME src is dir, dst is file 96 err := errors.New("can't overwrite file with directory") 97 fs.Errorf(dst, "%v", err) 98 mt.processError(err) 99 default: 100 panic("Bad object in DirEntries") 101 } 102 return false 103 } 104 105 func (mt *marchTester) processError(err error) { 106 if err == nil { 107 return 108 } 109 mt.errorMu.Lock() 110 defer mt.errorMu.Unlock() 111 switch { 112 case fserrors.IsFatalError(err): 113 if !mt.aborting() { 114 fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err) 115 mt.cancel() 116 } 117 mt.fatalErr = err 118 case fserrors.IsNoRetryError(err): 119 mt.noRetryErr = err 120 default: 121 mt.err = err 122 } 123 } 124 125 func (mt *marchTester) currentError() error { 126 mt.errorMu.Lock() 127 defer mt.errorMu.Unlock() 128 if mt.fatalErr != nil { 129 return mt.fatalErr 130 } 131 if mt.err != nil { 132 return mt.err 133 } 134 return mt.noRetryErr 135 } 136 137 func (mt *marchTester) aborting() bool { 138 return mt.ctx.Err() != nil 139 } 140 141 func TestMarch(t *testing.T) { 142 for _, test := range []struct { 143 what string 144 fileSrcOnly []string 145 dirSrcOnly []string 146 fileDstOnly []string 147 dirDstOnly []string 148 fileMatch []string 149 dirMatch []string 150 }{ 151 { 152 what: "source only", 153 fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"}, 154 dirSrcOnly: []string{"sub dir"}, 155 }, 156 { 157 what: "identical", 158 fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"}, 159 dirMatch: []string{"sub dir", "sub dir/sub sub dir"}, 160 }, 161 { 162 what: "typical sync", 163 fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"}, 164 dirSrcOnly: []string{"srcOnlyDir"}, 165 fileMatch: []string{"match", "matchDir/match file"}, 166 dirMatch: []string{"matchDir"}, 167 fileDstOnly: []string{"dstOnly", "dstOnlyDir/sub"}, 168 dirDstOnly: []string{"dstOnlyDir"}, 169 }, 170 } { 171 t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) { 172 r := fstest.NewRun(t) 173 174 var srcOnly []fstest.Item 175 var dstOnly []fstest.Item 176 var match []fstest.Item 177 178 ctx, cancel := context.WithCancel(context.Background()) 179 180 for _, f := range test.fileSrcOnly { 181 srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1)) 182 } 183 for _, f := range test.fileDstOnly { 184 dstOnly = append(dstOnly, r.WriteObject(ctx, f, "hello world", t1)) 185 } 186 for _, f := range test.fileMatch { 187 match = append(match, r.WriteBoth(ctx, f, "hello world", t1)) 188 } 189 190 mt := &marchTester{ 191 ctx: ctx, 192 cancel: cancel, 193 noTraverse: false, 194 } 195 fi := filter.GetConfig(ctx) 196 m := &March{ 197 Ctx: ctx, 198 Fdst: r.Fremote, 199 Fsrc: r.Flocal, 200 Dir: "", 201 NoTraverse: mt.noTraverse, 202 Callback: mt, 203 DstIncludeAll: fi.Opt.DeleteExcluded, 204 } 205 206 mt.processError(m.Run(ctx)) 207 mt.cancel() 208 err := mt.currentError() 209 require.NoError(t, err) 210 211 precision := fs.GetModifyWindow(ctx, r.Fremote, r.Flocal) 212 fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly") 213 fstest.CompareItems(t, mt.dstOnly, dstOnly, test.dirDstOnly, precision, "dstOnly") 214 fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match") 215 }) 216 } 217 } 218 219 func TestMarchNoTraverse(t *testing.T) { 220 for _, test := range []struct { 221 what string 222 fileSrcOnly []string 223 dirSrcOnly []string 224 fileMatch []string 225 dirMatch []string 226 }{ 227 { 228 what: "source only", 229 fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"}, 230 dirSrcOnly: []string{"sub dir"}, 231 }, 232 { 233 what: "identical", 234 fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"}, 235 }, 236 { 237 what: "typical sync", 238 fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"}, 239 fileMatch: []string{"match", "matchDir/match file"}, 240 }, 241 } { 242 t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) { 243 r := fstest.NewRun(t) 244 245 var srcOnly []fstest.Item 246 var match []fstest.Item 247 248 ctx, cancel := context.WithCancel(context.Background()) 249 250 for _, f := range test.fileSrcOnly { 251 srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1)) 252 } 253 for _, f := range test.fileMatch { 254 match = append(match, r.WriteBoth(ctx, f, "hello world", t1)) 255 } 256 257 mt := &marchTester{ 258 ctx: ctx, 259 cancel: cancel, 260 noTraverse: true, 261 } 262 fi := filter.GetConfig(ctx) 263 m := &March{ 264 Ctx: ctx, 265 Fdst: r.Fremote, 266 Fsrc: r.Flocal, 267 Dir: "", 268 NoTraverse: mt.noTraverse, 269 Callback: mt, 270 DstIncludeAll: fi.Opt.DeleteExcluded, 271 } 272 273 mt.processError(m.Run(ctx)) 274 mt.cancel() 275 err := mt.currentError() 276 require.NoError(t, err) 277 278 precision := fs.GetModifyWindow(ctx, r.Fremote, r.Flocal) 279 fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly") 280 fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match") 281 }) 282 } 283 } 284 285 func TestNewMatchEntries(t *testing.T) { 286 var ( 287 a = mockobject.Object("path/a") 288 A = mockobject.Object("path/A") 289 B = mockobject.Object("path/B") 290 c = mockobject.Object("path/c") 291 ) 292 293 es := newMatchEntries(fs.DirEntries{a, A, B, c}, nil) 294 assert.Equal(t, es, matchEntries{ 295 {name: "A", leaf: "A", entry: A}, 296 {name: "B", leaf: "B", entry: B}, 297 {name: "a", leaf: "a", entry: a}, 298 {name: "c", leaf: "c", entry: c}, 299 }) 300 301 es = newMatchEntries(fs.DirEntries{a, A, B, c}, []matchTransformFn{strings.ToLower}) 302 assert.Equal(t, es, matchEntries{ 303 {name: "a", leaf: "A", entry: A}, 304 {name: "a", leaf: "a", entry: a}, 305 {name: "b", leaf: "B", entry: B}, 306 {name: "c", leaf: "c", entry: c}, 307 }) 308 } 309 310 func TestMatchListings(t *testing.T) { 311 var ( 312 a = mockobject.Object("a") 313 A = mockobject.Object("A") 314 b = mockobject.Object("b") 315 c = mockobject.Object("c") 316 d = mockobject.Object("d") 317 uE1 = mockobject.Object("é") // one of the unicode E characters 318 uE2 = mockobject.Object("é") // a different unicode E character 319 dirA = mockdir.New("A") 320 dirb = mockdir.New("b") 321 ) 322 323 for _, test := range []struct { 324 what string 325 input fs.DirEntries // pairs of input src, dst 326 srcOnly fs.DirEntries 327 dstOnly fs.DirEntries 328 matches []matchPair // pairs of output 329 transforms []matchTransformFn 330 }{ 331 { 332 what: "only src or dst", 333 input: fs.DirEntries{ 334 a, nil, 335 b, nil, 336 c, nil, 337 d, nil, 338 }, 339 srcOnly: fs.DirEntries{ 340 a, b, c, d, 341 }, 342 }, 343 { 344 what: "typical sync #1", 345 input: fs.DirEntries{ 346 a, nil, 347 b, b, 348 nil, c, 349 nil, d, 350 }, 351 srcOnly: fs.DirEntries{ 352 a, 353 }, 354 dstOnly: fs.DirEntries{ 355 c, d, 356 }, 357 matches: []matchPair{ 358 {b, b}, 359 }, 360 }, 361 { 362 what: "typical sync #2", 363 input: fs.DirEntries{ 364 a, a, 365 b, b, 366 nil, c, 367 d, d, 368 }, 369 dstOnly: fs.DirEntries{ 370 c, 371 }, 372 matches: []matchPair{ 373 {a, a}, 374 {b, b}, 375 {d, d}, 376 }, 377 }, 378 { 379 what: "One duplicate", 380 input: fs.DirEntries{ 381 A, A, 382 a, a, 383 a, nil, 384 b, b, 385 }, 386 matches: []matchPair{ 387 {A, A}, 388 {a, a}, 389 {b, b}, 390 }, 391 }, 392 { 393 what: "Two duplicates", 394 input: fs.DirEntries{ 395 a, a, 396 a, a, 397 a, nil, 398 }, 399 matches: []matchPair{ 400 {a, a}, 401 }, 402 }, 403 { 404 what: "Case insensitive duplicate - no transform", 405 input: fs.DirEntries{ 406 a, a, 407 A, A, 408 }, 409 matches: []matchPair{ 410 {A, A}, 411 {a, a}, 412 }, 413 }, 414 { 415 what: "Case insensitive duplicate - transform to lower case", 416 input: fs.DirEntries{ 417 a, a, 418 A, A, 419 }, 420 matches: []matchPair{ 421 {A, A}, 422 }, 423 transforms: []matchTransformFn{strings.ToLower}, 424 }, 425 { 426 what: "Unicode near-duplicate that becomes duplicate with normalization", 427 input: fs.DirEntries{ 428 uE1, uE1, 429 uE2, uE2, 430 }, 431 matches: []matchPair{ 432 {uE1, uE1}, 433 }, 434 transforms: []matchTransformFn{norm.NFC.String}, 435 }, 436 { 437 what: "Unicode near-duplicate with no normalization", 438 input: fs.DirEntries{ 439 uE1, uE1, 440 uE2, uE2, 441 }, 442 matches: []matchPair{ 443 {uE1, uE1}, 444 {uE2, uE2}, 445 }, 446 }, 447 { 448 what: "File and directory are not duplicates - srcOnly", 449 input: fs.DirEntries{ 450 dirA, nil, 451 A, nil, 452 }, 453 srcOnly: fs.DirEntries{ 454 dirA, 455 A, 456 }, 457 }, 458 { 459 what: "File and directory are not duplicates - matches", 460 input: fs.DirEntries{ 461 dirA, dirA, 462 A, A, 463 }, 464 matches: []matchPair{ 465 {dirA, dirA}, 466 {A, A}, 467 }, 468 }, 469 { 470 what: "Sync with directory #1", 471 input: fs.DirEntries{ 472 dirA, nil, 473 A, nil, 474 b, b, 475 nil, c, 476 nil, d, 477 }, 478 srcOnly: fs.DirEntries{ 479 dirA, 480 A, 481 }, 482 dstOnly: fs.DirEntries{ 483 c, d, 484 }, 485 matches: []matchPair{ 486 {b, b}, 487 }, 488 }, 489 { 490 what: "Sync with 2 directories", 491 input: fs.DirEntries{ 492 dirA, dirA, 493 A, nil, 494 nil, dirb, 495 nil, b, 496 }, 497 srcOnly: fs.DirEntries{ 498 A, 499 }, 500 dstOnly: fs.DirEntries{ 501 dirb, 502 b, 503 }, 504 matches: []matchPair{ 505 {dirA, dirA}, 506 }, 507 }, 508 } { 509 t.Run(fmt.Sprintf("TestMatchListings-%s", test.what), func(t *testing.T) { 510 var srcList, dstList fs.DirEntries 511 for i := 0; i < len(test.input); i += 2 { 512 src, dst := test.input[i], test.input[i+1] 513 if src != nil { 514 srcList = append(srcList, src) 515 } 516 if dst != nil { 517 dstList = append(dstList, dst) 518 } 519 } 520 srcOnly, dstOnly, matches := matchListings(srcList, dstList, test.transforms) 521 assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ") 522 assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ") 523 assert.Equal(t, test.matches, matches, test.what, "matches differ") 524 // now swap src and dst 525 dstOnly, srcOnly, matches = matchListings(dstList, srcList, test.transforms) 526 assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ") 527 assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ") 528 assert.Equal(t, test.matches, matches, test.what, "matches differ") 529 }) 530 } 531 }