github.com/xhghs/rclone@v1.51.1-0.20200430155106-e186a28cced8/fs/march/march_test.go (about) 1 // Internal tests for march 2 3 package march 4 5 import ( 6 "context" 7 "errors" 8 "fmt" 9 "strings" 10 "sync" 11 "testing" 12 13 _ "github.com/rclone/rclone/backend/local" 14 "github.com/rclone/rclone/fs" 15 "github.com/rclone/rclone/fs/filter" 16 "github.com/rclone/rclone/fs/fserrors" 17 "github.com/rclone/rclone/fstest" 18 "github.com/rclone/rclone/fstest/mockdir" 19 "github.com/rclone/rclone/fstest/mockobject" 20 "github.com/stretchr/testify/assert" 21 "github.com/stretchr/testify/require" 22 ) 23 24 // Some times used in the tests 25 var ( 26 t1 = fstest.Time("2001-02-03T04:05:06.499999999Z") 27 ) 28 29 func TestMain(m *testing.M) { 30 fstest.TestMain(m) 31 } 32 33 type marchTester struct { 34 ctx context.Context // internal context for controlling go-routines 35 cancel func() // cancel the context 36 srcOnly fs.DirEntries 37 dstOnly fs.DirEntries 38 match fs.DirEntries 39 entryMutex sync.Mutex 40 errorMu sync.Mutex // Mutex covering the error variables 41 err error 42 noRetryErr error 43 fatalErr error 44 noTraverse bool 45 } 46 47 // DstOnly have an object which is in the destination only 48 func (mt *marchTester) DstOnly(dst fs.DirEntry) (recurse bool) { 49 mt.entryMutex.Lock() 50 mt.dstOnly = append(mt.dstOnly, dst) 51 mt.entryMutex.Unlock() 52 53 switch dst.(type) { 54 case fs.Object: 55 return false 56 case fs.Directory: 57 return true 58 default: 59 panic("Bad object in DirEntries") 60 } 61 } 62 63 // SrcOnly have an object which is in the source only 64 func (mt *marchTester) SrcOnly(src fs.DirEntry) (recurse bool) { 65 mt.entryMutex.Lock() 66 mt.srcOnly = append(mt.srcOnly, src) 67 mt.entryMutex.Unlock() 68 69 switch src.(type) { 70 case fs.Object: 71 return false 72 case fs.Directory: 73 return true 74 default: 75 panic("Bad object in DirEntries") 76 } 77 } 78 79 // Match is called when src and dst are present, so sync src to dst 80 func (mt *marchTester) Match(ctx context.Context, dst, src fs.DirEntry) (recurse bool) { 81 mt.entryMutex.Lock() 82 mt.match = append(mt.match, src) 83 mt.entryMutex.Unlock() 84 85 switch src.(type) { 86 case fs.Object: 87 return false 88 case fs.Directory: 89 // Do the same thing to the entire contents of the directory 90 _, ok := dst.(fs.Directory) 91 if ok { 92 return true 93 } 94 // FIXME src is dir, dst is file 95 err := errors.New("can't overwrite file with directory") 96 fs.Errorf(dst, "%v", err) 97 mt.processError(err) 98 default: 99 panic("Bad object in DirEntries") 100 } 101 return false 102 } 103 104 func (mt *marchTester) processError(err error) { 105 if err == nil { 106 return 107 } 108 mt.errorMu.Lock() 109 defer mt.errorMu.Unlock() 110 switch { 111 case fserrors.IsFatalError(err): 112 if !mt.aborting() { 113 fs.Errorf(nil, "Cancelling sync due to fatal error: %v", err) 114 mt.cancel() 115 } 116 mt.fatalErr = err 117 case fserrors.IsNoRetryError(err): 118 mt.noRetryErr = err 119 default: 120 mt.err = err 121 } 122 } 123 124 func (mt *marchTester) currentError() error { 125 mt.errorMu.Lock() 126 defer mt.errorMu.Unlock() 127 if mt.fatalErr != nil { 128 return mt.fatalErr 129 } 130 if mt.err != nil { 131 return mt.err 132 } 133 return mt.noRetryErr 134 } 135 136 func (mt *marchTester) aborting() bool { 137 return mt.ctx.Err() != nil 138 } 139 140 func TestMarch(t *testing.T) { 141 for _, test := range []struct { 142 what string 143 fileSrcOnly []string 144 dirSrcOnly []string 145 fileDstOnly []string 146 dirDstOnly []string 147 fileMatch []string 148 dirMatch []string 149 }{ 150 { 151 what: "source only", 152 fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"}, 153 dirSrcOnly: []string{"sub dir"}, 154 }, 155 { 156 what: "identical", 157 fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"}, 158 dirMatch: []string{"sub dir", "sub dir/sub sub dir"}, 159 }, 160 { 161 what: "typical sync", 162 fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"}, 163 dirSrcOnly: []string{"srcOnlyDir"}, 164 fileMatch: []string{"match", "matchDir/match file"}, 165 dirMatch: []string{"matchDir"}, 166 fileDstOnly: []string{"dstOnly", "dstOnlyDir/sub"}, 167 dirDstOnly: []string{"dstOnlyDir"}, 168 }, 169 } { 170 t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) { 171 r := fstest.NewRun(t) 172 defer r.Finalise() 173 174 var srcOnly []fstest.Item 175 var dstOnly []fstest.Item 176 var match []fstest.Item 177 178 ctx, cancel := context.WithCancel(context.Background()) 179 180 for _, f := range test.fileSrcOnly { 181 srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1)) 182 } 183 for _, f := range test.fileDstOnly { 184 dstOnly = append(dstOnly, r.WriteObject(ctx, f, "hello world", t1)) 185 } 186 for _, f := range test.fileMatch { 187 match = append(match, r.WriteBoth(ctx, f, "hello world", t1)) 188 } 189 190 mt := &marchTester{ 191 ctx: ctx, 192 cancel: cancel, 193 noTraverse: false, 194 } 195 m := &March{ 196 Ctx: ctx, 197 Fdst: r.Fremote, 198 Fsrc: r.Flocal, 199 Dir: "", 200 NoTraverse: mt.noTraverse, 201 Callback: mt, 202 DstIncludeAll: filter.Active.Opt.DeleteExcluded, 203 } 204 205 mt.processError(m.Run()) 206 mt.cancel() 207 err := mt.currentError() 208 require.NoError(t, err) 209 210 precision := fs.GetModifyWindow(r.Fremote, r.Flocal) 211 fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly") 212 fstest.CompareItems(t, mt.dstOnly, dstOnly, test.dirDstOnly, precision, "dstOnly") 213 fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match") 214 }) 215 } 216 } 217 218 func TestMarchNoTraverse(t *testing.T) { 219 for _, test := range []struct { 220 what string 221 fileSrcOnly []string 222 dirSrcOnly []string 223 fileMatch []string 224 dirMatch []string 225 }{ 226 { 227 what: "source only", 228 fileSrcOnly: []string{"test", "test2", "test3", "sub dir/test4"}, 229 dirSrcOnly: []string{"sub dir"}, 230 }, 231 { 232 what: "identical", 233 fileMatch: []string{"test", "test2", "sub dir/test3", "sub dir/sub sub dir/test4"}, 234 }, 235 { 236 what: "typical sync", 237 fileSrcOnly: []string{"srcOnly", "srcOnlyDir/sub"}, 238 fileMatch: []string{"match", "matchDir/match file"}, 239 }, 240 } { 241 t.Run(fmt.Sprintf("TestMarch-%s", test.what), func(t *testing.T) { 242 r := fstest.NewRun(t) 243 defer r.Finalise() 244 245 var srcOnly []fstest.Item 246 var match []fstest.Item 247 248 ctx, cancel := context.WithCancel(context.Background()) 249 250 for _, f := range test.fileSrcOnly { 251 srcOnly = append(srcOnly, r.WriteFile(f, "hello world", t1)) 252 } 253 for _, f := range test.fileMatch { 254 match = append(match, r.WriteBoth(ctx, f, "hello world", t1)) 255 } 256 257 mt := &marchTester{ 258 ctx: ctx, 259 cancel: cancel, 260 noTraverse: true, 261 } 262 m := &March{ 263 Ctx: ctx, 264 Fdst: r.Fremote, 265 Fsrc: r.Flocal, 266 Dir: "", 267 NoTraverse: mt.noTraverse, 268 Callback: mt, 269 DstIncludeAll: filter.Active.Opt.DeleteExcluded, 270 } 271 272 mt.processError(m.Run()) 273 mt.cancel() 274 err := mt.currentError() 275 require.NoError(t, err) 276 277 precision := fs.GetModifyWindow(r.Fremote, r.Flocal) 278 fstest.CompareItems(t, mt.srcOnly, srcOnly, test.dirSrcOnly, precision, "srcOnly") 279 fstest.CompareItems(t, mt.match, match, test.dirMatch, precision, "match") 280 }) 281 } 282 } 283 284 func TestNewMatchEntries(t *testing.T) { 285 var ( 286 a = mockobject.Object("path/a") 287 A = mockobject.Object("path/A") 288 B = mockobject.Object("path/B") 289 c = mockobject.Object("path/c") 290 ) 291 292 es := newMatchEntries(fs.DirEntries{a, A, B, c}, nil) 293 assert.Equal(t, es, matchEntries{ 294 {name: "A", leaf: "A", entry: A}, 295 {name: "B", leaf: "B", entry: B}, 296 {name: "a", leaf: "a", entry: a}, 297 {name: "c", leaf: "c", entry: c}, 298 }) 299 300 es = newMatchEntries(fs.DirEntries{a, A, B, c}, []matchTransformFn{strings.ToLower}) 301 assert.Equal(t, es, matchEntries{ 302 {name: "a", leaf: "A", entry: A}, 303 {name: "a", leaf: "a", entry: a}, 304 {name: "b", leaf: "B", entry: B}, 305 {name: "c", leaf: "c", entry: c}, 306 }) 307 } 308 309 func TestMatchListings(t *testing.T) { 310 var ( 311 a = mockobject.Object("a") 312 A = mockobject.Object("A") 313 b = mockobject.Object("b") 314 c = mockobject.Object("c") 315 d = mockobject.Object("d") 316 dirA = mockdir.New("A") 317 dirb = mockdir.New("b") 318 ) 319 320 for _, test := range []struct { 321 what string 322 input fs.DirEntries // pairs of input src, dst 323 srcOnly fs.DirEntries 324 dstOnly fs.DirEntries 325 matches []matchPair // pairs of output 326 transforms []matchTransformFn 327 }{ 328 { 329 what: "only src or dst", 330 input: fs.DirEntries{ 331 a, nil, 332 b, nil, 333 c, nil, 334 d, nil, 335 }, 336 srcOnly: fs.DirEntries{ 337 a, b, c, d, 338 }, 339 }, 340 { 341 what: "typical sync #1", 342 input: fs.DirEntries{ 343 a, nil, 344 b, b, 345 nil, c, 346 nil, d, 347 }, 348 srcOnly: fs.DirEntries{ 349 a, 350 }, 351 dstOnly: fs.DirEntries{ 352 c, d, 353 }, 354 matches: []matchPair{ 355 {b, b}, 356 }, 357 }, 358 { 359 what: "typical sync #2", 360 input: fs.DirEntries{ 361 a, a, 362 b, b, 363 nil, c, 364 d, d, 365 }, 366 dstOnly: fs.DirEntries{ 367 c, 368 }, 369 matches: []matchPair{ 370 {a, a}, 371 {b, b}, 372 {d, d}, 373 }, 374 }, 375 { 376 what: "One duplicate", 377 input: fs.DirEntries{ 378 A, A, 379 a, a, 380 a, nil, 381 b, b, 382 }, 383 matches: []matchPair{ 384 {A, A}, 385 {a, a}, 386 {b, b}, 387 }, 388 }, 389 { 390 what: "Two duplicates", 391 input: fs.DirEntries{ 392 a, a, 393 a, a, 394 a, nil, 395 }, 396 matches: []matchPair{ 397 {a, a}, 398 }, 399 }, 400 { 401 what: "Case insensitive duplicate - no transform", 402 input: fs.DirEntries{ 403 a, a, 404 A, A, 405 }, 406 matches: []matchPair{ 407 {A, A}, 408 {a, a}, 409 }, 410 }, 411 { 412 what: "Case insensitive duplicate - transform to lower case", 413 input: fs.DirEntries{ 414 a, a, 415 A, A, 416 }, 417 matches: []matchPair{ 418 {A, A}, 419 }, 420 transforms: []matchTransformFn{strings.ToLower}, 421 }, 422 { 423 what: "File and directory are not duplicates - srcOnly", 424 input: fs.DirEntries{ 425 dirA, nil, 426 A, nil, 427 }, 428 srcOnly: fs.DirEntries{ 429 dirA, 430 A, 431 }, 432 }, 433 { 434 what: "File and directory are not duplicates - matches", 435 input: fs.DirEntries{ 436 dirA, dirA, 437 A, A, 438 }, 439 matches: []matchPair{ 440 {dirA, dirA}, 441 {A, A}, 442 }, 443 }, 444 { 445 what: "Sync with directory #1", 446 input: fs.DirEntries{ 447 dirA, nil, 448 A, nil, 449 b, b, 450 nil, c, 451 nil, d, 452 }, 453 srcOnly: fs.DirEntries{ 454 dirA, 455 A, 456 }, 457 dstOnly: fs.DirEntries{ 458 c, d, 459 }, 460 matches: []matchPair{ 461 {b, b}, 462 }, 463 }, 464 { 465 what: "Sync with 2 directories", 466 input: fs.DirEntries{ 467 dirA, dirA, 468 A, nil, 469 nil, dirb, 470 nil, b, 471 }, 472 srcOnly: fs.DirEntries{ 473 A, 474 }, 475 dstOnly: fs.DirEntries{ 476 dirb, 477 b, 478 }, 479 matches: []matchPair{ 480 {dirA, dirA}, 481 }, 482 }, 483 } { 484 t.Run(fmt.Sprintf("TestMatchListings-%s", test.what), func(t *testing.T) { 485 var srcList, dstList fs.DirEntries 486 for i := 0; i < len(test.input); i += 2 { 487 src, dst := test.input[i], test.input[i+1] 488 if src != nil { 489 srcList = append(srcList, src) 490 } 491 if dst != nil { 492 dstList = append(dstList, dst) 493 } 494 } 495 srcOnly, dstOnly, matches := matchListings(srcList, dstList, test.transforms) 496 assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ") 497 assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ") 498 assert.Equal(t, test.matches, matches, test.what, "matches differ") 499 // now swap src and dst 500 dstOnly, srcOnly, matches = matchListings(dstList, srcList, test.transforms) 501 assert.Equal(t, test.srcOnly, srcOnly, test.what, "srcOnly differ") 502 assert.Equal(t, test.dstOnly, dstOnly, test.what, "dstOnly differ") 503 assert.Equal(t, test.matches, matches, test.what, "matches differ") 504 }) 505 } 506 }