github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/fs/operations/rc_test.go (about) 1 package operations_test 2 3 import ( 4 "context" 5 "fmt" 6 "net/http" 7 "net/http/httptest" 8 "net/url" 9 "os" 10 "path" 11 "sort" 12 "strings" 13 "testing" 14 "time" 15 16 "github.com/rclone/rclone/fs" 17 "github.com/rclone/rclone/fs/cache" 18 "github.com/rclone/rclone/fs/hash" 19 "github.com/rclone/rclone/fs/operations" 20 "github.com/rclone/rclone/fs/rc" 21 "github.com/rclone/rclone/fstest" 22 "github.com/rclone/rclone/lib/diskusage" 23 "github.com/rclone/rclone/lib/rest" 24 "github.com/stretchr/testify/assert" 25 "github.com/stretchr/testify/require" 26 ) 27 28 func rcNewRun(t *testing.T, method string) (*fstest.Run, *rc.Call) { 29 if *fstest.RemoteName != "" { 30 t.Skip("Skipping test on non local remote") 31 } 32 r := fstest.NewRun(t) 33 call := rc.Calls.Get(method) 34 assert.NotNil(t, call) 35 cache.Put(r.LocalName, r.Flocal) 36 cache.Put(r.FremoteName, r.Fremote) 37 return r, call 38 } 39 40 // operations/about: Return the space used on the remote 41 func TestRcAbout(t *testing.T) { 42 r, call := rcNewRun(t, "operations/about") 43 r.Mkdir(context.Background(), r.Fremote) 44 45 // Will get an error if remote doesn't support About 46 expectedErr := r.Fremote.Features().About == nil 47 48 in := rc.Params{ 49 "fs": r.FremoteName, 50 } 51 out, err := call.Fn(context.Background(), in) 52 if expectedErr { 53 assert.Error(t, err) 54 return 55 } 56 require.NoError(t, err) 57 58 // Can't really check the output much! 59 assert.NotEqual(t, int64(0), out["Total"]) 60 } 61 62 // operations/cleanup: Remove trashed files in the remote or path 63 func TestRcCleanup(t *testing.T) { 64 r, call := rcNewRun(t, "operations/cleanup") 65 66 in := rc.Params{ 67 "fs": r.LocalName, 68 } 69 out, err := call.Fn(context.Background(), in) 70 require.Error(t, err) 71 assert.Equal(t, rc.Params(nil), out) 72 assert.Contains(t, err.Error(), "doesn't support cleanup") 73 } 74 75 // operations/copyfile: Copy a file from source remote to destination remote 76 func TestRcCopyfile(t *testing.T) { 77 r, call := rcNewRun(t, "operations/copyfile") 78 file1 := r.WriteFile("file1", "file1 contents", t1) 79 r.Mkdir(context.Background(), r.Fremote) 80 r.CheckLocalItems(t, file1) 81 r.CheckRemoteItems(t) 82 83 in := rc.Params{ 84 "srcFs": r.LocalName, 85 "srcRemote": "file1", 86 "dstFs": r.FremoteName, 87 "dstRemote": "file1-renamed", 88 } 89 out, err := call.Fn(context.Background(), in) 90 require.NoError(t, err) 91 assert.Equal(t, rc.Params(nil), out) 92 93 r.CheckLocalItems(t, file1) 94 file1.Path = "file1-renamed" 95 r.CheckRemoteItems(t, file1) 96 } 97 98 // operations/copyurl: Copy the URL to the object 99 func TestRcCopyurl(t *testing.T) { 100 r, call := rcNewRun(t, "operations/copyurl") 101 contents := "file1 contents\n" 102 file1 := r.WriteFile("file1", contents, t1) 103 r.Mkdir(context.Background(), r.Fremote) 104 r.CheckRemoteItems(t) 105 106 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 107 _, err := w.Write([]byte(contents)) 108 assert.NoError(t, err) 109 })) 110 defer ts.Close() 111 112 in := rc.Params{ 113 "fs": r.FremoteName, 114 "remote": "file1", 115 "url": ts.URL, 116 "autoFilename": false, 117 "noClobber": false, 118 } 119 out, err := call.Fn(context.Background(), in) 120 require.NoError(t, err) 121 assert.Equal(t, rc.Params(nil), out) 122 123 in = rc.Params{ 124 "fs": r.FremoteName, 125 "remote": "file1", 126 "url": ts.URL, 127 "autoFilename": false, 128 "noClobber": true, 129 } 130 out, err = call.Fn(context.Background(), in) 131 require.Error(t, err) 132 assert.Equal(t, rc.Params(nil), out) 133 134 urlFileName := "filename.txt" 135 in = rc.Params{ 136 "fs": r.FremoteName, 137 "remote": "", 138 "url": ts.URL + "/" + urlFileName, 139 "autoFilename": true, 140 "noClobber": false, 141 } 142 out, err = call.Fn(context.Background(), in) 143 require.NoError(t, err) 144 assert.Equal(t, rc.Params(nil), out) 145 146 in = rc.Params{ 147 "fs": r.FremoteName, 148 "remote": "", 149 "url": ts.URL, 150 "autoFilename": true, 151 "noClobber": false, 152 } 153 out, err = call.Fn(context.Background(), in) 154 require.Error(t, err) 155 assert.Equal(t, rc.Params(nil), out) 156 157 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1, fstest.NewItem(urlFileName, contents, t1)}, nil, fs.ModTimeNotSupported) 158 } 159 160 // operations/delete: Remove files in the path 161 func TestRcDelete(t *testing.T) { 162 r, call := rcNewRun(t, "operations/delete") 163 164 file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes 165 file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes 166 file3 := r.WriteObject(context.Background(), "large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 100 bytes 167 r.CheckRemoteItems(t, file1, file2, file3) 168 169 in := rc.Params{ 170 "fs": r.FremoteName, 171 } 172 out, err := call.Fn(context.Background(), in) 173 require.NoError(t, err) 174 assert.Equal(t, rc.Params(nil), out) 175 176 r.CheckRemoteItems(t) 177 } 178 179 // operations/deletefile: Remove the single file pointed to 180 func TestRcDeletefile(t *testing.T) { 181 r, call := rcNewRun(t, "operations/deletefile") 182 183 file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes 184 file2 := r.WriteObject(context.Background(), "medium", "------------------------------------------------------------", t1) // 60 bytes 185 r.CheckRemoteItems(t, file1, file2) 186 187 in := rc.Params{ 188 "fs": r.FremoteName, 189 "remote": "small", 190 } 191 out, err := call.Fn(context.Background(), in) 192 require.NoError(t, err) 193 assert.Equal(t, rc.Params(nil), out) 194 195 r.CheckRemoteItems(t, file2) 196 } 197 198 // operations/list: List the given remote and path in JSON format. 199 func TestRcList(t *testing.T) { 200 r, call := rcNewRun(t, "operations/list") 201 202 file1 := r.WriteObject(context.Background(), "a", "a", t1) 203 file2 := r.WriteObject(context.Background(), "subdir/b", "bb", t2) 204 205 r.CheckRemoteItems(t, file1, file2) 206 207 in := rc.Params{ 208 "fs": r.FremoteName, 209 "remote": "", 210 } 211 out, err := call.Fn(context.Background(), in) 212 require.NoError(t, err) 213 214 list := out["list"].([]*operations.ListJSONItem) 215 assert.Equal(t, 2, len(list)) 216 217 checkFile1 := func(got *operations.ListJSONItem) { 218 assert.WithinDuration(t, t1, got.ModTime.When, time.Second) 219 assert.Equal(t, "a", got.Path) 220 assert.Equal(t, "a", got.Name) 221 assert.Equal(t, int64(1), got.Size) 222 assert.Equal(t, "application/octet-stream", got.MimeType) 223 assert.Equal(t, false, got.IsDir) 224 } 225 checkFile1(list[0]) 226 227 checkSubdir := func(got *operations.ListJSONItem) { 228 assert.Equal(t, "subdir", got.Path) 229 assert.Equal(t, "subdir", got.Name) 230 // assert.Equal(t, int64(-1), got.Size) // size can vary for directories 231 assert.Equal(t, "inode/directory", got.MimeType) 232 assert.Equal(t, true, got.IsDir) 233 } 234 checkSubdir(list[1]) 235 236 in = rc.Params{ 237 "fs": r.FremoteName, 238 "remote": "", 239 "opt": rc.Params{ 240 "recurse": true, 241 }, 242 } 243 out, err = call.Fn(context.Background(), in) 244 require.NoError(t, err) 245 246 list = out["list"].([]*operations.ListJSONItem) 247 assert.Equal(t, 3, len(list)) 248 checkFile1(list[0]) 249 checkSubdir(list[1]) 250 251 checkFile2 := func(got *operations.ListJSONItem) { 252 assert.WithinDuration(t, t2, got.ModTime.When, time.Second) 253 assert.Equal(t, "subdir/b", got.Path) 254 assert.Equal(t, "b", got.Name) 255 assert.Equal(t, int64(2), got.Size) 256 assert.Equal(t, "application/octet-stream", got.MimeType) 257 assert.Equal(t, false, got.IsDir) 258 } 259 checkFile2(list[2]) 260 } 261 262 // operations/stat: Stat the given remote and path in JSON format. 263 func TestRcStat(t *testing.T) { 264 r, call := rcNewRun(t, "operations/stat") 265 266 file1 := r.WriteObject(context.Background(), "subdir/a", "a", t1) 267 268 r.CheckRemoteItems(t, file1) 269 270 fetch := func(t *testing.T, remotePath string) *operations.ListJSONItem { 271 in := rc.Params{ 272 "fs": r.FremoteName, 273 "remote": remotePath, 274 } 275 out, err := call.Fn(context.Background(), in) 276 require.NoError(t, err) 277 return out["item"].(*operations.ListJSONItem) 278 } 279 280 t.Run("Root", func(t *testing.T) { 281 stat := fetch(t, "") 282 assert.Equal(t, "", stat.Path) 283 assert.Equal(t, "", stat.Name) 284 assert.Equal(t, int64(-1), stat.Size) 285 assert.Equal(t, "inode/directory", stat.MimeType) 286 assert.Equal(t, true, stat.IsDir) 287 }) 288 289 t.Run("File", func(t *testing.T) { 290 stat := fetch(t, "subdir/a") 291 assert.WithinDuration(t, t1, stat.ModTime.When, time.Second) 292 assert.Equal(t, "subdir/a", stat.Path) 293 assert.Equal(t, "a", stat.Name) 294 assert.Equal(t, int64(1), stat.Size) 295 assert.Equal(t, "application/octet-stream", stat.MimeType) 296 assert.Equal(t, false, stat.IsDir) 297 }) 298 299 t.Run("Subdir", func(t *testing.T) { 300 stat := fetch(t, "subdir") 301 assert.Equal(t, "subdir", stat.Path) 302 assert.Equal(t, "subdir", stat.Name) 303 // assert.Equal(t, int64(-1), stat.Size) // size can vary for directories 304 assert.Equal(t, "inode/directory", stat.MimeType) 305 assert.Equal(t, true, stat.IsDir) 306 }) 307 308 t.Run("NotFound", func(t *testing.T) { 309 stat := fetch(t, "notfound") 310 assert.Nil(t, stat) 311 }) 312 } 313 314 // operations/settier: Set the storage tier of a fs 315 func TestRcSetTier(t *testing.T) { 316 ctx := context.Background() 317 r, call := rcNewRun(t, "operations/settier") 318 if !r.Fremote.Features().SetTier { 319 t.Skip("settier not supported") 320 } 321 file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1) 322 r.CheckRemoteItems(t, file1) 323 324 // Because we don't know what the current tier options here are, let's 325 // just get the current tier, and reuse that 326 o, err := r.Fremote.NewObject(ctx, file1.Path) 327 require.NoError(t, err) 328 trr, ok := o.(fs.GetTierer) 329 require.True(t, ok) 330 ctier := trr.GetTier() 331 in := rc.Params{ 332 "fs": r.FremoteName, 333 "tier": ctier, 334 } 335 out, err := call.Fn(context.Background(), in) 336 require.NoError(t, err) 337 assert.Equal(t, rc.Params(nil), out) 338 339 } 340 341 // operations/settier: Set the storage tier of a file 342 func TestRcSetTierFile(t *testing.T) { 343 ctx := context.Background() 344 r, call := rcNewRun(t, "operations/settierfile") 345 if !r.Fremote.Features().SetTier { 346 t.Skip("settier not supported") 347 } 348 file1 := r.WriteObject(context.Background(), "file1", "file1 contents", t1) 349 r.CheckRemoteItems(t, file1) 350 351 // Because we don't know what the current tier options here are, let's 352 // just get the current tier, and reuse that 353 o, err := r.Fremote.NewObject(ctx, file1.Path) 354 require.NoError(t, err) 355 trr, ok := o.(fs.GetTierer) 356 require.True(t, ok) 357 ctier := trr.GetTier() 358 in := rc.Params{ 359 "fs": r.FremoteName, 360 "remote": "file1", 361 "tier": ctier, 362 } 363 out, err := call.Fn(context.Background(), in) 364 require.NoError(t, err) 365 assert.Equal(t, rc.Params(nil), out) 366 367 } 368 369 // operations/mkdir: Make a destination directory or container 370 func TestRcMkdir(t *testing.T) { 371 ctx := context.Background() 372 r, call := rcNewRun(t, "operations/mkdir") 373 r.Mkdir(context.Background(), r.Fremote) 374 375 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) 376 377 in := rc.Params{ 378 "fs": r.FremoteName, 379 "remote": "subdir", 380 } 381 out, err := call.Fn(context.Background(), in) 382 require.NoError(t, err) 383 assert.Equal(t, rc.Params(nil), out) 384 385 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) 386 } 387 388 // operations/movefile: Move a file from source remote to destination remote 389 func TestRcMovefile(t *testing.T) { 390 r, call := rcNewRun(t, "operations/movefile") 391 file1 := r.WriteFile("file1", "file1 contents", t1) 392 r.Mkdir(context.Background(), r.Fremote) 393 r.CheckLocalItems(t, file1) 394 r.CheckRemoteItems(t) 395 396 in := rc.Params{ 397 "srcFs": r.LocalName, 398 "srcRemote": "file1", 399 "dstFs": r.FremoteName, 400 "dstRemote": "file1-renamed", 401 } 402 out, err := call.Fn(context.Background(), in) 403 require.NoError(t, err) 404 assert.Equal(t, rc.Params(nil), out) 405 406 r.CheckLocalItems(t) 407 file1.Path = "file1-renamed" 408 r.CheckRemoteItems(t, file1) 409 } 410 411 // operations/purge: Remove a directory or container and all of its contents 412 func TestRcPurge(t *testing.T) { 413 ctx := context.Background() 414 r, call := rcNewRun(t, "operations/purge") 415 file1 := r.WriteObject(context.Background(), "subdir/file1", "subdir/file1 contents", t1) 416 417 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{file1}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) 418 419 in := rc.Params{ 420 "fs": r.FremoteName, 421 "remote": "subdir", 422 } 423 out, err := call.Fn(context.Background(), in) 424 require.NoError(t, err) 425 assert.Equal(t, rc.Params(nil), out) 426 427 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) 428 } 429 430 // operations/rmdir: Remove an empty directory or container 431 func TestRcRmdir(t *testing.T) { 432 ctx := context.Background() 433 r, call := rcNewRun(t, "operations/rmdir") 434 r.Mkdir(context.Background(), r.Fremote) 435 assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) 436 437 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) 438 439 in := rc.Params{ 440 "fs": r.FremoteName, 441 "remote": "subdir", 442 } 443 out, err := call.Fn(context.Background(), in) 444 require.NoError(t, err) 445 assert.Equal(t, rc.Params(nil), out) 446 447 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) 448 } 449 450 // operations/rmdirs: Remove all the empty directories in the path 451 func TestRcRmdirs(t *testing.T) { 452 ctx := context.Background() 453 r, call := rcNewRun(t, "operations/rmdirs") 454 r.Mkdir(context.Background(), r.Fremote) 455 assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) 456 assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir")) 457 458 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir", "subdir/subsubdir"}, fs.GetModifyWindow(ctx, r.Fremote)) 459 460 in := rc.Params{ 461 "fs": r.FremoteName, 462 "remote": "subdir", 463 } 464 out, err := call.Fn(context.Background(), in) 465 require.NoError(t, err) 466 assert.Equal(t, rc.Params(nil), out) 467 468 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{}, fs.GetModifyWindow(ctx, r.Fremote)) 469 470 assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) 471 assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir/subsubdir")) 472 473 in = rc.Params{ 474 "fs": r.FremoteName, 475 "remote": "subdir", 476 "leaveRoot": true, 477 } 478 out, err = call.Fn(context.Background(), in) 479 require.NoError(t, err) 480 assert.Equal(t, rc.Params(nil), out) 481 482 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{}, []string{"subdir"}, fs.GetModifyWindow(ctx, r.Fremote)) 483 484 } 485 486 // operations/size: Count the number of bytes and files in remote 487 func TestRcSize(t *testing.T) { 488 r, call := rcNewRun(t, "operations/size") 489 file1 := r.WriteObject(context.Background(), "small", "1234567890", t2) // 10 bytes 490 file2 := r.WriteObject(context.Background(), "subdir/medium", "------------------------------------------------------------", t1) // 60 bytes 491 file3 := r.WriteObject(context.Background(), "subdir/subsubdir/large", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", t1) // 50 bytes 492 r.CheckRemoteItems(t, file1, file2, file3) 493 494 in := rc.Params{ 495 "fs": r.FremoteName, 496 } 497 out, err := call.Fn(context.Background(), in) 498 require.NoError(t, err) 499 assert.Equal(t, rc.Params{ 500 "count": int64(3), 501 "bytes": int64(120), 502 "sizeless": int64(0), 503 }, out) 504 } 505 506 // operations/publiclink: Create or retrieve a public link to the given file or folder. 507 func TestRcPublicLink(t *testing.T) { 508 r, call := rcNewRun(t, "operations/publiclink") 509 in := rc.Params{ 510 "fs": r.FremoteName, 511 "remote": "", 512 "expire": "5m", 513 "unlink": false, 514 } 515 _, err := call.Fn(context.Background(), in) 516 require.Error(t, err) 517 assert.Contains(t, err.Error(), "doesn't support public links") 518 } 519 520 // operations/fsinfo: Return information about the remote 521 func TestRcFsInfo(t *testing.T) { 522 r, call := rcNewRun(t, "operations/fsinfo") 523 in := rc.Params{ 524 "fs": r.FremoteName, 525 } 526 got, err := call.Fn(context.Background(), in) 527 require.NoError(t, err) 528 want := operations.GetFsInfo(r.Fremote) 529 assert.Equal(t, want.Name, got["Name"]) 530 assert.Equal(t, want.Root, got["Root"]) 531 assert.Equal(t, want.String, got["String"]) 532 assert.Equal(t, float64(want.Precision), got["Precision"]) 533 var hashes []interface{} 534 for _, hash := range want.Hashes { 535 hashes = append(hashes, hash) 536 } 537 assert.Equal(t, hashes, got["Hashes"]) 538 var features = map[string]interface{}{} 539 for k, v := range want.Features { 540 features[k] = v 541 } 542 assert.Equal(t, features, got["Features"]) 543 544 } 545 546 // operations/uploadfile : Tests if upload file succeeds 547 func TestUploadFile(t *testing.T) { 548 r, call := rcNewRun(t, "operations/uploadfile") 549 ctx := context.Background() 550 551 testFileName := "uploadfile-test.txt" 552 testFileContent := "Hello World" 553 r.WriteFile(testFileName, testFileContent, t1) 554 testItem1 := fstest.NewItem(testFileName, testFileContent, t1) 555 testItem2 := fstest.NewItem(path.Join("subdir", testFileName), testFileContent, t1) 556 557 currentFile, err := os.Open(path.Join(r.LocalName, testFileName)) 558 require.NoError(t, err) 559 560 defer func() { 561 assert.NoError(t, currentFile.Close()) 562 }() 563 564 formReader, contentType, _, err := rest.MultipartUpload(ctx, currentFile, url.Values{}, "file", testFileName) 565 require.NoError(t, err) 566 567 httpReq := httptest.NewRequest("POST", "/", formReader) 568 httpReq.Header.Add("Content-Type", contentType) 569 570 in := rc.Params{ 571 "_request": httpReq, 572 "fs": r.FremoteName, 573 "remote": "", 574 } 575 576 _, err = call.Fn(context.Background(), in) 577 require.NoError(t, err) 578 579 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{testItem1}, nil, fs.ModTimeNotSupported) 580 581 assert.NoError(t, r.Fremote.Mkdir(context.Background(), "subdir")) 582 583 currentFile2, err := os.Open(path.Join(r.LocalName, testFileName)) 584 require.NoError(t, err) 585 586 defer func() { 587 assert.NoError(t, currentFile2.Close()) 588 }() 589 590 formReader, contentType, _, err = rest.MultipartUpload(ctx, currentFile2, url.Values{}, "file", testFileName) 591 require.NoError(t, err) 592 593 httpReq = httptest.NewRequest("POST", "/", formReader) 594 httpReq.Header.Add("Content-Type", contentType) 595 596 in = rc.Params{ 597 "_request": httpReq, 598 "fs": r.FremoteName, 599 "remote": "subdir", 600 } 601 602 _, err = call.Fn(context.Background(), in) 603 require.NoError(t, err) 604 605 fstest.CheckListingWithPrecision(t, r.Fremote, []fstest.Item{testItem1, testItem2}, nil, fs.ModTimeNotSupported) 606 607 } 608 609 // operations/command: Runs a backend command 610 func TestRcCommand(t *testing.T) { 611 r, call := rcNewRun(t, "backend/command") 612 in := rc.Params{ 613 "fs": r.FremoteName, 614 "command": "noop", 615 "opt": map[string]string{ 616 "echo": "true", 617 "blue": "", 618 }, 619 "arg": []string{ 620 "path1", 621 "path2", 622 }, 623 } 624 got, err := call.Fn(context.Background(), in) 625 if err != nil { 626 assert.False(t, r.Fremote.Features().IsLocal, "mustn't fail on local remote") 627 assert.Contains(t, err.Error(), "command not found") 628 return 629 } 630 want := rc.Params{"result": map[string]interface{}{ 631 "arg": []string{ 632 "path1", 633 "path2", 634 }, 635 "name": "noop", 636 "opt": map[string]string{ 637 "blue": "", 638 "echo": "true", 639 }, 640 }} 641 assert.Equal(t, want, got) 642 errTxt := "explosion in the sausage factory" 643 in["opt"].(map[string]string)["error"] = errTxt 644 _, err = call.Fn(context.Background(), in) 645 assert.Error(t, err) 646 assert.Contains(t, err.Error(), errTxt) 647 } 648 649 // operations/command: Runs a backend command 650 func TestRcDu(t *testing.T) { 651 ctx := context.Background() 652 _, call := rcNewRun(t, "core/du") 653 in := rc.Params{} 654 out, err := call.Fn(ctx, in) 655 if err == diskusage.ErrUnsupported { 656 t.Skip(err) 657 } 658 assert.NotEqual(t, "", out["dir"]) 659 info := out["info"].(diskusage.Info) 660 assert.True(t, info.Total != 0) 661 assert.True(t, info.Total > info.Free) 662 assert.True(t, info.Total > info.Available) 663 assert.True(t, info.Free >= info.Available) 664 } 665 666 // operations/check: check the source and destination are the same 667 func TestRcCheck(t *testing.T) { 668 ctx := context.Background() 669 r, call := rcNewRun(t, "operations/check") 670 r.Mkdir(ctx, r.Fremote) 671 672 MD5SUMS := ` 673 0ef726ce9b1a7692357ff70dd321d595 file1 674 deadbeefcafe00000000000000000000 subdir/file2 675 0386a8b8fcf672c326845c00ba41b9e2 subdir/subsubdir/file4 676 ` 677 678 file1 := r.WriteBoth(ctx, "file1", "file1 contents", t1) 679 file2 := r.WriteFile("subdir/file2", MD5SUMS, t2) 680 file3 := r.WriteObject(ctx, "subdir/subsubdir/file3", "file3 contents", t3) 681 file4a := r.WriteFile("subdir/subsubdir/file4", "file4 contents", t3) 682 file4b := r.WriteObject(ctx, "subdir/subsubdir/file4", "file4 different contents", t3) 683 // operations.HashLister(ctx, hash.MD5, false, false, r.Fremote, os.Stdout) 684 685 r.CheckLocalItems(t, file1, file2, file4a) 686 r.CheckRemoteItems(t, file1, file3, file4b) 687 688 pstring := func(items ...fstest.Item) *[]string { 689 xs := make([]string, len(items)) 690 for i, item := range items { 691 xs[i] = item.Path 692 } 693 return &xs 694 } 695 696 for _, testName := range []string{"Normal", "Download"} { 697 t.Run(testName, func(t *testing.T) { 698 in := rc.Params{ 699 "srcFs": r.LocalName, 700 "dstFs": r.FremoteName, 701 "combined": true, 702 "missingOnSrc": true, 703 "missingOnDst": true, 704 "match": true, 705 "differ": true, 706 "error": true, 707 } 708 if testName == "Download" { 709 in["download"] = true 710 } 711 out, err := call.Fn(ctx, in) 712 require.NoError(t, err) 713 714 combined := []string{ 715 "= " + file1.Path, 716 "+ " + file2.Path, 717 "- " + file3.Path, 718 "* " + file4a.Path, 719 } 720 sort.Strings(combined) 721 sort.Strings(*out["combined"].(*[]string)) 722 want := rc.Params{ 723 "missingOnSrc": pstring(file3), 724 "missingOnDst": pstring(file2), 725 "differ": pstring(file4a), 726 "error": pstring(), 727 "match": pstring(file1), 728 "combined": &combined, 729 "status": "3 differences found", 730 "success": false, 731 } 732 if testName == "Normal" { 733 want["hashType"] = "md5" 734 } 735 736 assert.Equal(t, want, out) 737 }) 738 } 739 740 t.Run("CheckFile", func(t *testing.T) { 741 // The checksum file is treated as the source and srcFs is not used 742 in := rc.Params{ 743 "dstFs": r.FremoteName, 744 "combined": true, 745 "missingOnSrc": true, 746 "missingOnDst": true, 747 "match": true, 748 "differ": true, 749 "error": true, 750 "checkFileFs": r.LocalName, 751 "checkFileRemote": file2.Path, 752 "checkFileHash": "md5", 753 } 754 out, err := call.Fn(ctx, in) 755 require.NoError(t, err) 756 757 combined := []string{ 758 "= " + file1.Path, 759 "+ " + file2.Path, 760 "- " + file3.Path, 761 "* " + file4a.Path, 762 } 763 sort.Strings(combined) 764 sort.Strings(*out["combined"].(*[]string)) 765 if strings.HasPrefix(out["status"].(string), "file not in") { 766 out["status"] = "file not in" 767 } 768 want := rc.Params{ 769 "missingOnSrc": pstring(file3), 770 "missingOnDst": pstring(file2), 771 "differ": pstring(file4a), 772 "error": pstring(), 773 "match": pstring(file1), 774 "combined": &combined, 775 "hashType": "md5", 776 "status": "file not in", 777 "success": false, 778 } 779 780 assert.Equal(t, want, out) 781 }) 782 783 } 784 785 // operations/hashsum: hashsum a directory 786 func TestRcHashsum(t *testing.T) { 787 ctx := context.Background() 788 r, call := rcNewRun(t, "operations/hashsum") 789 r.Mkdir(ctx, r.Fremote) 790 791 file1Contents := "file1 contents" 792 file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1) 793 r.CheckLocalItems(t, file1) 794 r.CheckRemoteItems(t, file1) 795 796 hasher := hash.NewMultiHasher() 797 _, err := hasher.Write([]byte(file1Contents)) 798 require.NoError(t, err) 799 800 for _, test := range []struct { 801 ht hash.Type 802 base64 bool 803 download bool 804 }{ 805 { 806 ht: r.Fremote.Hashes().GetOne(), 807 }, { 808 ht: r.Fremote.Hashes().GetOne(), 809 base64: true, 810 }, { 811 ht: hash.Whirlpool, 812 base64: false, 813 download: true, 814 }, { 815 ht: hash.Whirlpool, 816 base64: true, 817 download: true, 818 }, 819 } { 820 t.Run(fmt.Sprintf("hash=%v,base64=%v,download=%v", test.ht, test.base64, test.download), func(t *testing.T) { 821 file1Hash, err := hasher.SumString(test.ht, test.base64) 822 require.NoError(t, err) 823 824 in := rc.Params{ 825 "fs": r.FremoteName, 826 "hashType": test.ht.String(), 827 "base64": test.base64, 828 "download": test.download, 829 } 830 831 out, err := call.Fn(ctx, in) 832 require.NoError(t, err) 833 assert.Equal(t, test.ht.String(), out["hashType"]) 834 want := []string{ 835 fmt.Sprintf("%s hashsum-file1", file1Hash), 836 } 837 assert.Equal(t, want, out["hashsum"]) 838 }) 839 } 840 } 841 842 // operations/hashsum: hashsum a single file 843 func TestRcHashsumFile(t *testing.T) { 844 ctx := context.Background() 845 r, call := rcNewRun(t, "operations/hashsum") 846 r.Mkdir(ctx, r.Fremote) 847 848 file1Contents := "file1 contents" 849 file1 := r.WriteBoth(ctx, "hashsum-file1", file1Contents, t1) 850 file2Contents := "file2 contents" 851 file2 := r.WriteBoth(ctx, "hashsum-file2", file2Contents, t1) 852 r.CheckLocalItems(t, file1, file2) 853 r.CheckRemoteItems(t, file1, file2) 854 855 // Make an fs pointing to just the file 856 fsString := path.Join(r.FremoteName, file1.Path) 857 858 in := rc.Params{ 859 "fs": fsString, 860 "hashType": "MD5", 861 "download": true, 862 } 863 864 out, err := call.Fn(ctx, in) 865 require.NoError(t, err) 866 assert.Equal(t, "md5", out["hashType"]) 867 assert.Equal(t, []string{"0ef726ce9b1a7692357ff70dd321d595 hashsum-file1"}, out["hashsum"]) 868 }