github.com/rclone/rclone@v1.66.1-0.20240517100346-7b89735ae726/backend/http/http_internal_test.go (about) 1 package http 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "net/http" 8 "net/http/httptest" 9 "net/url" 10 "os" 11 "path" 12 "path/filepath" 13 "sort" 14 "strconv" 15 "strings" 16 "testing" 17 "time" 18 19 "github.com/rclone/rclone/fs" 20 "github.com/rclone/rclone/fs/config/configfile" 21 "github.com/rclone/rclone/fs/config/configmap" 22 "github.com/rclone/rclone/fstest" 23 "github.com/rclone/rclone/lib/rest" 24 "github.com/stretchr/testify/assert" 25 "github.com/stretchr/testify/require" 26 ) 27 28 var ( 29 remoteName = "TestHTTP" 30 testPath = "test" 31 filesPath = filepath.Join(testPath, "files") 32 headers = []string{"X-Potato", "sausage", "X-Rhubarb", "cucumber"} 33 lineEndSize = 1 34 ) 35 36 // prepareServer prepares the test server and shuts it down automatically 37 // when the test completes. 38 func prepareServer(t *testing.T) configmap.Simple { 39 // file server for test/files 40 fileServer := http.FileServer(http.Dir(filesPath)) 41 42 // verify the file path is correct, and also check which line endings 43 // are used to get sizes right ("\n" except on Windows, but even there 44 // we may have "\n" or "\r\n" depending on git crlf setting) 45 fileList, err := os.ReadDir(filesPath) 46 require.NoError(t, err) 47 require.Greater(t, len(fileList), 0) 48 for _, file := range fileList { 49 if !file.IsDir() { 50 data, _ := os.ReadFile(filepath.Join(filesPath, file.Name())) 51 if strings.HasSuffix(string(data), "\r\n") { 52 lineEndSize = 2 53 } 54 break 55 } 56 } 57 58 // test the headers are there then pass on to fileServer 59 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 60 what := fmt.Sprintf("%s %s: Header ", r.Method, r.URL.Path) 61 assert.Equal(t, headers[1], r.Header.Get(headers[0]), what+headers[0]) 62 assert.Equal(t, headers[3], r.Header.Get(headers[2]), what+headers[2]) 63 fileServer.ServeHTTP(w, r) 64 }) 65 66 // Make the test server 67 ts := httptest.NewServer(handler) 68 69 // Configure the remote 70 configfile.Install() 71 // fs.Config.LogLevel = fs.LogLevelDebug 72 // fs.Config.DumpHeaders = true 73 // fs.Config.DumpBodies = true 74 // config.FileSet(remoteName, "type", "http") 75 // config.FileSet(remoteName, "url", ts.URL) 76 77 m := configmap.Simple{ 78 "type": "http", 79 "url": ts.URL, 80 "headers": strings.Join(headers, ","), 81 } 82 t.Cleanup(ts.Close) 83 84 return m 85 } 86 87 // prepare prepares the test server and shuts it down automatically 88 // when the test completes. 89 func prepare(t *testing.T) fs.Fs { 90 m := prepareServer(t) 91 92 // Instantiate it 93 f, err := NewFs(context.Background(), remoteName, "", m) 94 require.NoError(t, err) 95 96 return f 97 } 98 99 func testListRoot(t *testing.T, f fs.Fs, noSlash bool) { 100 entries, err := f.List(context.Background(), "") 101 require.NoError(t, err) 102 103 sort.Sort(entries) 104 105 require.Equal(t, 4, len(entries)) 106 107 e := entries[0] 108 assert.Equal(t, "four", e.Remote()) 109 assert.Equal(t, int64(-1), e.Size()) 110 _, ok := e.(fs.Directory) 111 assert.True(t, ok) 112 113 e = entries[1] 114 assert.Equal(t, "one%.txt", e.Remote()) 115 assert.Equal(t, int64(5+lineEndSize), e.Size()) 116 _, ok = e.(*Object) 117 assert.True(t, ok) 118 119 e = entries[2] 120 assert.Equal(t, "three", e.Remote()) 121 assert.Equal(t, int64(-1), e.Size()) 122 _, ok = e.(fs.Directory) 123 assert.True(t, ok) 124 125 e = entries[3] 126 assert.Equal(t, "two.html", e.Remote()) 127 if noSlash { 128 assert.Equal(t, int64(-1), e.Size()) 129 _, ok = e.(fs.Directory) 130 assert.True(t, ok) 131 } else { 132 assert.Equal(t, int64(40+lineEndSize), e.Size()) 133 _, ok = e.(*Object) 134 assert.True(t, ok) 135 } 136 } 137 138 func TestListRoot(t *testing.T) { 139 f := prepare(t) 140 testListRoot(t, f, false) 141 } 142 143 func TestListRootNoSlash(t *testing.T) { 144 f := prepare(t) 145 f.(*Fs).opt.NoSlash = true 146 147 testListRoot(t, f, true) 148 } 149 150 func TestListSubDir(t *testing.T) { 151 f := prepare(t) 152 153 entries, err := f.List(context.Background(), "three") 154 require.NoError(t, err) 155 156 sort.Sort(entries) 157 158 assert.Equal(t, 1, len(entries)) 159 160 e := entries[0] 161 assert.Equal(t, "three/underthree.txt", e.Remote()) 162 assert.Equal(t, int64(8+lineEndSize), e.Size()) 163 _, ok := e.(*Object) 164 assert.True(t, ok) 165 } 166 167 func TestNewObject(t *testing.T) { 168 f := prepare(t) 169 170 o, err := f.NewObject(context.Background(), "four/under four.txt") 171 require.NoError(t, err) 172 173 assert.Equal(t, "four/under four.txt", o.Remote()) 174 assert.Equal(t, int64(8+lineEndSize), o.Size()) 175 _, ok := o.(*Object) 176 assert.True(t, ok) 177 178 // Test the time is correct on the object 179 180 tObj := o.ModTime(context.Background()) 181 182 fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt")) 183 require.NoError(t, err) 184 tFile := fi.ModTime() 185 186 fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second) 187 188 // check object not found 189 o, err = f.NewObject(context.Background(), "not found.txt") 190 assert.Nil(t, o) 191 assert.Equal(t, fs.ErrorObjectNotFound, err) 192 } 193 194 func TestOpen(t *testing.T) { 195 m := prepareServer(t) 196 197 for _, head := range []bool{false, true} { 198 if !head { 199 m.Set("no_head", "true") 200 } 201 f, err := NewFs(context.Background(), remoteName, "", m) 202 require.NoError(t, err) 203 204 for _, rangeRead := range []bool{false, true} { 205 o, err := f.NewObject(context.Background(), "four/under four.txt") 206 require.NoError(t, err) 207 208 if !head { 209 // Test mod time is still indeterminate 210 tObj := o.ModTime(context.Background()) 211 assert.Equal(t, time.Duration(0), time.Unix(0, 0).Sub(tObj)) 212 213 // Test file size is still indeterminate 214 assert.Equal(t, int64(-1), o.Size()) 215 } 216 217 var data []byte 218 if !rangeRead { 219 // Test normal read 220 fd, err := o.Open(context.Background()) 221 require.NoError(t, err) 222 data, err = io.ReadAll(fd) 223 require.NoError(t, err) 224 require.NoError(t, fd.Close()) 225 if lineEndSize == 2 { 226 assert.Equal(t, "beetroot\r\n", string(data)) 227 } else { 228 assert.Equal(t, "beetroot\n", string(data)) 229 } 230 } else { 231 // Test with range request 232 fd, err := o.Open(context.Background(), &fs.RangeOption{Start: 1, End: 5}) 233 require.NoError(t, err) 234 data, err = io.ReadAll(fd) 235 require.NoError(t, err) 236 require.NoError(t, fd.Close()) 237 assert.Equal(t, "eetro", string(data)) 238 } 239 240 fi, err := os.Stat(filepath.Join(filesPath, "four", "under four.txt")) 241 require.NoError(t, err) 242 tFile := fi.ModTime() 243 244 // Test the time is always correct on the object after file open 245 tObj := o.ModTime(context.Background()) 246 fstest.AssertTimeEqualWithPrecision(t, o.Remote(), tFile, tObj, time.Second) 247 248 if !rangeRead { 249 // Test the file size 250 assert.Equal(t, int64(len(data)), o.Size()) 251 } 252 } 253 } 254 } 255 256 func TestMimeType(t *testing.T) { 257 f := prepare(t) 258 259 o, err := f.NewObject(context.Background(), "four/under four.txt") 260 require.NoError(t, err) 261 262 do, ok := o.(fs.MimeTyper) 263 require.True(t, ok) 264 assert.Equal(t, "text/plain; charset=utf-8", do.MimeType(context.Background())) 265 } 266 267 func TestIsAFileRoot(t *testing.T) { 268 m := prepareServer(t) 269 270 f, err := NewFs(context.Background(), remoteName, "one%.txt", m) 271 assert.Equal(t, err, fs.ErrorIsFile) 272 273 testListRoot(t, f, false) 274 } 275 276 func TestIsAFileSubDir(t *testing.T) { 277 m := prepareServer(t) 278 279 f, err := NewFs(context.Background(), remoteName, "three/underthree.txt", m) 280 assert.Equal(t, err, fs.ErrorIsFile) 281 282 entries, err := f.List(context.Background(), "") 283 require.NoError(t, err) 284 285 sort.Sort(entries) 286 287 assert.Equal(t, 1, len(entries)) 288 289 e := entries[0] 290 assert.Equal(t, "underthree.txt", e.Remote()) 291 assert.Equal(t, int64(8+lineEndSize), e.Size()) 292 _, ok := e.(*Object) 293 assert.True(t, ok) 294 } 295 296 func TestParseName(t *testing.T) { 297 for i, test := range []struct { 298 base string 299 val string 300 wantErr error 301 want string 302 }{ 303 {"http://example.com/", "potato", nil, "potato"}, 304 {"http://example.com/dir/", "potato", nil, "potato"}, 305 {"http://example.com/dir/", "potato?download=true", errFoundQuestionMark, ""}, 306 {"http://example.com/dir/", "../dir/potato", nil, "potato"}, 307 {"http://example.com/dir/", "..", errNotUnderRoot, ""}, 308 {"http://example.com/dir/", "http://example.com/", errNotUnderRoot, ""}, 309 {"http://example.com/dir/", "http://example.com/dir/", errNameIsEmpty, ""}, 310 {"http://example.com/dir/", "http://example.com/dir/potato", nil, "potato"}, 311 {"http://example.com/dir/", "https://example.com/dir/potato", errSchemeMismatch, ""}, 312 {"http://example.com/dir/", "http://notexample.com/dir/potato", errHostMismatch, ""}, 313 {"http://example.com/dir/", "/dir/", errNameIsEmpty, ""}, 314 {"http://example.com/dir/", "/dir/potato", nil, "potato"}, 315 {"http://example.com/dir/", "subdir/potato", errNameContainsSlash, ""}, 316 {"http://example.com/dir/", "With percent %25.txt", nil, "With percent %.txt"}, 317 {"http://example.com/dir/", "With colon :", errURLJoinFailed, ""}, 318 {"http://example.com/dir/", rest.URLPathEscape("With colon :"), nil, "With colon :"}, 319 {"http://example.com/Dungeons%20%26%20Dragons/", "/Dungeons%20&%20Dragons/D%26D%20Basic%20%28Holmes%2C%20B%2C%20X%2C%20BECMI%29/", nil, "D&D Basic (Holmes, B, X, BECMI)/"}, 320 } { 321 u, err := url.Parse(test.base) 322 require.NoError(t, err) 323 got, gotErr := parseName(u, test.val) 324 what := fmt.Sprintf("test %d base=%q, val=%q", i, test.base, test.val) 325 assert.Equal(t, test.wantErr, gotErr, what) 326 assert.Equal(t, test.want, got, what) 327 } 328 } 329 330 // Load HTML from the file given and parse it, checking it against the entries passed in 331 func parseHTML(t *testing.T, name string, base string, want []string) { 332 in, err := os.Open(filepath.Join(testPath, "index_files", name)) 333 require.NoError(t, err) 334 defer func() { 335 require.NoError(t, in.Close()) 336 }() 337 if base == "" { 338 base = "http://example.com/" 339 } 340 u, err := url.Parse(base) 341 require.NoError(t, err) 342 entries, err := parse(u, in) 343 require.NoError(t, err) 344 assert.Equal(t, want, entries) 345 } 346 347 func TestParseEmpty(t *testing.T) { 348 parseHTML(t, "empty.html", "", []string(nil)) 349 } 350 351 func TestParseApache(t *testing.T) { 352 parseHTML(t, "apache.html", "http://example.com/nick/pub/", []string{ 353 "SWIG-embed.tar.gz", 354 "avi2dvd.pl", 355 "cambert.exe", 356 "cambert.gz", 357 "fedora_demo.gz", 358 "gchq-challenge/", 359 "mandelterm/", 360 "pgp-key.txt", 361 "pymath/", 362 "rclone", 363 "readdir.exe", 364 "rush_hour_solver_cut_down.py", 365 "snake-puzzle/", 366 "stressdisk/", 367 "timer-test", 368 "words-to-regexp.pl", 369 "Now 100% better.mp3", 370 "Now better.mp3", 371 }) 372 } 373 374 func TestParseMemstore(t *testing.T) { 375 parseHTML(t, "memstore.html", "", []string{ 376 "test/", 377 "v1.35/", 378 "v1.36-01-g503cd84/", 379 "rclone-beta-latest-freebsd-386.zip", 380 "rclone-beta-latest-freebsd-amd64.zip", 381 "rclone-beta-latest-windows-amd64.zip", 382 }) 383 } 384 385 func TestParseNginx(t *testing.T) { 386 parseHTML(t, "nginx.html", "", []string{ 387 "deltas/", 388 "objects/", 389 "refs/", 390 "state/", 391 "config", 392 "summary", 393 }) 394 } 395 396 func TestParseCaddy(t *testing.T) { 397 parseHTML(t, "caddy.html", "", []string{ 398 "mimetype.zip", 399 "rclone-delete-empty-dirs.py", 400 "rclone-show-empty-dirs.py", 401 "stat-windows-386.zip", 402 "v1.36-155-gcf29ee8b-team-driveβ/", 403 "v1.36-156-gca76b3fb-team-driveβ/", 404 "v1.36-156-ge1f0e0f5-team-driveβ/", 405 "v1.36-22-g06ea13a-ssh-agentβ/", 406 }) 407 } 408 409 func TestFsNoSlashRoots(t *testing.T) { 410 // Test Fs with roots that does not end with '/', the logic that 411 // decides if url is to be considered a file or directory, based 412 // on result from a HEAD request. 413 414 // Handler for faking HEAD responses with different status codes 415 headCount := 0 416 handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 417 if r.Method == "HEAD" { 418 headCount++ 419 responseCode, err := strconv.Atoi(path.Base(r.URL.String())) 420 require.NoError(t, err) 421 if strings.HasPrefix(r.URL.String(), "/redirect/") { 422 var redir string 423 if strings.HasPrefix(r.URL.String(), "/redirect/file/") { 424 redir = "/redirected" 425 } else if strings.HasPrefix(r.URL.String(), "/redirect/dir/") { 426 redir = "/redirected/" 427 } else { 428 require.Fail(t, "Redirect test requests must start with '/redirect/file/' or '/redirect/dir/'") 429 } 430 http.Redirect(w, r, redir, responseCode) 431 } else { 432 http.Error(w, http.StatusText(responseCode), responseCode) 433 } 434 } 435 }) 436 437 // Make the test server 438 ts := httptest.NewServer(handler) 439 defer ts.Close() 440 441 // Configure the remote 442 configfile.Install() 443 m := configmap.Simple{ 444 "type": "http", 445 "url": ts.URL, 446 } 447 448 // Test 449 for i, test := range []struct { 450 root string 451 isFile bool 452 }{ 453 // 2xx success 454 {"parent/200", true}, 455 {"parent/204", true}, 456 457 // 3xx redirection Redirect status 301, 302, 303, 307, 308 458 {"redirect/file/301", true}, // Request is redirected to "/redirected" 459 {"redirect/dir/301", false}, // Request is redirected to "/redirected/" 460 {"redirect/file/302", true}, // Request is redirected to "/redirected" 461 {"redirect/dir/302", false}, // Request is redirected to "/redirected/" 462 {"redirect/file/303", true}, // Request is redirected to "/redirected" 463 {"redirect/dir/303", false}, // Request is redirected to "/redirected/" 464 465 {"redirect/file/304", true}, // Not really a redirect, handled like 4xx errors (below) 466 {"redirect/file/305", true}, // Not really a redirect, handled like 4xx errors (below) 467 {"redirect/file/306", true}, // Not really a redirect, handled like 4xx errors (below) 468 469 {"redirect/file/307", true}, // Request is redirected to "/redirected" 470 {"redirect/dir/307", false}, // Request is redirected to "/redirected/" 471 {"redirect/file/308", true}, // Request is redirected to "/redirected" 472 {"redirect/dir/308", false}, // Request is redirected to "/redirected/" 473 474 // 4xx client errors 475 {"parent/403", true}, // Forbidden status (head request blocked) 476 {"parent/404", false}, // Not found status 477 } { 478 for _, noHead := range []bool{false, true} { 479 var isFile bool 480 if noHead { 481 m.Set("no_head", "true") 482 isFile = true 483 } else { 484 m.Set("no_head", "false") 485 isFile = test.isFile 486 } 487 headCount = 0 488 f, err := NewFs(context.Background(), remoteName, test.root, m) 489 if noHead { 490 assert.Equal(t, 0, headCount) 491 } else { 492 assert.Equal(t, 1, headCount) 493 } 494 if isFile { 495 assert.ErrorIs(t, err, fs.ErrorIsFile) 496 } else { 497 assert.NoError(t, err) 498 } 499 var endpoint string 500 if isFile { 501 parent, _ := path.Split(test.root) 502 endpoint = "/" + parent 503 } else { 504 endpoint = "/" + test.root + "/" 505 } 506 what := fmt.Sprintf("i=%d, root=%q, isFile=%v, noHead=%v", i, test.root, isFile, noHead) 507 assert.Equal(t, ts.URL+endpoint, f.String(), what) 508 } 509 } 510 }