github.com/htcondor/osdf-client/v6@v6.13.0-rc1.0.20231009141709-766e7b4d1dc8/handle_http_test.go (about) 1 package stashcp 2 3 import ( 4 "bytes" 5 "net" 6 "net/http" 7 "net/http/httptest" 8 "net/url" 9 "os" 10 "path/filepath" 11 "strings" 12 "testing" 13 "time" 14 "github.com/stretchr/testify/assert" 15 16 "net/http/httputil" 17 18 namespaces "github.com/htcondor/osdf-client/v6/namespaces" 19 ) 20 21 // TestIsPort calls main.hasPort with a hostname, checking 22 // for a valid return value. 23 func TestIsPort(t *testing.T) { 24 25 if HasPort("blah.not.port:") { 26 t.Fatal("Failed to parse port when : at end") 27 } 28 29 if !HasPort("host:1") { 30 t.Fatal("Failed to parse with port = 1") 31 } 32 33 if HasPort("https://example.com") { 34 t.Fatal("Failed when scheme is specified") 35 } 36 } 37 38 // TestNewTransferDetails checks the creation of transfer details 39 func TestNewTransferDetails(t *testing.T) { 40 os.Setenv("http_proxy", "http://proxy.edu:3128") 41 42 // Case 1: cache with http 43 testCache := namespaces.Cache{ 44 AuthEndpoint: "cache.edu:8443", 45 Endpoint: "cache.edu:8000", 46 Resource: "Cache", 47 } 48 transfers := NewTransferDetails(testCache, false) 49 assert.Equal(t, 2, len(transfers)) 50 assert.Equal(t, "cache.edu:8000", transfers[0].Url.Host) 51 assert.Equal(t, "http", transfers[0].Url.Scheme) 52 assert.Equal(t, true, transfers[0].Proxy) 53 assert.Equal(t, "cache.edu:8000", transfers[1].Url.Host) 54 assert.Equal(t, "http", transfers[1].Url.Scheme) 55 assert.Equal(t, false, transfers[1].Proxy) 56 57 // Case 2: cache with https 58 transfers = NewTransferDetails(testCache, true) 59 assert.Equal(t, 1, len(transfers)) 60 assert.Equal(t, "cache.edu:8443", transfers[0].Url.Host) 61 assert.Equal(t, "https", transfers[0].Url.Scheme) 62 assert.Equal(t, false, transfers[0].Proxy) 63 64 testCache.Endpoint = "cache.edu" 65 // Case 3: cache without port with http 66 transfers = NewTransferDetails(testCache, false) 67 assert.Equal(t, 2, len(transfers)) 68 assert.Equal(t, "cache.edu:8000", transfers[0].Url.Host) 69 assert.Equal(t, "http", transfers[0].Url.Scheme) 70 assert.Equal(t, true, transfers[0].Proxy) 71 assert.Equal(t, "cache.edu:8000", transfers[1].Url.Host) 72 assert.Equal(t, "http", transfers[1].Url.Scheme) 73 assert.Equal(t, false, transfers[1].Proxy) 74 75 // Case 4. cache without port with https 76 testCache.AuthEndpoint = "cache.edu" 77 transfers = NewTransferDetails(testCache, true) 78 assert.Equal(t, 2, len(transfers)) 79 assert.Equal(t, "cache.edu:8444", transfers[0].Url.Host) 80 assert.Equal(t, "https", transfers[0].Url.Scheme) 81 assert.Equal(t, false, transfers[0].Proxy) 82 assert.Equal(t, "cache.edu:8443", transfers[1].Url.Host) 83 assert.Equal(t, "https", transfers[1].Url.Scheme) 84 assert.Equal(t, false, transfers[1].Proxy) 85 } 86 87 func TestNewTransferDetailsEnv(t *testing.T) { 88 89 testCache := namespaces.Cache{ 90 AuthEndpoint: "cache.edu:8443", 91 Endpoint: "cache.edu:8000", 92 Resource: "Cache", 93 } 94 95 os.Setenv("OSG_DISABLE_PROXY_FALLBACK", "") 96 transfers := NewTransferDetails(testCache, false) 97 assert.Equal(t, 1, len(transfers)) 98 assert.Equal(t, true, transfers[0].Proxy) 99 100 transfers = NewTransferDetails(testCache, true) 101 assert.Equal(t, 1, len(transfers)) 102 assert.Equal(t, "https", transfers[0].Url.Scheme) 103 assert.Equal(t, false, transfers[0].Proxy) 104 os.Unsetenv("OSG_DISABLE_PROXY_FALLBACK") 105 } 106 107 func TestSlowTransfers(t *testing.T) { 108 channel := make(chan bool) 109 slowDownload := 1024 * 10 // 10 KiB/s < 100 KiB/s 110 svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 111 buffer := make([]byte, slowDownload) 112 for { 113 select { 114 case <-channel: 115 return 116 default: 117 _, err := w.Write(buffer) 118 if err != nil { 119 return 120 } 121 w.(http.Flusher).Flush() 122 time.Sleep(1 * time.Second) 123 } 124 } 125 })) 126 127 defer svr.CloseClientConnections() 128 defer svr.Close() 129 130 testCache := namespaces.Cache{ 131 AuthEndpoint: svr.URL, 132 Endpoint: svr.URL, 133 Resource: "Cache", 134 } 135 transfers := NewTransferDetails(testCache, false) 136 assert.Equal(t, 2, len(transfers)) 137 assert.Equal(t, svr.URL, transfers[0].Url.String()) 138 139 finishedChannel := make(chan bool) 140 var err error 141 // Do a quick timeout 142 go func() { 143 _, err = DownloadHTTP(transfers[0], filepath.Join(t.TempDir(), "test.txt"), "") 144 finishedChannel <- true 145 }() 146 147 select { 148 case <-finishedChannel: 149 if err == nil { 150 t.Fatal("Error is nil, download should have failed") 151 } 152 case <-time.After(time.Second * 160): 153 // 120 seconds for warmup, 30 seconds for download 154 t.Fatal("Maximum downloading time reach, download should have failed") 155 } 156 157 // Close the channel to allow the download to complete 158 channel <- true 159 160 // Make sure the errors are correct 161 assert.NotNil(t, err) 162 assert.IsType(t, &SlowTransferError{}, err) 163 } 164 165 // Test stopped transfer 166 func TestStoppedTransfer(t *testing.T) { 167 channel := make(chan bool) 168 svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 169 buffer := make([]byte, 1024 * 100) 170 for { 171 select { 172 case <-channel: 173 return 174 default: 175 _, err := w.Write(buffer) 176 if err != nil { 177 return 178 } 179 w.(http.Flusher).Flush() 180 time.Sleep(1 * time.Second) 181 buffer = make([]byte, 0) 182 } 183 } 184 })) 185 186 defer svr.CloseClientConnections() 187 defer svr.Close() 188 189 testCache := namespaces.Cache{ 190 AuthEndpoint: svr.URL, 191 Endpoint: svr.URL, 192 Resource: "Cache", 193 } 194 transfers := NewTransferDetails(testCache, false) 195 assert.Equal(t, 2, len(transfers)) 196 assert.Equal(t, svr.URL, transfers[0].Url.String()) 197 198 finishedChannel := make(chan bool) 199 var err error 200 201 go func() { 202 _, err = DownloadHTTP(transfers[0], filepath.Join(t.TempDir(), "test.txt"), "") 203 finishedChannel <- true 204 }() 205 206 select { 207 case <-finishedChannel: 208 if err == nil { 209 t.Fatal("Download should have failed") 210 } 211 case <-time.After(time.Second * 150): 212 t.Fatal("Download should have failed") 213 } 214 215 // Close the channel to allow the download to complete 216 channel <- true 217 218 // Make sure the errors are correct 219 assert.NotNil(t, err) 220 assert.IsType(t, &StoppedTransferError{}, err, err.Error()) 221 } 222 223 224 // Test connection error 225 func TestConnectionError(t *testing.T) { 226 l, err := net.Listen("tcp", "127.0.0.1:0") 227 if err != nil { 228 t.Fatalf("dialClosedPort: Listen failed: %v", err) 229 } 230 addr := l.Addr().String() 231 l.Close() 232 233 _, err = DownloadHTTP(TransferDetails{Url: url.URL{Host: addr, Scheme: "http"}, Proxy: false}, filepath.Join(t.TempDir(), "test.txt"), "") 234 235 assert.IsType(t, &ConnectionSetupError{}, err) 236 237 } 238 239 func TestTrailerError(t *testing.T) { 240 // Set up an HTTP server that returns an error trailer 241 svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 242 w.Header().Set("Trailer", "X-Transfer-Status") 243 w.Header().Set("X-Transfer-Status", "500: Unable to read test.txt; input/output error") 244 245 chunkedWriter := httputil.NewChunkedWriter(w) 246 defer chunkedWriter.Close() 247 248 _, err := chunkedWriter.Write([]byte("Test data")) 249 if err != nil { 250 t.Fatalf("Error writing to chunked writer: %v", err) 251 } 252 })) 253 254 defer svr.Close() 255 256 testCache := namespaces.Cache{ 257 AuthEndpoint: svr.URL, 258 Endpoint: svr.URL, 259 Resource: "Cache", 260 } 261 transfers := NewTransferDetails(testCache, false) 262 assert.Equal(t, 2, len(transfers)) 263 assert.Equal(t, svr.URL, transfers[0].Url.String()) 264 265 // Call DownloadHTTP and check if the error is returned correctly 266 _, err := DownloadHTTP(transfers[0], filepath.Join(t.TempDir(), "test.txt"), "") 267 268 assert.NotNil(t, err) 269 assert.EqualError(t, err, "transfer error: Unable to read test.txt; input/output error") 270 } 271 272 func TestUploadZeroLengthFile(t *testing.T) { 273 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 274 275 //t.Logf("%s", dump) 276 assert.Equal(t, "PUT", r.Method, "Not PUT Method") 277 assert.Equal(t, int64(0), r.ContentLength, "ContentLength should be 0") 278 })) 279 defer ts.Close() 280 reader := bytes.NewReader([]byte{}) 281 request, err := http.NewRequest("PUT", ts.URL, reader) 282 if err != nil { 283 assert.NoError(t, err) 284 } 285 286 request.Header.Set("Authorization", "Bearer test") 287 errorChan := make(chan error, 1) 288 responseChan := make(chan *http.Response) 289 go doPut(request, responseChan, errorChan) 290 select { 291 case err := <-errorChan: 292 assert.NoError(t, err) 293 case response := <-responseChan: 294 assert.Equal(t, http.StatusOK, response.StatusCode) 295 case <-time.After(time.Second * 2): 296 assert.Fail(t, "Timeout while waiting for response") 297 } 298 } 299 300 func TestFailedUpload(t *testing.T) { 301 ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 302 303 //t.Logf("%s", dump) 304 assert.Equal(t, "PUT", r.Method, "Not PUT Method") 305 w.WriteHeader(http.StatusInternalServerError) 306 _, err := w.Write([]byte("Error")) 307 assert.NoError(t, err) 308 309 })) 310 defer ts.Close() 311 reader := strings.NewReader("test") 312 request, err := http.NewRequest("PUT", ts.URL, reader) 313 if err != nil { 314 assert.NoError(t, err) 315 } 316 request.Header.Set("Authorization", "Bearer test") 317 errorChan := make(chan error, 1) 318 responseChan := make(chan *http.Response) 319 go doPut(request, responseChan, errorChan) 320 select { 321 case err := <-errorChan: 322 assert.Error(t, err) 323 case response := <-responseChan: 324 assert.Equal(t, http.StatusInternalServerError, response.StatusCode) 325 case <-time.After(time.Second * 2): 326 assert.Fail(t, "Timeout while waiting for response") 327 } 328 } 329 330 func TestFullUpload(t *testing.T) { 331 testFileContent := "test file content" 332 ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 333 334 //t.Logf("%s", dump) 335 assert.Equal(t, "PUT", r.Method, "Not PUT Method") 336 _, err := w.Write([]byte(":)")) 337 assert.NoError(t, err) 338 })) 339 defer ts.Close() 340 341 // Create the temporary file to upload 342 tempFile, err := os.CreateTemp(t.TempDir(), "test") 343 assert.NoError(t, err, "Error creating temp file") 344 defer os.Remove(tempFile.Name()) 345 _, err = tempFile.WriteString(testFileContent) 346 assert.NoError(t, err, "Error writing to temp file") 347 tempFile.Close() 348 349 // Create the namespace (only the write back host is read) 350 testURL, err := url.Parse(ts.URL) 351 assert.NoError(t, err, "Error parsing test URL") 352 testNamespace := namespaces.Namespace{ 353 WriteBackHost: "https://" + testURL.Host, 354 } 355 356 // Upload the file 357 uploadURL, err := url.Parse("stash:///test/stuff/blah.txt") 358 assert.NoError(t, err, "Error parsing upload URL") 359 // Set the upload client to trust the server 360 UploadClient = ts.Client() 361 uploaded, err := UploadFile(tempFile.Name(), uploadURL, "Bearer test", testNamespace) 362 assert.NoError(t, err, "Error uploading file") 363 assert.Equal(t, int64(len(testFileContent)), uploaded, "Uploaded file size does not match") 364 365 // Upload an osdf file 366 uploadURL, err = url.Parse("osdf:///test/stuff/blah.txt") 367 assert.NoError(t, err, "Error parsing upload URL") 368 // Set the upload client to trust the server 369 UploadClient = ts.Client() 370 uploaded, err = UploadFile(tempFile.Name(), uploadURL, "Bearer test", testNamespace) 371 assert.NoError(t, err, "Error uploading file") 372 assert.Equal(t, int64(len(testFileContent)), uploaded, "Uploaded file size does not match") 373 } 374