github.com/useflyent/fhttp@v0.0.0-20211004035111-333f430cfbbf/example_client_test.go (about) 1 package http_test 2 3 import ( 4 "crypto/tls" 5 "crypto/x509" 6 "encoding/json" 7 "flag" 8 "fmt" 9 "io" 10 "net/url" 11 "os" 12 "strings" 13 "testing" 14 15 http "github.com/useflyent/fhttp" 16 "github.com/useflyent/fhttp/http2" 17 ) 18 19 // Basic http test with Header Order + enable push 20 func TestExample(t *testing.T) { 21 c := http.Client{} 22 23 req, err := http.NewRequest("GET", "https://httpbin.org/headers", strings.NewReader("")) 24 25 if err != nil { 26 t.Errorf(err.Error()) 27 return 28 } 29 30 req.Header = http.Header{ 31 "sec-ch-ua": {"\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"90\", \"Google Chrome\";v=\"90\""}, 32 "sec-ch-ua-mobile": {"?0"}, 33 "upgrade-insecure-requests": {"1"}, 34 "user-agent": {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"}, 35 "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"}, 36 "sec-fetch-site": {"none"}, 37 "sec-fetch-mode": {"navigate"}, 38 "sec-fetch-user": {"?1"}, 39 "sec-fetch-dest": {"document"}, 40 "accept-encoding": {"gzip, deflate, br"}, 41 http.HeaderOrderKey: { 42 "sec-ch-ua", 43 "sec-ch-ua-mobile", 44 "upgrade-insecure-requests", 45 "user-agent", 46 "accept", 47 "sec-fetch-site", 48 "sec-fetch-mode", 49 "sec-fetch-user", 50 "sec-fetch-dest", 51 "accept-encoding", 52 }, 53 } 54 resp, err := c.Do(req) 55 if err != nil { 56 t.Errorf(err.Error()) 57 return 58 } 59 defer resp.Body.Close() 60 if resp.StatusCode != 200 { 61 t.Errorf("Expected status code 200, got %v", resp.StatusCode) 62 } 63 64 var data interface{} 65 err = json.NewDecoder(resp.Body).Decode(&data) 66 if err != nil { 67 t.Errorf(err.Error()) 68 } 69 } 70 71 func getCharlesCert() (*x509.CertPool, error) { 72 home, err := os.UserHomeDir() 73 if err != nil { 74 return nil, err 75 } 76 caCert, err := os.ReadFile(fmt.Sprintf("%v/charles_cert.pem", home)) 77 if err != nil { 78 return nil, err 79 } 80 certPool := x509.NewCertPool() 81 certPool.AppendCertsFromPEM(caCert) 82 return certPool, nil 83 } 84 85 func addCharlesToTransport(tr *http.Transport, proxy string) error { 86 caCertPool, err := getCharlesCert() 87 if err != nil { 88 return err 89 } 90 proxyURL, err := url.Parse(proxy) 91 if err != nil { 92 return err 93 } 94 tr.TLSClientConfig = &tls.Config{ 95 RootCAs: caCertPool, 96 } 97 tr.Proxy = http.ProxyURL(proxyURL) 98 99 return nil 100 } 101 102 func addWiresharkToTransport(tr *http.Transport) error { 103 kl := flag.String("keylog", "ssl-keylog.txt", "file to dump ssl keys") 104 keylog, err := os.OpenFile(*kl, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) 105 if err != nil { 106 return err 107 } 108 tr.TLSClientConfig = &tls.Config{ 109 InsecureSkipVerify: true, 110 KeyLogWriter: keylog, 111 } 112 return nil 113 } 114 115 // Test with Charles cert + proxy 116 func TestWithCert(t *testing.T) { 117 h1t := &http.Transport{ 118 ForceAttemptHTTP2: true, 119 } 120 if err := addCharlesToTransport(h1t, "http://localhost:8888"); err != nil { 121 t.Fatalf(err.Error()) 122 } 123 124 t2, err := http2.ConfigureTransports(h1t) 125 if err != nil { 126 t.Fatalf(err.Error()) 127 } 128 t2.Settings = []http2.Setting{ 129 {ID: http2.SettingMaxConcurrentStreams, Val: 1000}, 130 {ID: http2.SettingMaxFrameSize, Val: 16384}, 131 {ID: http2.SettingMaxHeaderListSize, Val: 262144}, 132 } 133 t2.InitialWindowSize = 6291456 134 t2.HeaderTableSize = 65536 135 h1t.H2transport = t2 136 137 client := http.Client{ 138 Transport: h1t, 139 } 140 141 req, err := http.NewRequest("GET", "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding", nil) 142 if err != nil { 143 t.Errorf(err.Error()) 144 return 145 } 146 147 req.Header = http.Header{ 148 "sec-ch-ua": {"\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"90\", \"Google Chrome\";v=\"90\""}, 149 "sec-ch-ua-mobile": {"?0"}, 150 "upgrade-insecure-requests": {"1"}, 151 "user-agent": {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36", "I shouldn't be here"}, 152 "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"}, 153 "sec-fetch-site": {"none"}, 154 "sec-fetch-mode": {"navigate"}, 155 "sec-fetch-user": {"?1"}, 156 "cookie": {"cf_clearance=67f509a97bae8bb8349523a14c0ca3d7d8460c93-1620778862-0-250", "wp_customerGroup=NOT+LOGGED+IN"}, 157 "sec-fetch-dest": {"document"}, 158 "accept-encoding": {"gzip, deflate, br"}, 159 "not-included-header": {"should be last"}, 160 http.HeaderOrderKey: { 161 "sec-ch-ua", 162 "sec-ch-ua-mobile", 163 "upgrade-insecure-requests", 164 "user-agent", 165 "cookie", 166 "accept", 167 "sec-fetch-site", 168 "sec-fetch-mode", 169 "sec-fetch-user", 170 "sec-fetch-dest", 171 "accept-encoding", 172 }, 173 http.PHeaderOrderKey: {":method", ":authority", ":scheme", ":path"}, 174 } 175 176 resp, err := client.Do(req) 177 if err != nil { 178 t.Errorf(err.Error()) 179 return 180 } 181 defer resp.Body.Close() 182 183 if resp.StatusCode != 200 { 184 t.Errorf("Expected status code 200, got %v", resp.StatusCode) 185 } 186 } 187 188 // Test with push handler 189 func TestEnablePush(t *testing.T) { 190 t1 := &http.Transport{ 191 ForceAttemptHTTP2: true, 192 } 193 t2, err := http2.ConfigureTransports(t1) 194 if err != nil { 195 t.Fatalf(err.Error()) 196 } 197 t2.PushHandler = &http2.DefaultPushHandler{} 198 t1.H2transport = t2 199 c := &http.Client{ 200 Transport: t1, 201 } 202 var req *http.Request 203 req, err = http.NewRequest("GET", "https://httpbin.org/headers", nil) 204 if err != nil { 205 t.Fatalf(err.Error()) 206 } 207 _, err = c.Do(req) 208 if err != nil { 209 t.Fatalf(err.Error()) 210 } 211 212 req, err = http.NewRequest("POST", "https://httpbin.org/post", nil) 213 if err != nil { 214 t.Fatalf(err.Error()) 215 } 216 _, err = c.Do(req) 217 if err != nil { 218 t.Fatalf(err.Error()) 219 } 220 } 221 222 // Test finishline 223 func TestFinishLine(t *testing.T) { 224 t1 := &http.Transport{ 225 ForceAttemptHTTP2: true, 226 } 227 228 if err := addCharlesToTransport(t1, "http://localhost:8888"); err != nil { 229 t.Fatalf(err.Error()) 230 } 231 // if err := addWiresharkToTransport(t1); err != nil { 232 // t.Fatalf(err.Error()) 233 // } 234 t2, err := http2.ConfigureTransports(t1) 235 if err != nil { 236 t.Fatalf(err.Error()) 237 } 238 t2.Settings = []http2.Setting{ 239 {ID: http2.SettingMaxConcurrentStreams, Val: 1000}, 240 {ID: http2.SettingMaxFrameSize, Val: 16384}, 241 {ID: http2.SettingMaxHeaderListSize, Val: 262144}, 242 } 243 t2.InitialWindowSize = 6291456 244 t2.HeaderTableSize = 65536 245 t2.PushHandler = &http2.DefaultPushHandler{} 246 t1.H2transport = t2 247 248 c := &http.Client{ 249 Transport: t1, 250 } 251 req, err := http.NewRequest("GET", "https://www.finishline.com/", nil) 252 if err != nil { 253 t.Fatalf(err.Error()) 254 } 255 req.Header = http.Header{ 256 "sec-ch-ua": {"\" Not A;Brand\";v=\"99\", \"Chromium\";v=\"90\", \"Google Chrome\";v=\"90\""}, 257 "sec-ch-ua-mobile": {"?0"}, 258 "upgrade-insecure-requests": {"1"}, 259 "user-agent": {"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"}, 260 "accept": {"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9"}, 261 "sec-fetch-site": {"none"}, 262 "sec-fetch-mode": {"navigate"}, 263 "sec-fetch-user": {"?1"}, 264 "sec-fetch-dest": {"document"}, 265 "accept-encoding": {"gzip, deflate, br"}, 266 http.HeaderOrderKey: { 267 "sec-ch-ua", 268 "sec-ch-ua-mobile", 269 "upgrade-insecure-requests", 270 "user-agent", 271 "accept", 272 "sec-fetch-site", 273 "sec-fetch-mode", 274 "sec-fetch-user", 275 "sec-fetch-dest", 276 "accept-encoding", 277 }, 278 http.PHeaderOrderKey: {":method", ":authority", ":scheme", ":path"}, 279 } 280 resp, err := c.Do(req) 281 if err != nil { 282 t.Fatalf(err.Error()) 283 } 284 defer resp.Body.Close() 285 if resp.StatusCode != 200 { 286 t.Fatalf("Got status %v from finishline, expected 200", resp.StatusCode) 287 } 288 b, err := io.ReadAll(resp.Body) 289 if err != nil { 290 t.Fatalf(err.Error()) 291 } 292 fmt.Printf("resp: %v\n", string(b)[1]) 293 } 294 295 // Test compression brotli 296 func TestCompressionBrotli(t *testing.T) { 297 t1 := &http.Transport{ 298 ForceAttemptHTTP2: true, 299 } 300 c := http.Client{ 301 Transport: t1, 302 } 303 req, _ := http.NewRequest("GET", "https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Encoding", nil) 304 req.Header = http.Header{ 305 "accept-encoding": {"br"}, 306 } 307 resp, err := c.Do(req) 308 if err != nil { 309 t.Fatalf(err.Error()) 310 } 311 312 if h := resp.Header.Get("content-encoding"); h == "" || h != "br" { 313 t.Fatalf("Got content-encoding header %v, expected br", h) 314 } 315 } 316 317 // Test compression zlib deflate 318 func TestCompressionZlibDeflate(t *testing.T) { 319 t1 := &http.Transport{ 320 ForceAttemptHTTP2: true, 321 } 322 addCharlesToTransport(t1, "http://localhost:8888") 323 c := http.Client{ 324 Transport: t1, 325 } 326 req, _ := http.NewRequest("GET", "http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on&zlib=on", nil) 327 req.Header = http.Header{ 328 "accept-encoding": {"deflate"}, 329 } 330 resp, err := c.Do(req) 331 if err != nil { 332 t.Fatalf(err.Error()) 333 } 334 335 if h := resp.Header.Get("content-encoding"); h == "" || h != "deflate" { 336 t.Fatalf("Expected content encoding deflate, got %v", h) 337 } 338 } 339 340 // Test compression deflate 341 func TestCompressionDeflate(t *testing.T) { 342 c := http.Client{} 343 req, _ := http.NewRequest("GET", "http://carsten.codimi.de/gzip.yaws/daniels.html?deflate=on", nil) 344 req.Header = http.Header{ 345 "accept-encoding": {"deflate"}, 346 } 347 resp, err := c.Do(req) 348 if err != nil { 349 t.Fatalf(err.Error()) 350 } 351 if h := resp.Header.Get("content-encoding"); h == "" || h != "deflate" { 352 t.Fatalf("Expected content encoding deflate, got %v", h) 353 } 354 } 355 356 // Test with cookies 357 // Test with missing in header order, that should be added 358 // Test for UA that has empty string, excluding UA from being part of headers