github.com/nwidger/bat@v0.0.2/httplib/httplib.go (about) 1 // Copyright 2014 beego Author. All Rights Reserved. 2 // Copyright 2015 bat authors 3 // 4 // Licensed under the Apache License, Version 2.0 (the "License"); 5 // you may not use this file except in compliance with the License. 6 // You may obtain a copy of the License at 7 // 8 // http://www.apache.org/licenses/LICENSE-2.0 9 // 10 // Unless required by applicable law or agreed to in writing, software 11 // distributed under the License is distributed on an "AS IS" BASIS, 12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 // See the License for the specific language governing permissions and 14 // limitations under the License. 15 16 // Usage: 17 // 18 // import "github.com/astaxie/beego/httplib" 19 // 20 // b := httplib.Post("http://beego.me/") 21 // b.Param("username","astaxie") 22 // b.Param("password","123456") 23 // b.PostFile("uploadfile1", "httplib.pdf") 24 // b.PostFile("uploadfile2", "httplib.txt") 25 // str, err := b.String() 26 // if err != nil { 27 // t.Fatal(err) 28 // } 29 // fmt.Println(str) 30 // 31 // more docs http://beego.me/docs/module/httplib.md 32 package httplib 33 34 import ( 35 "bytes" 36 "compress/gzip" 37 "crypto/tls" 38 "encoding/json" 39 "encoding/xml" 40 "io" 41 "io/ioutil" 42 "log" 43 "mime/multipart" 44 "net" 45 "net/http" 46 "net/http/cookiejar" 47 "net/http/httputil" 48 "net/url" 49 "os" 50 "strings" 51 "sync" 52 "time" 53 ) 54 55 var defaultSetting = BeegoHttpSettings{false, "beegoServer", 60 * time.Second, 60 * time.Second, nil, nil, nil, false, true, true} 56 var defaultCookieJar http.CookieJar 57 var settingMutex sync.Mutex 58 59 // createDefaultCookie creates a global cookiejar to store cookies. 60 func createDefaultCookie() { 61 settingMutex.Lock() 62 defer settingMutex.Unlock() 63 defaultCookieJar, _ = cookiejar.New(nil) 64 } 65 66 // Overwrite default settings 67 func SetDefaultSetting(setting BeegoHttpSettings) { 68 settingMutex.Lock() 69 defer settingMutex.Unlock() 70 defaultSetting = setting 71 if defaultSetting.ConnectTimeout == 0 { 72 defaultSetting.ConnectTimeout = 60 * time.Second 73 } 74 if defaultSetting.ReadWriteTimeout == 0 { 75 defaultSetting.ReadWriteTimeout = 60 * time.Second 76 } 77 } 78 79 // return *BeegoHttpRequest with specific method 80 func NewBeegoRequest(rawurl, method string) *BeegoHttpRequest { 81 var resp http.Response 82 u, err := url.Parse(rawurl) 83 if err != nil { 84 log.Fatal(err) 85 } 86 req := http.Request{ 87 URL: u, 88 Method: method, 89 Header: make(http.Header), 90 Proto: "HTTP/1.1", 91 ProtoMajor: 1, 92 ProtoMinor: 1, 93 } 94 return &BeegoHttpRequest{rawurl, &req, map[string]string{}, map[string]string{}, defaultSetting, &resp, nil, nil} 95 } 96 97 // Get returns *BeegoHttpRequest with GET method. 98 func Get(url string) *BeegoHttpRequest { 99 return NewBeegoRequest(url, "GET") 100 } 101 102 // Post returns *BeegoHttpRequest with POST method. 103 func Post(url string) *BeegoHttpRequest { 104 return NewBeegoRequest(url, "POST") 105 } 106 107 // Put returns *BeegoHttpRequest with PUT method. 108 func Put(url string) *BeegoHttpRequest { 109 return NewBeegoRequest(url, "PUT") 110 } 111 112 // Delete returns *BeegoHttpRequest DELETE method. 113 func Delete(url string) *BeegoHttpRequest { 114 return NewBeegoRequest(url, "DELETE") 115 } 116 117 // Head returns *BeegoHttpRequest with HEAD method. 118 func Head(url string) *BeegoHttpRequest { 119 return NewBeegoRequest(url, "HEAD") 120 } 121 122 // BeegoHttpSettings 123 type BeegoHttpSettings struct { 124 ShowDebug bool 125 UserAgent string 126 ConnectTimeout time.Duration 127 ReadWriteTimeout time.Duration 128 TlsClientConfig *tls.Config 129 Proxy func(*http.Request) (*url.URL, error) 130 Transport http.RoundTripper 131 EnableCookie bool 132 Gzip bool 133 DumpBody bool 134 } 135 136 // BeegoHttpRequest provides more useful methods for requesting one url than http.Request. 137 type BeegoHttpRequest struct { 138 url string 139 req *http.Request 140 params map[string]string 141 files map[string]string 142 setting BeegoHttpSettings 143 resp *http.Response 144 body []byte 145 dump []byte 146 } 147 148 // get request 149 func (b *BeegoHttpRequest) GetRequest() *http.Request { 150 return b.req 151 } 152 153 // Change request settings 154 func (b *BeegoHttpRequest) Setting(setting BeegoHttpSettings) *BeegoHttpRequest { 155 b.setting = setting 156 return b 157 } 158 159 // SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password. 160 func (b *BeegoHttpRequest) SetBasicAuth(username, password string) *BeegoHttpRequest { 161 b.req.SetBasicAuth(username, password) 162 return b 163 } 164 165 // SetEnableCookie sets enable/disable cookiejar 166 func (b *BeegoHttpRequest) SetEnableCookie(enable bool) *BeegoHttpRequest { 167 b.setting.EnableCookie = enable 168 return b 169 } 170 171 // SetUserAgent sets User-Agent header field 172 func (b *BeegoHttpRequest) SetUserAgent(useragent string) *BeegoHttpRequest { 173 b.setting.UserAgent = useragent 174 return b 175 } 176 177 // Debug sets show debug or not when executing request. 178 func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest { 179 b.setting.ShowDebug = isdebug 180 return b 181 } 182 183 // Dump Body. 184 func (b *BeegoHttpRequest) DumpBody(isdump bool) *BeegoHttpRequest { 185 b.setting.DumpBody = isdump 186 return b 187 } 188 189 // return the DumpRequest 190 func (b *BeegoHttpRequest) DumpRequest() []byte { 191 return b.dump 192 } 193 194 // SetTimeout sets connect time out and read-write time out for BeegoRequest. 195 func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest { 196 b.setting.ConnectTimeout = connectTimeout 197 b.setting.ReadWriteTimeout = readWriteTimeout 198 return b 199 } 200 201 // SetTLSClientConfig sets tls connection configurations if visiting https url. 202 func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest { 203 b.setting.TlsClientConfig = config 204 return b 205 } 206 207 // Header add header item string in request. 208 func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest { 209 b.req.Header.Set(key, value) 210 return b 211 } 212 213 // Set HOST 214 func (b *BeegoHttpRequest) SetHost(host string) *BeegoHttpRequest { 215 b.req.Host = host 216 return b 217 } 218 219 // Set the protocol version for incoming requests. 220 // Client requests always use HTTP/1.1. 221 func (b *BeegoHttpRequest) SetProtocolVersion(vers string) *BeegoHttpRequest { 222 if len(vers) == 0 { 223 vers = "HTTP/1.1" 224 } 225 226 major, minor, ok := http.ParseHTTPVersion(vers) 227 if ok { 228 b.req.Proto = vers 229 b.req.ProtoMajor = major 230 b.req.ProtoMinor = minor 231 } 232 233 return b 234 } 235 236 // SetCookie add cookie into request. 237 func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest { 238 b.req.Header.Add("Cookie", cookie.String()) 239 return b 240 } 241 242 // Set transport to 243 func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest { 244 b.setting.Transport = transport 245 return b 246 } 247 248 // Set http proxy 249 // example: 250 // 251 // func(req *http.Request) (*url.URL, error) { 252 // u, _ := url.ParseRequestURI("http://127.0.0.1:8118") 253 // return u, nil 254 // } 255 func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest { 256 b.setting.Proxy = proxy 257 return b 258 } 259 260 // Param adds query param in to request. 261 // params build query string as ?key1=value1&key2=value2... 262 func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest { 263 b.params[key] = value 264 return b 265 } 266 267 func (b *BeegoHttpRequest) PostFile(formname, filename string) *BeegoHttpRequest { 268 b.files[formname] = filename 269 return b 270 } 271 272 // Body adds request raw body. 273 // it supports string and []byte. 274 func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest { 275 switch t := data.(type) { 276 case string: 277 bf := bytes.NewBufferString(t) 278 b.req.Body = ioutil.NopCloser(bf) 279 b.req.ContentLength = int64(len(t)) 280 case []byte: 281 bf := bytes.NewBuffer(t) 282 b.req.Body = ioutil.NopCloser(bf) 283 b.req.ContentLength = int64(len(t)) 284 } 285 return b 286 } 287 288 // JsonBody adds request raw body encoding by JSON. 289 func (b *BeegoHttpRequest) JsonBody(obj interface{}) (*BeegoHttpRequest, error) { 290 if b.req.Body == nil && obj != nil { 291 buf := bytes.NewBuffer(nil) 292 enc := json.NewEncoder(buf) 293 if err := enc.Encode(obj); err != nil { 294 return b, err 295 } 296 b.req.Body = ioutil.NopCloser(buf) 297 b.req.ContentLength = int64(buf.Len()) 298 b.req.Header.Set("Content-Type", "application/json") 299 } 300 return b, nil 301 } 302 303 func (b *BeegoHttpRequest) buildUrl(paramBody string) { 304 // build GET url with query string 305 if b.req.Method == "GET" && len(paramBody) > 0 { 306 if strings.Index(b.url, "?") != -1 { 307 b.url += "&" + paramBody 308 } else { 309 b.url = b.url + "?" + paramBody 310 } 311 return 312 } 313 314 // build POST/PUT/PATCH url and body 315 if (b.req.Method == "POST" || b.req.Method == "PUT" || b.req.Method == "PATCH") && b.req.Body == nil { 316 // with files 317 if len(b.files) > 0 { 318 pr, pw := io.Pipe() 319 bodyWriter := multipart.NewWriter(pw) 320 go func() { 321 for formname, filename := range b.files { 322 fileWriter, err := bodyWriter.CreateFormFile(formname, filename) 323 if err != nil { 324 log.Fatal(err) 325 } 326 fh, err := os.Open(filename) 327 if err != nil { 328 log.Fatal(err) 329 } 330 //iocopy 331 _, err = io.Copy(fileWriter, fh) 332 fh.Close() 333 if err != nil { 334 log.Fatal(err) 335 } 336 } 337 for k, v := range b.params { 338 bodyWriter.WriteField(k, v) 339 } 340 bodyWriter.Close() 341 pw.Close() 342 }() 343 b.Header("Content-Type", bodyWriter.FormDataContentType()) 344 b.req.Body = ioutil.NopCloser(pr) 345 return 346 } 347 348 // with params 349 if len(paramBody) > 0 { 350 b.Header("Content-Type", "application/x-www-form-urlencoded") 351 b.Body(paramBody) 352 } 353 } 354 } 355 356 func (b *BeegoHttpRequest) getResponse() (*http.Response, error) { 357 if b.resp.StatusCode != 0 { 358 return b.resp, nil 359 } 360 resp, err := b.SendOut() 361 if err != nil { 362 return nil, err 363 } 364 b.resp = resp 365 return resp, nil 366 } 367 368 func (b *BeegoHttpRequest) SendOut() (*http.Response, error) { 369 var paramBody string 370 if len(b.params) > 0 { 371 var buf bytes.Buffer 372 for k, v := range b.params { 373 buf.WriteString(url.QueryEscape(k)) 374 buf.WriteByte('=') 375 buf.WriteString(url.QueryEscape(v)) 376 buf.WriteByte('&') 377 } 378 paramBody = buf.String() 379 paramBody = paramBody[0 : len(paramBody)-1] 380 } 381 382 b.buildUrl(paramBody) 383 url, err := url.Parse(b.url) 384 if err != nil { 385 return nil, err 386 } 387 388 b.req.URL = url 389 390 trans := b.setting.Transport 391 392 if trans == nil { 393 // create default transport 394 trans = &http.Transport{ 395 TLSClientConfig: b.setting.TlsClientConfig, 396 Proxy: b.setting.Proxy, 397 Dial: TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout), 398 } 399 } else { 400 // if b.transport is *http.Transport then set the settings. 401 if t, ok := trans.(*http.Transport); ok { 402 if t.TLSClientConfig == nil { 403 t.TLSClientConfig = b.setting.TlsClientConfig 404 } 405 if t.Proxy == nil { 406 t.Proxy = b.setting.Proxy 407 } 408 if t.Dial == nil { 409 t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout) 410 } 411 } 412 } 413 414 var jar http.CookieJar = nil 415 if b.setting.EnableCookie { 416 if defaultCookieJar == nil { 417 createDefaultCookie() 418 } 419 jar = defaultCookieJar 420 } 421 422 client := &http.Client{ 423 Transport: trans, 424 Jar: jar, 425 } 426 427 if b.setting.UserAgent != "" && b.req.Header.Get("User-Agent") == "" { 428 b.req.Header.Set("User-Agent", b.setting.UserAgent) 429 } 430 431 if b.setting.ShowDebug { 432 dump, err := httputil.DumpRequest(b.req, b.setting.DumpBody) 433 if err != nil { 434 println(err.Error()) 435 } 436 b.dump = dump 437 } 438 return client.Do(b.req) 439 } 440 441 // String returns the body string in response. 442 // it calls Response inner. 443 func (b *BeegoHttpRequest) String() (string, error) { 444 data, err := b.Bytes() 445 if err != nil { 446 return "", err 447 } 448 449 return string(data), nil 450 } 451 452 // Bytes returns the body []byte in response. 453 // it calls Response inner. 454 func (b *BeegoHttpRequest) Bytes() ([]byte, error) { 455 if b.body != nil { 456 return b.body, nil 457 } 458 resp, err := b.getResponse() 459 if err != nil { 460 return nil, err 461 } 462 if resp.Body == nil { 463 return nil, nil 464 } 465 defer resp.Body.Close() 466 if b.setting.Gzip && resp.Header.Get("Content-Encoding") == "gzip" { 467 reader, err := gzip.NewReader(resp.Body) 468 if err != nil { 469 return nil, err 470 } 471 b.body, err = ioutil.ReadAll(reader) 472 } else { 473 b.body, err = ioutil.ReadAll(resp.Body) 474 } 475 if err != nil { 476 return nil, err 477 } 478 return b.body, nil 479 } 480 481 // ToFile saves the body data in response to one file. 482 // it calls Response inner. 483 func (b *BeegoHttpRequest) ToFile(filename string) error { 484 f, err := os.Create(filename) 485 if err != nil { 486 return err 487 } 488 defer f.Close() 489 490 resp, err := b.getResponse() 491 if err != nil { 492 return err 493 } 494 if resp.Body == nil { 495 return nil 496 } 497 defer resp.Body.Close() 498 _, err = io.Copy(f, resp.Body) 499 return err 500 } 501 502 // ToJson returns the map that marshals from the body bytes as json in response . 503 // it calls Response inner. 504 func (b *BeegoHttpRequest) ToJson(v interface{}) error { 505 data, err := b.Bytes() 506 if err != nil { 507 return err 508 } 509 return json.Unmarshal(data, v) 510 } 511 512 // ToXml returns the map that marshals from the body bytes as xml in response . 513 // it calls Response inner. 514 func (b *BeegoHttpRequest) ToXml(v interface{}) error { 515 data, err := b.Bytes() 516 if err != nil { 517 return err 518 } 519 return xml.Unmarshal(data, v) 520 } 521 522 // Response executes request client gets response mannually. 523 func (b *BeegoHttpRequest) Response() (*http.Response, error) { 524 return b.getResponse() 525 } 526 527 // TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field. 528 func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) { 529 return func(netw, addr string) (net.Conn, error) { 530 conn, err := net.DialTimeout(netw, addr, cTimeout) 531 if err != nil { 532 return nil, err 533 } 534 conn.SetDeadline(time.Now().Add(rwTimeout)) 535 return conn, nil 536 } 537 }