github.com/nwidger/bat@v0.0.2/httplib/httplib.go (about)

     1  // Copyright 2014 beego Author. All Rights Reserved.
     2  // Copyright 2015 bat authors
     3  //
     4  // Licensed under the Apache License, Version 2.0 (the "License");
     5  // you may not use this file except in compliance with the License.
     6  // You may obtain a copy of the License at
     7  //
     8  //      http://www.apache.org/licenses/LICENSE-2.0
     9  //
    10  // Unless required by applicable law or agreed to in writing, software
    11  // distributed under the License is distributed on an "AS IS" BASIS,
    12  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  // See the License for the specific language governing permissions and
    14  // limitations under the License.
    15  
    16  // Usage:
    17  //
    18  // import "github.com/astaxie/beego/httplib"
    19  //
    20  //	b := httplib.Post("http://beego.me/")
    21  //	b.Param("username","astaxie")
    22  //	b.Param("password","123456")
    23  //	b.PostFile("uploadfile1", "httplib.pdf")
    24  //	b.PostFile("uploadfile2", "httplib.txt")
    25  //	str, err := b.String()
    26  //	if err != nil {
    27  //		t.Fatal(err)
    28  //	}
    29  //	fmt.Println(str)
    30  //
    31  //  more docs http://beego.me/docs/module/httplib.md
    32  package httplib
    33  
    34  import (
    35  	"bytes"
    36  	"compress/gzip"
    37  	"crypto/tls"
    38  	"encoding/json"
    39  	"encoding/xml"
    40  	"io"
    41  	"io/ioutil"
    42  	"log"
    43  	"mime/multipart"
    44  	"net"
    45  	"net/http"
    46  	"net/http/cookiejar"
    47  	"net/http/httputil"
    48  	"net/url"
    49  	"os"
    50  	"strings"
    51  	"sync"
    52  	"time"
    53  )
    54  
    55  var defaultSetting = BeegoHttpSettings{false, "beegoServer", 60 * time.Second, 60 * time.Second, nil, nil, nil, false, true, true}
    56  var defaultCookieJar http.CookieJar
    57  var settingMutex sync.Mutex
    58  
    59  // createDefaultCookie creates a global cookiejar to store cookies.
    60  func createDefaultCookie() {
    61  	settingMutex.Lock()
    62  	defer settingMutex.Unlock()
    63  	defaultCookieJar, _ = cookiejar.New(nil)
    64  }
    65  
    66  // Overwrite default settings
    67  func SetDefaultSetting(setting BeegoHttpSettings) {
    68  	settingMutex.Lock()
    69  	defer settingMutex.Unlock()
    70  	defaultSetting = setting
    71  	if defaultSetting.ConnectTimeout == 0 {
    72  		defaultSetting.ConnectTimeout = 60 * time.Second
    73  	}
    74  	if defaultSetting.ReadWriteTimeout == 0 {
    75  		defaultSetting.ReadWriteTimeout = 60 * time.Second
    76  	}
    77  }
    78  
    79  // return *BeegoHttpRequest with specific method
    80  func NewBeegoRequest(rawurl, method string) *BeegoHttpRequest {
    81  	var resp http.Response
    82  	u, err := url.Parse(rawurl)
    83  	if err != nil {
    84  		log.Fatal(err)
    85  	}
    86  	req := http.Request{
    87  		URL:        u,
    88  		Method:     method,
    89  		Header:     make(http.Header),
    90  		Proto:      "HTTP/1.1",
    91  		ProtoMajor: 1,
    92  		ProtoMinor: 1,
    93  	}
    94  	return &BeegoHttpRequest{rawurl, &req, map[string]string{}, map[string]string{}, defaultSetting, &resp, nil, nil}
    95  }
    96  
    97  // Get returns *BeegoHttpRequest with GET method.
    98  func Get(url string) *BeegoHttpRequest {
    99  	return NewBeegoRequest(url, "GET")
   100  }
   101  
   102  // Post returns *BeegoHttpRequest with POST method.
   103  func Post(url string) *BeegoHttpRequest {
   104  	return NewBeegoRequest(url, "POST")
   105  }
   106  
   107  // Put returns *BeegoHttpRequest with PUT method.
   108  func Put(url string) *BeegoHttpRequest {
   109  	return NewBeegoRequest(url, "PUT")
   110  }
   111  
   112  // Delete returns *BeegoHttpRequest DELETE method.
   113  func Delete(url string) *BeegoHttpRequest {
   114  	return NewBeegoRequest(url, "DELETE")
   115  }
   116  
   117  // Head returns *BeegoHttpRequest with HEAD method.
   118  func Head(url string) *BeegoHttpRequest {
   119  	return NewBeegoRequest(url, "HEAD")
   120  }
   121  
   122  // BeegoHttpSettings
   123  type BeegoHttpSettings struct {
   124  	ShowDebug        bool
   125  	UserAgent        string
   126  	ConnectTimeout   time.Duration
   127  	ReadWriteTimeout time.Duration
   128  	TlsClientConfig  *tls.Config
   129  	Proxy            func(*http.Request) (*url.URL, error)
   130  	Transport        http.RoundTripper
   131  	EnableCookie     bool
   132  	Gzip             bool
   133  	DumpBody         bool
   134  }
   135  
   136  // BeegoHttpRequest provides more useful methods for requesting one url than http.Request.
   137  type BeegoHttpRequest struct {
   138  	url     string
   139  	req     *http.Request
   140  	params  map[string]string
   141  	files   map[string]string
   142  	setting BeegoHttpSettings
   143  	resp    *http.Response
   144  	body    []byte
   145  	dump    []byte
   146  }
   147  
   148  // get request
   149  func (b *BeegoHttpRequest) GetRequest() *http.Request {
   150  	return b.req
   151  }
   152  
   153  // Change request settings
   154  func (b *BeegoHttpRequest) Setting(setting BeegoHttpSettings) *BeegoHttpRequest {
   155  	b.setting = setting
   156  	return b
   157  }
   158  
   159  // SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
   160  func (b *BeegoHttpRequest) SetBasicAuth(username, password string) *BeegoHttpRequest {
   161  	b.req.SetBasicAuth(username, password)
   162  	return b
   163  }
   164  
   165  // SetEnableCookie sets enable/disable cookiejar
   166  func (b *BeegoHttpRequest) SetEnableCookie(enable bool) *BeegoHttpRequest {
   167  	b.setting.EnableCookie = enable
   168  	return b
   169  }
   170  
   171  // SetUserAgent sets User-Agent header field
   172  func (b *BeegoHttpRequest) SetUserAgent(useragent string) *BeegoHttpRequest {
   173  	b.setting.UserAgent = useragent
   174  	return b
   175  }
   176  
   177  // Debug sets show debug or not when executing request.
   178  func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest {
   179  	b.setting.ShowDebug = isdebug
   180  	return b
   181  }
   182  
   183  // Dump Body.
   184  func (b *BeegoHttpRequest) DumpBody(isdump bool) *BeegoHttpRequest {
   185  	b.setting.DumpBody = isdump
   186  	return b
   187  }
   188  
   189  // return the DumpRequest
   190  func (b *BeegoHttpRequest) DumpRequest() []byte {
   191  	return b.dump
   192  }
   193  
   194  // SetTimeout sets connect time out and read-write time out for BeegoRequest.
   195  func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest {
   196  	b.setting.ConnectTimeout = connectTimeout
   197  	b.setting.ReadWriteTimeout = readWriteTimeout
   198  	return b
   199  }
   200  
   201  // SetTLSClientConfig sets tls connection configurations if visiting https url.
   202  func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest {
   203  	b.setting.TlsClientConfig = config
   204  	return b
   205  }
   206  
   207  // Header add header item string in request.
   208  func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest {
   209  	b.req.Header.Set(key, value)
   210  	return b
   211  }
   212  
   213  // Set HOST
   214  func (b *BeegoHttpRequest) SetHost(host string) *BeegoHttpRequest {
   215  	b.req.Host = host
   216  	return b
   217  }
   218  
   219  // Set the protocol version for incoming requests.
   220  // Client requests always use HTTP/1.1.
   221  func (b *BeegoHttpRequest) SetProtocolVersion(vers string) *BeegoHttpRequest {
   222  	if len(vers) == 0 {
   223  		vers = "HTTP/1.1"
   224  	}
   225  
   226  	major, minor, ok := http.ParseHTTPVersion(vers)
   227  	if ok {
   228  		b.req.Proto = vers
   229  		b.req.ProtoMajor = major
   230  		b.req.ProtoMinor = minor
   231  	}
   232  
   233  	return b
   234  }
   235  
   236  // SetCookie add cookie into request.
   237  func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest {
   238  	b.req.Header.Add("Cookie", cookie.String())
   239  	return b
   240  }
   241  
   242  // Set transport to
   243  func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest {
   244  	b.setting.Transport = transport
   245  	return b
   246  }
   247  
   248  // Set http proxy
   249  // example:
   250  //
   251  //	func(req *http.Request) (*url.URL, error) {
   252  // 		u, _ := url.ParseRequestURI("http://127.0.0.1:8118")
   253  // 		return u, nil
   254  // 	}
   255  func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest {
   256  	b.setting.Proxy = proxy
   257  	return b
   258  }
   259  
   260  // Param adds query param in to request.
   261  // params build query string as ?key1=value1&key2=value2...
   262  func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest {
   263  	b.params[key] = value
   264  	return b
   265  }
   266  
   267  func (b *BeegoHttpRequest) PostFile(formname, filename string) *BeegoHttpRequest {
   268  	b.files[formname] = filename
   269  	return b
   270  }
   271  
   272  // Body adds request raw body.
   273  // it supports string and []byte.
   274  func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest {
   275  	switch t := data.(type) {
   276  	case string:
   277  		bf := bytes.NewBufferString(t)
   278  		b.req.Body = ioutil.NopCloser(bf)
   279  		b.req.ContentLength = int64(len(t))
   280  	case []byte:
   281  		bf := bytes.NewBuffer(t)
   282  		b.req.Body = ioutil.NopCloser(bf)
   283  		b.req.ContentLength = int64(len(t))
   284  	}
   285  	return b
   286  }
   287  
   288  // JsonBody adds request raw body encoding by JSON.
   289  func (b *BeegoHttpRequest) JsonBody(obj interface{}) (*BeegoHttpRequest, error) {
   290  	if b.req.Body == nil && obj != nil {
   291  		buf := bytes.NewBuffer(nil)
   292  		enc := json.NewEncoder(buf)
   293  		if err := enc.Encode(obj); err != nil {
   294  			return b, err
   295  		}
   296  		b.req.Body = ioutil.NopCloser(buf)
   297  		b.req.ContentLength = int64(buf.Len())
   298  		b.req.Header.Set("Content-Type", "application/json")
   299  	}
   300  	return b, nil
   301  }
   302  
   303  func (b *BeegoHttpRequest) buildUrl(paramBody string) {
   304  	// build GET url with query string
   305  	if b.req.Method == "GET" && len(paramBody) > 0 {
   306  		if strings.Index(b.url, "?") != -1 {
   307  			b.url += "&" + paramBody
   308  		} else {
   309  			b.url = b.url + "?" + paramBody
   310  		}
   311  		return
   312  	}
   313  
   314  	// build POST/PUT/PATCH url and body
   315  	if (b.req.Method == "POST" || b.req.Method == "PUT" || b.req.Method == "PATCH") && b.req.Body == nil {
   316  		// with files
   317  		if len(b.files) > 0 {
   318  			pr, pw := io.Pipe()
   319  			bodyWriter := multipart.NewWriter(pw)
   320  			go func() {
   321  				for formname, filename := range b.files {
   322  					fileWriter, err := bodyWriter.CreateFormFile(formname, filename)
   323  					if err != nil {
   324  						log.Fatal(err)
   325  					}
   326  					fh, err := os.Open(filename)
   327  					if err != nil {
   328  						log.Fatal(err)
   329  					}
   330  					//iocopy
   331  					_, err = io.Copy(fileWriter, fh)
   332  					fh.Close()
   333  					if err != nil {
   334  						log.Fatal(err)
   335  					}
   336  				}
   337  				for k, v := range b.params {
   338  					bodyWriter.WriteField(k, v)
   339  				}
   340  				bodyWriter.Close()
   341  				pw.Close()
   342  			}()
   343  			b.Header("Content-Type", bodyWriter.FormDataContentType())
   344  			b.req.Body = ioutil.NopCloser(pr)
   345  			return
   346  		}
   347  
   348  		// with params
   349  		if len(paramBody) > 0 {
   350  			b.Header("Content-Type", "application/x-www-form-urlencoded")
   351  			b.Body(paramBody)
   352  		}
   353  	}
   354  }
   355  
   356  func (b *BeegoHttpRequest) getResponse() (*http.Response, error) {
   357  	if b.resp.StatusCode != 0 {
   358  		return b.resp, nil
   359  	}
   360  	resp, err := b.SendOut()
   361  	if err != nil {
   362  		return nil, err
   363  	}
   364  	b.resp = resp
   365  	return resp, nil
   366  }
   367  
   368  func (b *BeegoHttpRequest) SendOut() (*http.Response, error) {
   369  	var paramBody string
   370  	if len(b.params) > 0 {
   371  		var buf bytes.Buffer
   372  		for k, v := range b.params {
   373  			buf.WriteString(url.QueryEscape(k))
   374  			buf.WriteByte('=')
   375  			buf.WriteString(url.QueryEscape(v))
   376  			buf.WriteByte('&')
   377  		}
   378  		paramBody = buf.String()
   379  		paramBody = paramBody[0 : len(paramBody)-1]
   380  	}
   381  
   382  	b.buildUrl(paramBody)
   383  	url, err := url.Parse(b.url)
   384  	if err != nil {
   385  		return nil, err
   386  	}
   387  
   388  	b.req.URL = url
   389  
   390  	trans := b.setting.Transport
   391  
   392  	if trans == nil {
   393  		// create default transport
   394  		trans = &http.Transport{
   395  			TLSClientConfig: b.setting.TlsClientConfig,
   396  			Proxy:           b.setting.Proxy,
   397  			Dial:            TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout),
   398  		}
   399  	} else {
   400  		// if b.transport is *http.Transport then set the settings.
   401  		if t, ok := trans.(*http.Transport); ok {
   402  			if t.TLSClientConfig == nil {
   403  				t.TLSClientConfig = b.setting.TlsClientConfig
   404  			}
   405  			if t.Proxy == nil {
   406  				t.Proxy = b.setting.Proxy
   407  			}
   408  			if t.Dial == nil {
   409  				t.Dial = TimeoutDialer(b.setting.ConnectTimeout, b.setting.ReadWriteTimeout)
   410  			}
   411  		}
   412  	}
   413  
   414  	var jar http.CookieJar = nil
   415  	if b.setting.EnableCookie {
   416  		if defaultCookieJar == nil {
   417  			createDefaultCookie()
   418  		}
   419  		jar = defaultCookieJar
   420  	}
   421  
   422  	client := &http.Client{
   423  		Transport: trans,
   424  		Jar:       jar,
   425  	}
   426  
   427  	if b.setting.UserAgent != "" && b.req.Header.Get("User-Agent") == "" {
   428  		b.req.Header.Set("User-Agent", b.setting.UserAgent)
   429  	}
   430  
   431  	if b.setting.ShowDebug {
   432  		dump, err := httputil.DumpRequest(b.req, b.setting.DumpBody)
   433  		if err != nil {
   434  			println(err.Error())
   435  		}
   436  		b.dump = dump
   437  	}
   438  	return client.Do(b.req)
   439  }
   440  
   441  // String returns the body string in response.
   442  // it calls Response inner.
   443  func (b *BeegoHttpRequest) String() (string, error) {
   444  	data, err := b.Bytes()
   445  	if err != nil {
   446  		return "", err
   447  	}
   448  
   449  	return string(data), nil
   450  }
   451  
   452  // Bytes returns the body []byte in response.
   453  // it calls Response inner.
   454  func (b *BeegoHttpRequest) Bytes() ([]byte, error) {
   455  	if b.body != nil {
   456  		return b.body, nil
   457  	}
   458  	resp, err := b.getResponse()
   459  	if err != nil {
   460  		return nil, err
   461  	}
   462  	if resp.Body == nil {
   463  		return nil, nil
   464  	}
   465  	defer resp.Body.Close()
   466  	if b.setting.Gzip && resp.Header.Get("Content-Encoding") == "gzip" {
   467  		reader, err := gzip.NewReader(resp.Body)
   468  		if err != nil {
   469  			return nil, err
   470  		}
   471  		b.body, err = ioutil.ReadAll(reader)
   472  	} else {
   473  		b.body, err = ioutil.ReadAll(resp.Body)
   474  	}
   475  	if err != nil {
   476  		return nil, err
   477  	}
   478  	return b.body, nil
   479  }
   480  
   481  // ToFile saves the body data in response to one file.
   482  // it calls Response inner.
   483  func (b *BeegoHttpRequest) ToFile(filename string) error {
   484  	f, err := os.Create(filename)
   485  	if err != nil {
   486  		return err
   487  	}
   488  	defer f.Close()
   489  
   490  	resp, err := b.getResponse()
   491  	if err != nil {
   492  		return err
   493  	}
   494  	if resp.Body == nil {
   495  		return nil
   496  	}
   497  	defer resp.Body.Close()
   498  	_, err = io.Copy(f, resp.Body)
   499  	return err
   500  }
   501  
   502  // ToJson returns the map that marshals from the body bytes as json in response .
   503  // it calls Response inner.
   504  func (b *BeegoHttpRequest) ToJson(v interface{}) error {
   505  	data, err := b.Bytes()
   506  	if err != nil {
   507  		return err
   508  	}
   509  	return json.Unmarshal(data, v)
   510  }
   511  
   512  // ToXml returns the map that marshals from the body bytes as xml in response .
   513  // it calls Response inner.
   514  func (b *BeegoHttpRequest) ToXml(v interface{}) error {
   515  	data, err := b.Bytes()
   516  	if err != nil {
   517  		return err
   518  	}
   519  	return xml.Unmarshal(data, v)
   520  }
   521  
   522  // Response executes request client gets response mannually.
   523  func (b *BeegoHttpRequest) Response() (*http.Response, error) {
   524  	return b.getResponse()
   525  }
   526  
   527  // TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field.
   528  func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
   529  	return func(netw, addr string) (net.Conn, error) {
   530  		conn, err := net.DialTimeout(netw, addr, cTimeout)
   531  		if err != nil {
   532  			return nil, err
   533  		}
   534  		conn.SetDeadline(time.Now().Add(rwTimeout))
   535  		return conn, nil
   536  	}
   537  }