github.com/status-im/status-go@v1.1.0/protocol/linkpreview_unfurler.go (about)

     1  package protocol
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io/ioutil"
     7  	"net/http"
     8  	neturl "net/url"
     9  	"time"
    10  
    11  	"go.uber.org/zap"
    12  
    13  	"github.com/status-im/status-go/protocol/common"
    14  )
    15  
    16  const (
    17  	DefaultRequestTimeout = 15000 * time.Millisecond
    18  
    19  	headerAcceptJSON = "application/json; charset=utf-8"
    20  	headerAcceptText = "text/html; charset=utf-8"
    21  
    22  	// Without a particular user agent, many providers treat status-go as a
    23  	// gluttony bot, and either respond more frequently with a 429 (Too Many
    24  	// Requests), or simply refuse to return valid data. Note that using a known
    25  	// browser UA doesn't work well with some providers, such as Spotify,
    26  	// apparently they still flag status-go as a bad actor.
    27  	headerUserAgent = "status-go/v0.151.15"
    28  
    29  	// Currently set to English, but we could make this setting dynamic according
    30  	// to the user's language of choice.
    31  	headerAcceptLanguage = "en-US,en;q=0.5"
    32  )
    33  
    34  type Headers map[string]string
    35  
    36  type Unfurler interface {
    37  	Unfurl() (*common.LinkPreview, error)
    38  }
    39  
    40  func newDefaultLinkPreview(url *neturl.URL) *common.LinkPreview {
    41  	return &common.LinkPreview{
    42  		URL:      url.String(),
    43  		Hostname: url.Hostname(),
    44  	}
    45  }
    46  
    47  func fetchBody(logger *zap.Logger, httpClient *http.Client, url string, headers Headers) ([]byte, error) {
    48  	ctx, cancel := context.WithTimeout(context.Background(), DefaultRequestTimeout)
    49  	defer cancel()
    50  
    51  	req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
    52  	if err != nil {
    53  		return nil, fmt.Errorf("failed to perform HTTP request: %w", err)
    54  	}
    55  
    56  	for k, v := range headers {
    57  		req.Header.Set(k, v)
    58  	}
    59  
    60  	res, err := httpClient.Do(req)
    61  	if err != nil {
    62  		return nil, err
    63  	}
    64  	defer func() {
    65  		if err := res.Body.Close(); err != nil {
    66  			logger.Error("failed to close response body", zap.Error(err))
    67  		}
    68  	}()
    69  
    70  	if res.StatusCode >= http.StatusBadRequest {
    71  		return nil, fmt.Errorf("http request failed, statusCode='%d'", res.StatusCode)
    72  	}
    73  
    74  	bodyBytes, err := ioutil.ReadAll(res.Body)
    75  	if err != nil {
    76  		return nil, fmt.Errorf("failed to read body bytes: %w", err)
    77  	}
    78  
    79  	return bodyBytes, nil
    80  }