github.com/vnforks/kid/v5@v5.22.1-0.20200408055009-b89d99c65676/app/opengraph.go (about)

     1  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
     2  // See LICENSE.txt for license information.
     3  
     4  package app
     5  
     6  import (
     7  	"html"
     8  	"io"
     9  	"net/url"
    10  
    11  	"github.com/dyatlov/go-opengraph/opengraph"
    12  	"golang.org/x/net/html/charset"
    13  
    14  	"github.com/vnforks/kid/v5/mlog"
    15  )
    16  
    17  const MaxOpenGraphResponseSize = 1024 * 1024 * 50
    18  
    19  func (a *App) GetOpenGraphMetadata(requestURL string) *opengraph.OpenGraph {
    20  	res, err := a.HTTPService().MakeClient(false).Get(requestURL)
    21  	if err != nil {
    22  		mlog.Debug("GetOpenGraphMetadata request failed", mlog.String("requestURL", requestURL), mlog.Err(err))
    23  		return nil
    24  	}
    25  	defer res.Body.Close()
    26  	return a.parseOpenGraphMetadata(requestURL, res.Body, res.Header.Get("Content-Type"))
    27  }
    28  
    29  func (a *App) parseOpenGraphMetadata(requestURL string, body io.Reader, contentType string) *opengraph.OpenGraph {
    30  	og := opengraph.NewOpenGraph()
    31  	body = forceHTMLEncodingToUTF8(io.LimitReader(body, MaxOpenGraphResponseSize), contentType)
    32  
    33  	if err := og.ProcessHTML(body); err != nil {
    34  		mlog.Warn("parseOpenGraphMetadata processing failed", mlog.String("requestURL", requestURL), mlog.Err(err))
    35  	}
    36  
    37  	makeOpenGraphURLsAbsolute(og, requestURL)
    38  
    39  	openGraphDecodeHtmlEntities(og)
    40  
    41  	// If image proxy enabled modify open graph data to feed though proxy
    42  	if toProxyURL := a.ImageProxyAdder(); toProxyURL != nil {
    43  		og = openGraphDataWithProxyAddedToImageURLs(og, toProxyURL)
    44  	}
    45  
    46  	// The URL should be the link the user provided in their message, not a redirected one.
    47  	if og.URL != "" {
    48  		og.URL = requestURL
    49  	}
    50  
    51  	return og
    52  }
    53  
    54  func forceHTMLEncodingToUTF8(body io.Reader, contentType string) io.Reader {
    55  	r, err := charset.NewReader(body, contentType)
    56  	if err != nil {
    57  		mlog.Warn("forceHTMLEncodingToUTF8 failed to convert", mlog.String("contentType", contentType), mlog.Err(err))
    58  		return body
    59  	}
    60  	return r
    61  }
    62  
    63  func makeOpenGraphURLsAbsolute(og *opengraph.OpenGraph, requestURL string) {
    64  	parsedRequestURL, err := url.Parse(requestURL)
    65  	if err != nil {
    66  		mlog.Warn("makeOpenGraphURLsAbsolute failed to parse url", mlog.String("requestURL", requestURL), mlog.Err(err))
    67  		return
    68  	}
    69  
    70  	makeURLAbsolute := func(resultURL string) string {
    71  		if resultURL == "" {
    72  			return resultURL
    73  		}
    74  
    75  		parsedResultURL, err := url.Parse(resultURL)
    76  		if err != nil {
    77  			mlog.Warn("makeOpenGraphURLsAbsolute failed to parse result", mlog.String("requestURL", requestURL), mlog.Err(err))
    78  			return resultURL
    79  		}
    80  
    81  		if parsedResultURL.IsAbs() {
    82  			return resultURL
    83  		}
    84  
    85  		return parsedRequestURL.ResolveReference(parsedResultURL).String()
    86  	}
    87  
    88  	og.URL = makeURLAbsolute(og.URL)
    89  
    90  	for _, image := range og.Images {
    91  		image.URL = makeURLAbsolute(image.URL)
    92  		image.SecureURL = makeURLAbsolute(image.SecureURL)
    93  	}
    94  
    95  	for _, audio := range og.Audios {
    96  		audio.URL = makeURLAbsolute(audio.URL)
    97  		audio.SecureURL = makeURLAbsolute(audio.SecureURL)
    98  	}
    99  
   100  	for _, video := range og.Videos {
   101  		video.URL = makeURLAbsolute(video.URL)
   102  		video.SecureURL = makeURLAbsolute(video.SecureURL)
   103  	}
   104  }
   105  
   106  func openGraphDataWithProxyAddedToImageURLs(ogdata *opengraph.OpenGraph, toProxyURL func(string) string) *opengraph.OpenGraph {
   107  	for _, image := range ogdata.Images {
   108  		var url string
   109  		if image.SecureURL != "" {
   110  			url = image.SecureURL
   111  		} else {
   112  			url = image.URL
   113  		}
   114  
   115  		image.URL = ""
   116  		image.SecureURL = toProxyURL(url)
   117  	}
   118  
   119  	return ogdata
   120  }
   121  
   122  func openGraphDecodeHtmlEntities(og *opengraph.OpenGraph) {
   123  	og.Title = html.UnescapeString(og.Title)
   124  	og.Description = html.UnescapeString(og.Description)
   125  }