github.com/mattermosttest/mattermost-server/v5@v5.0.0-20200917143240-9dfa12e121f9/app/post_metadata_test.go (about)

     1  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
     2  // See LICENSE.txt for license information.
     3  
     4  package app
     5  
     6  import (
     7  	"bytes"
     8  	"fmt"
     9  	"image"
    10  	"image/png"
    11  	"io"
    12  	"net/http"
    13  	"net/http/httptest"
    14  	"net/url"
    15  	"os"
    16  	"strconv"
    17  	"strings"
    18  	"testing"
    19  	"time"
    20  
    21  	"github.com/dyatlov/go-opengraph/opengraph"
    22  	"github.com/mattermost/mattermost-server/v5/model"
    23  	"github.com/mattermost/mattermost-server/v5/services/httpservice"
    24  	"github.com/mattermost/mattermost-server/v5/services/imageproxy"
    25  	"github.com/mattermost/mattermost-server/v5/utils/testutils"
    26  	"github.com/stretchr/testify/assert"
    27  	"github.com/stretchr/testify/require"
    28  )
    29  
    30  func TestPreparePostListForClient(t *testing.T) {
    31  	// Most of this logic is covered by TestPreparePostForClient, so this just tests handling of multiple posts
    32  
    33  	th := Setup(t)
    34  	defer th.TearDown()
    35  
    36  	postList := model.NewPostList()
    37  	for i := 0; i < 5; i++ {
    38  		postList.AddPost(&model.Post{})
    39  	}
    40  
    41  	clientPostList := th.App.PreparePostListForClient(postList)
    42  
    43  	t.Run("doesn't mutate provided post list", func(t *testing.T) {
    44  		assert.NotEqual(t, clientPostList, postList, "should've returned a new post list")
    45  		assert.NotEqual(t, clientPostList.Posts, postList.Posts, "should've returned a new PostList.Posts")
    46  		assert.Equal(t, clientPostList.Order, postList.Order, "should've returned the existing PostList.Order")
    47  
    48  		for id, originalPost := range postList.Posts {
    49  			assert.NotEqual(t, clientPostList.Posts[id], originalPost, "should've returned new post objects")
    50  			assert.Equal(t, clientPostList.Posts[id].Id, originalPost.Id, "should've returned the same posts")
    51  		}
    52  	})
    53  
    54  	t.Run("adds metadata to each post", func(t *testing.T) {
    55  		for _, clientPost := range clientPostList.Posts {
    56  			assert.NotNil(t, clientPost.Metadata, "should've populated metadata for each post")
    57  		}
    58  	})
    59  }
    60  
    61  func TestPreparePostForClient(t *testing.T) {
    62  	var serverURL string
    63  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
    64  		switch r.URL.Path {
    65  		case "/":
    66  			w.Header().Set("Content-Type", "text/html")
    67  			w.Write([]byte(`
    68  			<html>
    69  			<head>
    70  			<meta property="og:image" content="` + serverURL + `/test-image3.png" />
    71  			<meta property="og:site_name" content="GitHub" />
    72  			<meta property="og:type" content="object" />
    73  			<meta property="og:title" content="hmhealey/test-files" />
    74  			<meta property="og:url" content="https://github.com/hmhealey/test-files" />
    75  			<meta property="og:description" content="Contribute to hmhealey/test-files development by creating an account on GitHub." />
    76  			</head>
    77  			</html>`))
    78  		case "/test-image1.png":
    79  			file, err := testutils.ReadTestFile("test.png")
    80  			require.Nil(t, err)
    81  
    82  			w.Header().Set("Content-Type", "image/png")
    83  			w.Write(file)
    84  		case "/test-image2.png":
    85  			file, err := testutils.ReadTestFile("test-data-graph.png")
    86  			require.Nil(t, err)
    87  
    88  			w.Header().Set("Content-Type", "image/png")
    89  			w.Write(file)
    90  		case "/test-image3.png":
    91  			file, err := testutils.ReadTestFile("qa-data-graph.png")
    92  			require.Nil(t, err)
    93  
    94  			w.Header().Set("Content-Type", "image/png")
    95  			w.Write(file)
    96  		default:
    97  			require.Fail(t, "Invalid path", r.URL.Path)
    98  		}
    99  	}))
   100  	serverURL = server.URL
   101  	defer server.Close()
   102  
   103  	setup := func(t *testing.T) *TestHelper {
   104  		th := Setup(t).InitBasic()
   105  
   106  		th.App.UpdateConfig(func(cfg *model.Config) {
   107  			*cfg.ServiceSettings.EnableLinkPreviews = true
   108  			*cfg.ImageProxySettings.Enable = false
   109  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "localhost,127.0.0.1"
   110  		})
   111  
   112  		return th
   113  	}
   114  
   115  	t.Run("no metadata needed", func(t *testing.T) {
   116  		th := setup(t)
   117  		defer th.TearDown()
   118  
   119  		message := model.NewId()
   120  		post := &model.Post{
   121  			Message: message,
   122  		}
   123  
   124  		clientPost := th.App.PreparePostForClient(post, false, false)
   125  
   126  		t.Run("doesn't mutate provided post", func(t *testing.T) {
   127  			assert.NotEqual(t, clientPost, post, "should've returned a new post")
   128  
   129  			assert.Equal(t, message, post.Message, "shouldn't have mutated post.Message")
   130  			assert.Equal(t, (*model.PostMetadata)(nil), post.Metadata, "shouldn't have mutated post.Metadata")
   131  		})
   132  
   133  		t.Run("populates all fields", func(t *testing.T) {
   134  			assert.Equal(t, message, clientPost.Message, "shouldn't have changed Message")
   135  			assert.NotEqual(t, nil, clientPost.Metadata, "should've populated Metadata")
   136  			assert.Empty(t, clientPost.Metadata.Embeds, "should've populated Embeds")
   137  			assert.Empty(t, clientPost.Metadata.Reactions, "should've populated Reactions")
   138  			assert.Empty(t, clientPost.Metadata.Files, "should've populated Files")
   139  			assert.Empty(t, clientPost.Metadata.Emojis, "should've populated Emojis")
   140  			assert.Empty(t, clientPost.Metadata.Images, "should've populated Images")
   141  		})
   142  	})
   143  
   144  	t.Run("metadata already set", func(t *testing.T) {
   145  		th := setup(t)
   146  		defer th.TearDown()
   147  
   148  		post := th.CreatePost(th.BasicChannel)
   149  
   150  		clientPost := th.App.PreparePostForClient(post, false, false)
   151  
   152  		assert.False(t, clientPost == post, "should've returned a new post")
   153  		assert.Equal(t, clientPost, post, "shouldn't have changed any metadata")
   154  	})
   155  
   156  	t.Run("reactions", func(t *testing.T) {
   157  		th := setup(t)
   158  		defer th.TearDown()
   159  
   160  		post := th.CreatePost(th.BasicChannel)
   161  		reaction1 := th.AddReactionToPost(post, th.BasicUser, "smile")
   162  		reaction2 := th.AddReactionToPost(post, th.BasicUser2, "smile")
   163  		reaction3 := th.AddReactionToPost(post, th.BasicUser2, "ice_cream")
   164  		post.HasReactions = true
   165  
   166  		clientPost := th.App.PreparePostForClient(post, false, false)
   167  
   168  		assert.Len(t, clientPost.Metadata.Reactions, 3, "should've populated Reactions")
   169  		assert.Equal(t, reaction1, clientPost.Metadata.Reactions[0], "first reaction is incorrect")
   170  		assert.Equal(t, reaction2, clientPost.Metadata.Reactions[1], "second reaction is incorrect")
   171  		assert.Equal(t, reaction3, clientPost.Metadata.Reactions[2], "third reaction is incorrect")
   172  	})
   173  
   174  	t.Run("files", func(t *testing.T) {
   175  		th := setup(t)
   176  		defer th.TearDown()
   177  
   178  		fileInfo, err := th.App.DoUploadFile(time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test"))
   179  		require.Nil(t, err)
   180  
   181  		post, err := th.App.CreatePost(&model.Post{
   182  			UserId:    th.BasicUser.Id,
   183  			ChannelId: th.BasicChannel.Id,
   184  			FileIds:   []string{fileInfo.Id},
   185  		}, th.BasicChannel, false, true)
   186  		require.Nil(t, err)
   187  
   188  		fileInfo.PostId = post.Id
   189  
   190  		clientPost := th.App.PreparePostForClient(post, false, false)
   191  
   192  		assert.Equal(t, []*model.FileInfo{fileInfo}, clientPost.Metadata.Files, "should've populated Files")
   193  	})
   194  
   195  	t.Run("emojis without custom emojis enabled", func(t *testing.T) {
   196  		th := setup(t)
   197  		defer th.TearDown()
   198  
   199  		th.App.UpdateConfig(func(cfg *model.Config) {
   200  			*cfg.ServiceSettings.EnableCustomEmoji = false
   201  		})
   202  
   203  		emoji := th.CreateEmoji()
   204  
   205  		post, err := th.App.CreatePost(&model.Post{
   206  			UserId:    th.BasicUser.Id,
   207  			ChannelId: th.BasicChannel.Id,
   208  			Message:   ":" + emoji.Name + ": :taco:",
   209  			Props: map[string]interface{}{
   210  				"attachments": []*model.SlackAttachment{
   211  					{
   212  						Text: ":" + emoji.Name + ":",
   213  					},
   214  				},
   215  			},
   216  		}, th.BasicChannel, false, true)
   217  		require.Nil(t, err)
   218  
   219  		th.AddReactionToPost(post, th.BasicUser, "smile")
   220  		th.AddReactionToPost(post, th.BasicUser, "angry")
   221  		th.AddReactionToPost(post, th.BasicUser2, "angry")
   222  		post.HasReactions = true
   223  
   224  		clientPost := th.App.PreparePostForClient(post, false, false)
   225  
   226  		t.Run("populates emojis", func(t *testing.T) {
   227  			assert.ElementsMatch(t, []*model.Emoji{}, clientPost.Metadata.Emojis, "should've populated empty Emojis")
   228  		})
   229  
   230  		t.Run("populates reaction counts", func(t *testing.T) {
   231  			reactions := clientPost.Metadata.Reactions
   232  			assert.Len(t, reactions, 3, "should've populated Reactions")
   233  		})
   234  	})
   235  
   236  	t.Run("emojis with custom emojis enabled", func(t *testing.T) {
   237  		th := setup(t)
   238  		defer th.TearDown()
   239  
   240  		th.App.UpdateConfig(func(cfg *model.Config) {
   241  			*cfg.ServiceSettings.EnableCustomEmoji = true
   242  		})
   243  
   244  		emoji1 := th.CreateEmoji()
   245  		emoji2 := th.CreateEmoji()
   246  		emoji3 := th.CreateEmoji()
   247  		emoji4 := th.CreateEmoji()
   248  
   249  		post, err := th.App.CreatePost(&model.Post{
   250  			UserId:    th.BasicUser.Id,
   251  			ChannelId: th.BasicChannel.Id,
   252  			Message:   ":" + emoji3.Name + ": :taco:",
   253  			Props: map[string]interface{}{
   254  				"attachments": []*model.SlackAttachment{
   255  					{
   256  						Text: ":" + emoji4.Name + ":",
   257  					},
   258  				},
   259  			},
   260  		}, th.BasicChannel, false, true)
   261  		require.Nil(t, err)
   262  
   263  		th.AddReactionToPost(post, th.BasicUser, emoji1.Name)
   264  		th.AddReactionToPost(post, th.BasicUser, emoji2.Name)
   265  		th.AddReactionToPost(post, th.BasicUser2, emoji2.Name)
   266  		th.AddReactionToPost(post, th.BasicUser2, "angry")
   267  		post.HasReactions = true
   268  
   269  		clientPost := th.App.PreparePostForClient(post, false, false)
   270  
   271  		t.Run("populates emojis", func(t *testing.T) {
   272  			assert.ElementsMatch(t, []*model.Emoji{emoji1, emoji2, emoji3, emoji4}, clientPost.Metadata.Emojis, "should've populated post.Emojis")
   273  		})
   274  
   275  		t.Run("populates reaction counts", func(t *testing.T) {
   276  			reactions := clientPost.Metadata.Reactions
   277  			assert.Len(t, reactions, 4, "should've populated Reactions")
   278  		})
   279  	})
   280  
   281  	t.Run("emojis overriding profile icon", func(t *testing.T) {
   282  		th := setup(t)
   283  		defer th.TearDown()
   284  
   285  		prepare := func(override bool, url, emoji string) *model.Post {
   286  			th.App.UpdateConfig(func(cfg *model.Config) {
   287  				*cfg.ServiceSettings.EnablePostIconOverride = override
   288  			})
   289  
   290  			post, err := th.App.CreatePost(&model.Post{
   291  				UserId:    th.BasicUser.Id,
   292  				ChannelId: th.BasicChannel.Id,
   293  				Message:   "Test",
   294  			}, th.BasicChannel, false, true)
   295  
   296  			require.Nil(t, err)
   297  
   298  			post.AddProp(model.POST_PROPS_OVERRIDE_ICON_URL, url)
   299  			post.AddProp(model.POST_PROPS_OVERRIDE_ICON_EMOJI, emoji)
   300  
   301  			return th.App.PreparePostForClient(post, false, false)
   302  		}
   303  
   304  		emoji := "basketball"
   305  		url := "http://host.com/image.png"
   306  		overridenUrl := "/static/emoji/1f3c0.png"
   307  
   308  		t.Run("does not override icon URL", func(t *testing.T) {
   309  			clientPost := prepare(false, url, emoji)
   310  
   311  			s, ok := clientPost.GetProps()[model.POST_PROPS_OVERRIDE_ICON_URL]
   312  			assert.True(t, ok)
   313  			assert.EqualValues(t, url, s)
   314  			s, ok = clientPost.GetProps()[model.POST_PROPS_OVERRIDE_ICON_EMOJI]
   315  			assert.True(t, ok)
   316  			assert.EqualValues(t, emoji, s)
   317  		})
   318  
   319  		t.Run("overrides icon URL", func(t *testing.T) {
   320  			clientPost := prepare(true, url, emoji)
   321  
   322  			s, ok := clientPost.GetProps()[model.POST_PROPS_OVERRIDE_ICON_URL]
   323  			assert.True(t, ok)
   324  			assert.EqualValues(t, overridenUrl, s)
   325  			s, ok = clientPost.GetProps()[model.POST_PROPS_OVERRIDE_ICON_EMOJI]
   326  			assert.True(t, ok)
   327  			assert.EqualValues(t, emoji, s)
   328  		})
   329  
   330  	})
   331  
   332  	t.Run("markdown image dimensions", func(t *testing.T) {
   333  		th := setup(t)
   334  		defer th.TearDown()
   335  
   336  		post, err := th.App.CreatePost(&model.Post{
   337  			UserId:    th.BasicUser.Id,
   338  			ChannelId: th.BasicChannel.Id,
   339  			Message:   fmt.Sprintf("This is ![our logo](%s/test-image2.png) and ![our icon](%s/test-image1.png)", server.URL, server.URL),
   340  		}, th.BasicChannel, false, true)
   341  		require.Nil(t, err)
   342  
   343  		clientPost := th.App.PreparePostForClient(post, false, false)
   344  
   345  		t.Run("populates image dimensions", func(t *testing.T) {
   346  			imageDimensions := clientPost.Metadata.Images
   347  			require.Len(t, imageDimensions, 2)
   348  			assert.Equal(t, &model.PostImage{
   349  				Format: "png",
   350  				Width:  1280,
   351  				Height: 1780,
   352  			}, imageDimensions[server.URL+"/test-image2.png"])
   353  			assert.Equal(t, &model.PostImage{
   354  				Format: "png",
   355  				Width:  408,
   356  				Height: 336,
   357  			}, imageDimensions[server.URL+"/test-image1.png"])
   358  		})
   359  	})
   360  
   361  	t.Run("proxy linked images", func(t *testing.T) {
   362  		th := setup(t)
   363  		defer th.TearDown()
   364  
   365  		testProxyLinkedImage(t, th, false)
   366  	})
   367  
   368  	t.Run("proxy opengraph images", func(t *testing.T) {
   369  		th := setup(t)
   370  		defer th.TearDown()
   371  
   372  		testProxyOpenGraphImage(t, th, false)
   373  	})
   374  
   375  	t.Run("image embed", func(t *testing.T) {
   376  		th := setup(t)
   377  		defer th.TearDown()
   378  
   379  		post, err := th.App.CreatePost(&model.Post{
   380  			UserId:    th.BasicUser.Id,
   381  			ChannelId: th.BasicChannel.Id,
   382  			Message: `This is our logo: ` + server.URL + `/test-image2.png
   383  	And this is our icon: ` + server.URL + `/test-image1.png`,
   384  		}, th.BasicChannel, false, true)
   385  		require.Nil(t, err)
   386  
   387  		clientPost := th.App.PreparePostForClient(post, false, false)
   388  
   389  		// Reminder that only the first link gets an embed and dimensions
   390  
   391  		t.Run("populates embeds", func(t *testing.T) {
   392  			assert.ElementsMatch(t, []*model.PostEmbed{
   393  				{
   394  					Type: model.POST_EMBED_IMAGE,
   395  					URL:  server.URL + "/test-image2.png",
   396  				},
   397  			}, clientPost.Metadata.Embeds)
   398  		})
   399  
   400  		t.Run("populates image dimensions", func(t *testing.T) {
   401  			imageDimensions := clientPost.Metadata.Images
   402  			require.Len(t, imageDimensions, 1)
   403  			assert.Equal(t, &model.PostImage{
   404  				Format: "png",
   405  				Width:  1280,
   406  				Height: 1780,
   407  			}, imageDimensions[server.URL+"/test-image2.png"])
   408  		})
   409  	})
   410  
   411  	t.Run("opengraph embed", func(t *testing.T) {
   412  		th := setup(t)
   413  		defer th.TearDown()
   414  
   415  		post, err := th.App.CreatePost(&model.Post{
   416  			UserId:    th.BasicUser.Id,
   417  			ChannelId: th.BasicChannel.Id,
   418  			Message:   `This is our web page: ` + server.URL,
   419  		}, th.BasicChannel, false, true)
   420  		require.Nil(t, err)
   421  
   422  		clientPost := th.App.PreparePostForClient(post, false, false)
   423  		firstEmbed := clientPost.Metadata.Embeds[0]
   424  		ogData := firstEmbed.Data.(*opengraph.OpenGraph)
   425  
   426  		t.Run("populates embeds", func(t *testing.T) {
   427  			assert.Equal(t, firstEmbed.Type, model.POST_EMBED_OPENGRAPH)
   428  			assert.Equal(t, firstEmbed.URL, server.URL)
   429  			assert.Equal(t, ogData.Description, "Contribute to hmhealey/test-files development by creating an account on GitHub.")
   430  			assert.Equal(t, ogData.SiteName, "GitHub")
   431  			assert.Equal(t, ogData.Title, "hmhealey/test-files")
   432  			assert.Equal(t, ogData.Type, "object")
   433  			assert.Equal(t, ogData.URL, server.URL)
   434  			assert.Equal(t, ogData.Images[0].URL, server.URL+"/test-image3.png")
   435  		})
   436  
   437  		t.Run("populates image dimensions", func(t *testing.T) {
   438  			imageDimensions := clientPost.Metadata.Images
   439  			require.Len(t, imageDimensions, 1)
   440  			assert.Equal(t, &model.PostImage{
   441  				Format: "png",
   442  				Width:  1790,
   443  				Height: 1340,
   444  			}, imageDimensions[server.URL+"/test-image3.png"])
   445  		})
   446  	})
   447  
   448  	t.Run("message attachment embed", func(t *testing.T) {
   449  		th := setup(t)
   450  		defer th.TearDown()
   451  
   452  		post, err := th.App.CreatePost(&model.Post{
   453  			UserId:    th.BasicUser.Id,
   454  			ChannelId: th.BasicChannel.Id,
   455  			Props: map[string]interface{}{
   456  				"attachments": []interface{}{
   457  					map[string]interface{}{
   458  						"text": "![icon](" + server.URL + "/test-image1.png)",
   459  					},
   460  				},
   461  			},
   462  		}, th.BasicChannel, false, true)
   463  		require.Nil(t, err)
   464  
   465  		clientPost := th.App.PreparePostForClient(post, false, false)
   466  
   467  		t.Run("populates embeds", func(t *testing.T) {
   468  			assert.ElementsMatch(t, []*model.PostEmbed{
   469  				{
   470  					Type: model.POST_EMBED_MESSAGE_ATTACHMENT,
   471  				},
   472  			}, clientPost.Metadata.Embeds)
   473  		})
   474  
   475  		t.Run("populates image dimensions", func(t *testing.T) {
   476  			imageDimensions := clientPost.Metadata.Images
   477  			require.Len(t, imageDimensions, 1)
   478  			assert.Equal(t, &model.PostImage{
   479  				Format: "png",
   480  				Width:  408,
   481  				Height: 336,
   482  			}, imageDimensions[server.URL+"/test-image1.png"])
   483  		})
   484  	})
   485  
   486  	t.Run("no metadata for deleted posts", func(t *testing.T) {
   487  		th := setup(t)
   488  		defer th.TearDown()
   489  
   490  		fileInfo, err := th.App.DoUploadFile(time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test"))
   491  		require.Nil(t, err)
   492  
   493  		post, err := th.App.CreatePost(&model.Post{
   494  			Message:   "test",
   495  			FileIds:   []string{fileInfo.Id},
   496  			UserId:    th.BasicUser.Id,
   497  			ChannelId: th.BasicChannel.Id,
   498  		}, th.BasicChannel, false, true)
   499  		require.Nil(t, err)
   500  
   501  		th.AddReactionToPost(post, th.BasicUser, "taco")
   502  
   503  		post, err = th.App.DeletePost(post.Id, th.BasicUser.Id)
   504  		require.Nil(t, err)
   505  
   506  		// DeleteAt isn't set on the post returned by App.DeletePost
   507  		post.DeleteAt = model.GetMillis()
   508  
   509  		clientPost := th.App.PreparePostForClient(post, false, false)
   510  
   511  		assert.NotEqual(t, nil, clientPost.Metadata, "should've populated Metadata“")
   512  		assert.Equal(t, "", clientPost.Message, "should've cleaned post content")
   513  		assert.Nil(t, clientPost.Metadata.Reactions, "should not have populated Reactions")
   514  		assert.Nil(t, clientPost.Metadata.Files, "should not have populated Files")
   515  	})
   516  }
   517  
   518  func TestPreparePostForClientWithImageProxy(t *testing.T) {
   519  	setup := func(t *testing.T) *TestHelper {
   520  		th := Setup(t).InitBasic()
   521  
   522  		th.App.UpdateConfig(func(cfg *model.Config) {
   523  			*cfg.ServiceSettings.EnableLinkPreviews = true
   524  			*cfg.ServiceSettings.SiteURL = "http://mymattermost.com"
   525  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "localhost,127.0.0.1"
   526  			*cfg.ImageProxySettings.Enable = true
   527  			*cfg.ImageProxySettings.ImageProxyType = "atmos/camo"
   528  			*cfg.ImageProxySettings.RemoteImageProxyURL = "https://127.0.0.1"
   529  			*cfg.ImageProxySettings.RemoteImageProxyOptions = "foo"
   530  		})
   531  
   532  		return th
   533  	}
   534  
   535  	t.Run("proxy linked images", func(t *testing.T) {
   536  		th := setup(t)
   537  		defer th.TearDown()
   538  
   539  		testProxyLinkedImage(t, th, true)
   540  	})
   541  
   542  	t.Run("proxy opengraph images", func(t *testing.T) {
   543  		th := setup(t)
   544  		defer th.TearDown()
   545  
   546  		testProxyOpenGraphImage(t, th, true)
   547  	})
   548  }
   549  
   550  func testProxyLinkedImage(t *testing.T, th *TestHelper, shouldProxy bool) {
   551  	postTemplate := "![foo](%v)"
   552  	imageURL := "http://mydomain.com/myimage"
   553  	proxiedImageURL := "http://mymattermost.com/api/v4/image?url=http%3A%2F%2Fmydomain.com%2Fmyimage"
   554  
   555  	post := &model.Post{
   556  		UserId:    th.BasicUser.Id,
   557  		ChannelId: th.BasicChannel.Id,
   558  		Message:   fmt.Sprintf(postTemplate, imageURL),
   559  	}
   560  
   561  	clientPost := th.App.PreparePostForClient(post, false, false)
   562  
   563  	if shouldProxy {
   564  		assert.Equal(t, fmt.Sprintf(postTemplate, imageURL), post.Message, "should not have mutated original post")
   565  		assert.Equal(t, fmt.Sprintf(postTemplate, proxiedImageURL), clientPost.Message, "should've replaced linked image URLs")
   566  	} else {
   567  		assert.Equal(t, fmt.Sprintf(postTemplate, imageURL), clientPost.Message, "shouldn't have replaced linked image URLs")
   568  	}
   569  }
   570  
   571  func testProxyOpenGraphImage(t *testing.T, th *TestHelper, shouldProxy bool) {
   572  	var serverURL string
   573  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   574  		switch r.URL.Path {
   575  		case "/":
   576  			w.Header().Set("Content-Type", "text/html")
   577  			w.Write([]byte(`
   578  			<html>
   579  			<head>
   580  			<meta property="og:image" content="` + serverURL + `/test-image3.png" />
   581  			<meta property="og:site_name" content="GitHub" />
   582  			<meta property="og:type" content="object" />
   583  			<meta property="og:title" content="hmhealey/test-files" />
   584  			<meta property="og:url" content="https://github.com/hmhealey/test-files" />
   585  			<meta property="og:description" content="Contribute to hmhealey/test-files development by creating an account on GitHub." />
   586  			</head>
   587  			</html>`))
   588  		case "/test-image3.png":
   589  			file, err := testutils.ReadTestFile("qa-data-graph.png")
   590  			require.Nil(t, err)
   591  
   592  			w.Header().Set("Content-Type", "image/png")
   593  			w.Write(file)
   594  		default:
   595  			require.Fail(t, "Invalid path", r.URL.Path)
   596  		}
   597  	}))
   598  	serverURL = server.URL
   599  	defer server.Close()
   600  
   601  	post, err := th.App.CreatePost(&model.Post{
   602  		UserId:    th.BasicUser.Id,
   603  		ChannelId: th.BasicChannel.Id,
   604  		Message:   `This is our web page: ` + server.URL,
   605  	}, th.BasicChannel, false, true)
   606  	require.Nil(t, err)
   607  
   608  	embeds := th.App.PreparePostForClient(post, false, false).Metadata.Embeds
   609  	require.Len(t, embeds, 1, "should have one embed")
   610  
   611  	embed := embeds[0]
   612  	assert.Equal(t, model.POST_EMBED_OPENGRAPH, embed.Type, "embed type should be OpenGraph")
   613  	assert.Equal(t, server.URL, embed.URL, "embed URL should be correct")
   614  
   615  	og, ok := embed.Data.(*opengraph.OpenGraph)
   616  	assert.True(t, ok, "data should be non-nil OpenGraph data")
   617  	assert.NotNil(t, og, "data should be non-nil OpenGraph data")
   618  	assert.Equal(t, "GitHub", og.SiteName, "OpenGraph data should be correctly populated")
   619  
   620  	require.Len(t, og.Images, 1, "OpenGraph data should have one image")
   621  
   622  	image := og.Images[0]
   623  	if shouldProxy {
   624  		assert.Equal(t, "", image.URL, "image URL should not be set with proxy")
   625  		assert.Equal(t, "http://mymattermost.com/api/v4/image?url="+url.QueryEscape(server.URL+"/test-image3.png"), image.SecureURL, "secure image URL should be sent through proxy")
   626  	} else {
   627  		assert.Equal(t, server.URL+"/test-image3.png", image.URL, "image URL should be set")
   628  		assert.Equal(t, "", image.SecureURL, "secure image URL should not be set")
   629  	}
   630  }
   631  
   632  func TestGetEmbedForPost(t *testing.T) {
   633  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   634  		if r.URL.Path == "/index.html" {
   635  			w.Header().Set("Content-Type", "text/html")
   636  			w.Write([]byte(`
   637  			<html>
   638  			<head>
   639  			<meta property="og:title" content="Title" />
   640  			</head>
   641  			</html>`))
   642  		} else if r.URL.Path == "/image.png" {
   643  			file, err := testutils.ReadTestFile("test.png")
   644  			require.Nil(t, err)
   645  
   646  			w.Header().Set("Content-Type", "image/png")
   647  			w.Write(file)
   648  		} else if r.URL.Path == "/other" {
   649  			w.Header().Set("Content-Type", "text/html")
   650  			w.Write([]byte(`
   651  			<html>
   652  			<head>
   653  			</head>
   654  			</html>`))
   655  		} else {
   656  			require.Fail(t, "Invalid path", r.URL.Path)
   657  		}
   658  	}))
   659  	defer server.Close()
   660  
   661  	ogURL := server.URL + "/index.html"
   662  	imageURL := server.URL + "/image.png"
   663  	otherURL := server.URL + "/other"
   664  
   665  	t.Run("with link previews enabled", func(t *testing.T) {
   666  		th := Setup(t)
   667  		defer th.TearDown()
   668  
   669  		th.App.UpdateConfig(func(cfg *model.Config) {
   670  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
   671  			*cfg.ServiceSettings.EnableLinkPreviews = true
   672  		})
   673  
   674  		t.Run("should return a message attachment when the post has one", func(t *testing.T) {
   675  			embed, err := th.App.getEmbedForPost(&model.Post{
   676  				Props: model.StringInterface{
   677  					"attachments": []*model.SlackAttachment{
   678  						{
   679  							Text: "test",
   680  						},
   681  					},
   682  				},
   683  			}, "", false)
   684  
   685  			assert.Equal(t, &model.PostEmbed{
   686  				Type: model.POST_EMBED_MESSAGE_ATTACHMENT,
   687  			}, embed)
   688  			assert.Nil(t, err)
   689  		})
   690  
   691  		t.Run("should return an image embed when the first link is an image", func(t *testing.T) {
   692  			embed, err := th.App.getEmbedForPost(&model.Post{}, imageURL, false)
   693  
   694  			assert.Equal(t, &model.PostEmbed{
   695  				Type: model.POST_EMBED_IMAGE,
   696  				URL:  imageURL,
   697  			}, embed)
   698  			assert.Nil(t, err)
   699  		})
   700  
   701  		t.Run("should return an image embed when the first link is an image", func(t *testing.T) {
   702  			embed, err := th.App.getEmbedForPost(&model.Post{}, ogURL, false)
   703  
   704  			assert.Equal(t, &model.PostEmbed{
   705  				Type: model.POST_EMBED_OPENGRAPH,
   706  				URL:  ogURL,
   707  				Data: &opengraph.OpenGraph{
   708  					Title: "Title",
   709  				},
   710  			}, embed)
   711  			assert.Nil(t, err)
   712  		})
   713  
   714  		t.Run("should return a link embed", func(t *testing.T) {
   715  			embed, err := th.App.getEmbedForPost(&model.Post{}, otherURL, false)
   716  
   717  			assert.Equal(t, &model.PostEmbed{
   718  				Type: model.POST_EMBED_LINK,
   719  				URL:  otherURL,
   720  			}, embed)
   721  			assert.Nil(t, err)
   722  		})
   723  	})
   724  
   725  	t.Run("with link previews disabled", func(t *testing.T) {
   726  		th := Setup(t)
   727  		defer th.TearDown()
   728  
   729  		th.App.UpdateConfig(func(cfg *model.Config) {
   730  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
   731  			*cfg.ServiceSettings.EnableLinkPreviews = false
   732  		})
   733  
   734  		t.Run("should return an embedded message attachment", func(t *testing.T) {
   735  			embed, err := th.App.getEmbedForPost(&model.Post{
   736  				Props: model.StringInterface{
   737  					"attachments": []*model.SlackAttachment{
   738  						{
   739  							Text: "test",
   740  						},
   741  					},
   742  				},
   743  			}, "", false)
   744  
   745  			assert.Equal(t, &model.PostEmbed{
   746  				Type: model.POST_EMBED_MESSAGE_ATTACHMENT,
   747  			}, embed)
   748  			assert.Nil(t, err)
   749  		})
   750  
   751  		t.Run("should not return an opengraph embed", func(t *testing.T) {
   752  			embed, err := th.App.getEmbedForPost(&model.Post{}, ogURL, false)
   753  
   754  			assert.Nil(t, embed)
   755  			assert.Nil(t, err)
   756  		})
   757  
   758  		t.Run("should not return an image embed", func(t *testing.T) {
   759  			embed, err := th.App.getEmbedForPost(&model.Post{}, imageURL, false)
   760  
   761  			assert.Nil(t, embed)
   762  			assert.Nil(t, err)
   763  		})
   764  
   765  		t.Run("should not return a link embed", func(t *testing.T) {
   766  			embed, err := th.App.getEmbedForPost(&model.Post{}, otherURL, false)
   767  
   768  			assert.Nil(t, embed)
   769  			assert.Nil(t, err)
   770  		})
   771  	})
   772  }
   773  
   774  func TestGetImagesForPost(t *testing.T) {
   775  	t.Run("with an image link", func(t *testing.T) {
   776  		th := Setup(t)
   777  		defer th.TearDown()
   778  
   779  		th.App.UpdateConfig(func(cfg *model.Config) {
   780  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
   781  		})
   782  
   783  		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   784  			file, err := testutils.ReadTestFile("test.png")
   785  			require.Nil(t, err)
   786  
   787  			w.Header().Set("Content-Type", "image/png")
   788  			w.Write(file)
   789  		}))
   790  
   791  		post := &model.Post{
   792  			Metadata: &model.PostMetadata{},
   793  		}
   794  		imageURL := server.URL + "/image.png"
   795  
   796  		images := th.App.getImagesForPost(post, []string{imageURL}, false)
   797  
   798  		assert.Equal(t, images, map[string]*model.PostImage{
   799  			imageURL: {
   800  				Format: "png",
   801  				Width:  408,
   802  				Height: 336,
   803  			},
   804  		})
   805  	})
   806  
   807  	t.Run("with an invalid image link", func(t *testing.T) {
   808  		th := Setup(t)
   809  		defer th.TearDown()
   810  
   811  		th.App.UpdateConfig(func(cfg *model.Config) {
   812  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
   813  		})
   814  
   815  		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   816  			w.WriteHeader(http.StatusInternalServerError)
   817  		}))
   818  
   819  		post := &model.Post{
   820  			Metadata: &model.PostMetadata{},
   821  		}
   822  		imageURL := server.URL + "/bad_image.png"
   823  
   824  		images := th.App.getImagesForPost(post, []string{imageURL}, false)
   825  
   826  		assert.Equal(t, images, map[string]*model.PostImage{})
   827  	})
   828  
   829  	t.Run("for an OpenGraph image", func(t *testing.T) {
   830  		th := Setup(t)
   831  		defer th.TearDown()
   832  
   833  		th.App.UpdateConfig(func(cfg *model.Config) {
   834  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
   835  		})
   836  
   837  		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   838  			if r.URL.Path == "/image.png" {
   839  				w.Header().Set("Content-Type", "image/png")
   840  
   841  				img := image.NewGray(image.Rect(0, 0, 200, 300))
   842  
   843  				var encoder png.Encoder
   844  				encoder.Encode(w, img)
   845  			} else {
   846  				w.WriteHeader(http.StatusNotFound)
   847  			}
   848  		}))
   849  		defer server.Close()
   850  
   851  		ogURL := server.URL + "/index.html"
   852  		imageURL := server.URL + "/image.png"
   853  
   854  		post := &model.Post{
   855  			Metadata: &model.PostMetadata{
   856  				Embeds: []*model.PostEmbed{
   857  					{
   858  						Type: model.POST_EMBED_OPENGRAPH,
   859  						URL:  ogURL,
   860  						Data: &opengraph.OpenGraph{
   861  							Images: []*opengraph.Image{
   862  								{
   863  									URL: imageURL,
   864  								},
   865  							},
   866  						},
   867  					},
   868  				},
   869  			},
   870  		}
   871  
   872  		images := th.App.getImagesForPost(post, []string{}, false)
   873  
   874  		assert.Equal(t, images, map[string]*model.PostImage{
   875  			imageURL: {
   876  				Format: "png",
   877  				Width:  200,
   878  				Height: 300,
   879  			},
   880  		})
   881  	})
   882  
   883  	t.Run("with an OpenGraph image with a secure_url", func(t *testing.T) {
   884  		th := Setup(t)
   885  		defer th.TearDown()
   886  
   887  		th.App.UpdateConfig(func(cfg *model.Config) {
   888  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
   889  		})
   890  
   891  		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   892  			if r.URL.Path == "/secure_image.png" {
   893  				w.Header().Set("Content-Type", "image/png")
   894  
   895  				img := image.NewGray(image.Rect(0, 0, 300, 400))
   896  
   897  				var encoder png.Encoder
   898  				encoder.Encode(w, img)
   899  			} else {
   900  				w.WriteHeader(http.StatusNotFound)
   901  			}
   902  		}))
   903  		defer server.Close()
   904  
   905  		ogURL := server.URL + "/index.html"
   906  		imageURL := server.URL + "/secure_image.png"
   907  
   908  		post := &model.Post{
   909  			Metadata: &model.PostMetadata{
   910  				Embeds: []*model.PostEmbed{
   911  					{
   912  						Type: model.POST_EMBED_OPENGRAPH,
   913  						URL:  ogURL,
   914  						Data: &opengraph.OpenGraph{
   915  							Images: []*opengraph.Image{
   916  								{
   917  									SecureURL: imageURL,
   918  								},
   919  							},
   920  						},
   921  					},
   922  				},
   923  			},
   924  		}
   925  
   926  		images := th.App.getImagesForPost(post, []string{}, false)
   927  
   928  		assert.Equal(t, images, map[string]*model.PostImage{
   929  			imageURL: {
   930  				Format: "png",
   931  				Width:  300,
   932  				Height: 400,
   933  			},
   934  		})
   935  	})
   936  
   937  	t.Run("with an OpenGraph image with a secure_url and no dimensions", func(t *testing.T) {
   938  		th := Setup(t)
   939  		defer th.TearDown()
   940  
   941  		th.App.UpdateConfig(func(cfg *model.Config) {
   942  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
   943  		})
   944  
   945  		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
   946  			if r.URL.Path == "/secure_image.png" {
   947  				w.Header().Set("Content-Type", "image/png")
   948  
   949  				img := image.NewGray(image.Rect(0, 0, 400, 500))
   950  
   951  				var encoder png.Encoder
   952  				encoder.Encode(w, img)
   953  			} else {
   954  				w.WriteHeader(http.StatusNotFound)
   955  			}
   956  		}))
   957  
   958  		ogURL := server.URL + "/index.html"
   959  		imageURL := server.URL + "/secure_image.png"
   960  
   961  		post := &model.Post{
   962  			Metadata: &model.PostMetadata{
   963  				Embeds: []*model.PostEmbed{
   964  					{
   965  						Type: model.POST_EMBED_OPENGRAPH,
   966  						URL:  ogURL,
   967  						Data: &opengraph.OpenGraph{
   968  							Images: []*opengraph.Image{
   969  								{
   970  									URL:       server.URL + "/image.png",
   971  									SecureURL: imageURL,
   972  								},
   973  							},
   974  						},
   975  					},
   976  				},
   977  			},
   978  		}
   979  
   980  		images := th.App.getImagesForPost(post, []string{}, false)
   981  
   982  		assert.Equal(t, images, map[string]*model.PostImage{
   983  			imageURL: {
   984  				Format: "png",
   985  				Width:  400,
   986  				Height: 500,
   987  			},
   988  		})
   989  	})
   990  }
   991  
   992  func TestGetEmojiNamesForString(t *testing.T) {
   993  	testCases := []struct {
   994  		Description string
   995  		Input       string
   996  		Expected    []string
   997  	}{
   998  		{
   999  			Description: "no emojis",
  1000  			Input:       "this is a string",
  1001  			Expected:    []string{},
  1002  		},
  1003  		{
  1004  			Description: "one emoji",
  1005  			Input:       "this is an :emoji1: string",
  1006  			Expected:    []string{"emoji1"},
  1007  		},
  1008  		{
  1009  			Description: "two emojis",
  1010  			Input:       "this is a :emoji3: :emoji2: string",
  1011  			Expected:    []string{"emoji3", "emoji2"},
  1012  		},
  1013  		{
  1014  			Description: "punctuation around emojis",
  1015  			Input:       ":emoji3:/:emoji1: (:emoji2:)",
  1016  			Expected:    []string{"emoji3", "emoji1", "emoji2"},
  1017  		},
  1018  		{
  1019  			Description: "adjacent emojis",
  1020  			Input:       ":emoji3::emoji1:",
  1021  			Expected:    []string{"emoji3", "emoji1"},
  1022  		},
  1023  		{
  1024  			Description: "duplicate emojis",
  1025  			Input:       ":emoji1: :emoji1: :emoji1::emoji2::emoji2: :emoji1:",
  1026  			Expected:    []string{"emoji1", "emoji1", "emoji1", "emoji2", "emoji2", "emoji1"},
  1027  		},
  1028  		{
  1029  			Description: "fake emojis",
  1030  			Input:       "these don't exist :tomato: :potato: :rotato:",
  1031  			Expected:    []string{"tomato", "potato", "rotato"},
  1032  		},
  1033  	}
  1034  
  1035  	for _, testCase := range testCases {
  1036  		testCase := testCase
  1037  		t.Run(testCase.Description, func(t *testing.T) {
  1038  			emojis := getEmojiNamesForString(testCase.Input)
  1039  			assert.ElementsMatch(t, emojis, testCase.Expected, "received incorrect emoji names")
  1040  		})
  1041  	}
  1042  }
  1043  
  1044  func TestGetEmojiNamesForPost(t *testing.T) {
  1045  	testCases := []struct {
  1046  		Description string
  1047  		Post        *model.Post
  1048  		Reactions   []*model.Reaction
  1049  		Expected    []string
  1050  	}{
  1051  		{
  1052  			Description: "no emojis",
  1053  			Post: &model.Post{
  1054  				Message: "this is a post",
  1055  			},
  1056  			Expected: []string{},
  1057  		},
  1058  		{
  1059  			Description: "in post message",
  1060  			Post: &model.Post{
  1061  				Message: "this is :emoji:",
  1062  			},
  1063  			Expected: []string{"emoji"},
  1064  		},
  1065  		{
  1066  			Description: "in reactions",
  1067  			Post:        &model.Post{},
  1068  			Reactions: []*model.Reaction{
  1069  				{
  1070  					EmojiName: "emoji1",
  1071  				},
  1072  				{
  1073  					EmojiName: "emoji2",
  1074  				},
  1075  			},
  1076  			Expected: []string{"emoji1", "emoji2"},
  1077  		},
  1078  		{
  1079  			Description: "in message attachments",
  1080  			Post: &model.Post{
  1081  				Message: "this is a post",
  1082  				Props: map[string]interface{}{
  1083  					"attachments": []*model.SlackAttachment{
  1084  						{
  1085  							Text:    ":emoji1:",
  1086  							Pretext: ":emoji2:",
  1087  						},
  1088  						{
  1089  							Fields: []*model.SlackAttachmentField{
  1090  								{
  1091  									Value: ":emoji3:",
  1092  								},
  1093  								{
  1094  									Value: ":emoji4:",
  1095  								},
  1096  							},
  1097  						},
  1098  					},
  1099  				},
  1100  			},
  1101  			Expected: []string{"emoji1", "emoji2", "emoji3", "emoji4"},
  1102  		},
  1103  		{
  1104  			Description: "with duplicates",
  1105  			Post: &model.Post{
  1106  				Message: "this is :emoji1",
  1107  				Props: map[string]interface{}{
  1108  					"attachments": []*model.SlackAttachment{
  1109  						{
  1110  							Text:    ":emoji2:",
  1111  							Pretext: ":emoji2:",
  1112  							Fields: []*model.SlackAttachmentField{
  1113  								{
  1114  									Value: ":emoji3:",
  1115  								},
  1116  								{
  1117  									Value: ":emoji1:",
  1118  								},
  1119  							},
  1120  						},
  1121  					},
  1122  				},
  1123  			},
  1124  			Expected: []string{"emoji1", "emoji2", "emoji3"},
  1125  		},
  1126  	}
  1127  
  1128  	for _, testCase := range testCases {
  1129  		testCase := testCase
  1130  		t.Run(testCase.Description, func(t *testing.T) {
  1131  			emojis := getEmojiNamesForPost(testCase.Post, testCase.Reactions)
  1132  			assert.ElementsMatch(t, emojis, testCase.Expected, "received incorrect emoji names")
  1133  		})
  1134  	}
  1135  }
  1136  
  1137  func TestGetCustomEmojisForPost(t *testing.T) {
  1138  	th := Setup(t).InitBasic()
  1139  	defer th.TearDown()
  1140  
  1141  	th.App.UpdateConfig(func(cfg *model.Config) {
  1142  		*cfg.ServiceSettings.EnableCustomEmoji = true
  1143  	})
  1144  
  1145  	emojis := []*model.Emoji{
  1146  		th.CreateEmoji(),
  1147  		th.CreateEmoji(),
  1148  		th.CreateEmoji(),
  1149  		th.CreateEmoji(),
  1150  		th.CreateEmoji(),
  1151  		th.CreateEmoji(),
  1152  	}
  1153  
  1154  	t.Run("from different parts of the post", func(t *testing.T) {
  1155  		reactions := []*model.Reaction{
  1156  			{
  1157  				UserId:    th.BasicUser.Id,
  1158  				EmojiName: emojis[0].Name,
  1159  			},
  1160  		}
  1161  
  1162  		post := &model.Post{
  1163  			Message: ":" + emojis[1].Name + ":",
  1164  			Props: map[string]interface{}{
  1165  				"attachments": []*model.SlackAttachment{
  1166  					{
  1167  						Pretext: ":" + emojis[2].Name + ":",
  1168  						Text:    ":" + emojis[3].Name + ":",
  1169  						Fields: []*model.SlackAttachmentField{
  1170  							{
  1171  								Value: ":" + emojis[4].Name + ":",
  1172  							},
  1173  							{
  1174  								Value: ":" + emojis[5].Name + ":",
  1175  							},
  1176  						},
  1177  					},
  1178  				},
  1179  			},
  1180  		}
  1181  
  1182  		emojisForPost, err := th.App.getCustomEmojisForPost(post, reactions)
  1183  		assert.Nil(t, err, "failed to get emojis for post")
  1184  		assert.ElementsMatch(t, emojisForPost, emojis, "received incorrect emojis")
  1185  	})
  1186  
  1187  	t.Run("with emojis that don't exist", func(t *testing.T) {
  1188  		post := &model.Post{
  1189  			Message: ":secret: :" + emojis[0].Name + ":",
  1190  			Props: map[string]interface{}{
  1191  				"attachments": []*model.SlackAttachment{
  1192  					{
  1193  						Text: ":imaginary:",
  1194  					},
  1195  				},
  1196  			},
  1197  		}
  1198  
  1199  		emojisForPost, err := th.App.getCustomEmojisForPost(post, nil)
  1200  		assert.Nil(t, err, "failed to get emojis for post")
  1201  		assert.ElementsMatch(t, emojisForPost, []*model.Emoji{emojis[0]}, "received incorrect emojis")
  1202  	})
  1203  
  1204  	t.Run("with no emojis", func(t *testing.T) {
  1205  		post := &model.Post{
  1206  			Message: "this post is boring",
  1207  			Props:   map[string]interface{}{},
  1208  		}
  1209  
  1210  		emojisForPost, err := th.App.getCustomEmojisForPost(post, nil)
  1211  		assert.Nil(t, err, "failed to get emojis for post")
  1212  		assert.ElementsMatch(t, emojisForPost, []*model.Emoji{}, "should have received no emojis")
  1213  	})
  1214  }
  1215  
  1216  func TestGetFirstLinkAndImages(t *testing.T) {
  1217  	for name, testCase := range map[string]struct {
  1218  		Input             string
  1219  		ExpectedFirstLink string
  1220  		ExpectedImages    []string
  1221  	}{
  1222  		"no links or images": {
  1223  			Input:             "this is a string",
  1224  			ExpectedFirstLink: "",
  1225  			ExpectedImages:    []string{},
  1226  		},
  1227  		"http link": {
  1228  			Input:             "this is a http://example.com",
  1229  			ExpectedFirstLink: "http://example.com",
  1230  			ExpectedImages:    []string{},
  1231  		},
  1232  		"www link": {
  1233  			Input:             "this is a www.example.com",
  1234  			ExpectedFirstLink: "http://www.example.com",
  1235  			ExpectedImages:    []string{},
  1236  		},
  1237  		"image": {
  1238  			Input:             "this is a ![our logo](http://example.com/logo)",
  1239  			ExpectedFirstLink: "",
  1240  			ExpectedImages:    []string{"http://example.com/logo"},
  1241  		},
  1242  		"multiple images": {
  1243  			Input:             "this is a ![our logo](http://example.com/logo) and ![their logo](http://example.com/logo2) and ![my logo](http://example.com/logo3)",
  1244  			ExpectedFirstLink: "",
  1245  			ExpectedImages:    []string{"http://example.com/logo", "http://example.com/logo2", "http://example.com/logo3"},
  1246  		},
  1247  		"multiple images with duplicate": {
  1248  			Input:             "this is a ![our logo](http://example.com/logo) and ![their logo](http://example.com/logo2) and ![my logo which is their logo](http://example.com/logo2)",
  1249  			ExpectedFirstLink: "",
  1250  			ExpectedImages:    []string{"http://example.com/logo", "http://example.com/logo2", "http://example.com/logo2"},
  1251  		},
  1252  		"reference image": {
  1253  			Input: `this is a ![our logo][logo]
  1254  
  1255  [logo]: http://example.com/logo`,
  1256  			ExpectedFirstLink: "",
  1257  			ExpectedImages:    []string{"http://example.com/logo"},
  1258  		},
  1259  		"image and link": {
  1260  			Input:             "this is a https://example.com and ![our logo](https://example.com/logo)",
  1261  			ExpectedFirstLink: "https://example.com",
  1262  			ExpectedImages:    []string{"https://example.com/logo"},
  1263  		},
  1264  		"markdown links (not returned)": {
  1265  			Input: `this is a [our page](http://example.com) and [another page][]
  1266  
  1267  [another page]: http://www.exaple.com/another_page`,
  1268  			ExpectedFirstLink: "",
  1269  			ExpectedImages:    []string{},
  1270  		},
  1271  	} {
  1272  		t.Run(name, func(t *testing.T) {
  1273  			firstLink, images := getFirstLinkAndImages(testCase.Input)
  1274  
  1275  			assert.Equal(t, firstLink, testCase.ExpectedFirstLink)
  1276  			assert.Equal(t, images, testCase.ExpectedImages)
  1277  		})
  1278  	}
  1279  }
  1280  
  1281  func TestGetImagesInMessageAttachments(t *testing.T) {
  1282  	for _, test := range []struct {
  1283  		Name     string
  1284  		Post     *model.Post
  1285  		Expected []string
  1286  	}{
  1287  		{
  1288  			Name:     "no attachments",
  1289  			Post:     &model.Post{},
  1290  			Expected: []string{},
  1291  		},
  1292  		{
  1293  			Name: "empty attachments",
  1294  			Post: &model.Post{
  1295  				Props: map[string]interface{}{
  1296  					"attachments": []*model.SlackAttachment{},
  1297  				},
  1298  			},
  1299  			Expected: []string{},
  1300  		},
  1301  		{
  1302  			Name: "attachment with no fields that can contain images",
  1303  			Post: &model.Post{
  1304  				Props: map[string]interface{}{
  1305  					"attachments": []*model.SlackAttachment{
  1306  						{
  1307  							Title: "This is the title",
  1308  						},
  1309  					},
  1310  				},
  1311  			},
  1312  			Expected: []string{},
  1313  		},
  1314  		{
  1315  			Name: "images in text",
  1316  			Post: &model.Post{
  1317  				Props: map[string]interface{}{
  1318  					"attachments": []*model.SlackAttachment{
  1319  						{
  1320  							Text: "![logo](https://example.com/logo) and ![icon](https://example.com/icon)",
  1321  						},
  1322  					},
  1323  				},
  1324  			},
  1325  			Expected: []string{"https://example.com/logo", "https://example.com/icon"},
  1326  		},
  1327  		{
  1328  			Name: "images in pretext",
  1329  			Post: &model.Post{
  1330  				Props: map[string]interface{}{
  1331  					"attachments": []*model.SlackAttachment{
  1332  						{
  1333  							Pretext: "![logo](https://example.com/logo1) and ![icon](https://example.com/icon1)",
  1334  						},
  1335  					},
  1336  				},
  1337  			},
  1338  			Expected: []string{"https://example.com/logo1", "https://example.com/icon1"},
  1339  		},
  1340  		{
  1341  			Name: "images in fields",
  1342  			Post: &model.Post{
  1343  				Props: map[string]interface{}{
  1344  					"attachments": []*model.SlackAttachment{
  1345  						{
  1346  							Fields: []*model.SlackAttachmentField{
  1347  								{
  1348  									Value: "![logo](https://example.com/logo2) and ![icon](https://example.com/icon2)",
  1349  								},
  1350  							},
  1351  						},
  1352  					},
  1353  				},
  1354  			},
  1355  			Expected: []string{"https://example.com/logo2", "https://example.com/icon2"},
  1356  		},
  1357  		{
  1358  			Name: "image in author_icon",
  1359  			Post: &model.Post{
  1360  				Props: map[string]interface{}{
  1361  					"attachments": []*model.SlackAttachment{
  1362  						{
  1363  							AuthorIcon: "https://example.com/icon2",
  1364  						},
  1365  					},
  1366  				},
  1367  			},
  1368  			Expected: []string{"https://example.com/icon2"},
  1369  		},
  1370  		{
  1371  			Name: "image in image_url",
  1372  			Post: &model.Post{
  1373  				Props: map[string]interface{}{
  1374  					"attachments": []*model.SlackAttachment{
  1375  						{
  1376  							ImageURL: "https://example.com/image",
  1377  						},
  1378  					},
  1379  				},
  1380  			},
  1381  			Expected: []string{"https://example.com/image"},
  1382  		},
  1383  		{
  1384  			Name: "image in thumb_url",
  1385  			Post: &model.Post{
  1386  				Props: map[string]interface{}{
  1387  					"attachments": []*model.SlackAttachment{
  1388  						{
  1389  							ThumbURL: "https://example.com/image",
  1390  						},
  1391  					},
  1392  				},
  1393  			},
  1394  			Expected: []string{"https://example.com/image"},
  1395  		},
  1396  		{
  1397  			Name: "image in footer_icon",
  1398  			Post: &model.Post{
  1399  				Props: map[string]interface{}{
  1400  					"attachments": []*model.SlackAttachment{
  1401  						{
  1402  							FooterIcon: "https://example.com/image",
  1403  						},
  1404  					},
  1405  				},
  1406  			},
  1407  			Expected: []string{"https://example.com/image"},
  1408  		},
  1409  		{
  1410  			Name: "images in multiple fields",
  1411  			Post: &model.Post{
  1412  				Props: map[string]interface{}{
  1413  					"attachments": []*model.SlackAttachment{
  1414  						{
  1415  							Fields: []*model.SlackAttachmentField{
  1416  								{
  1417  									Value: "![logo](https://example.com/logo)",
  1418  								},
  1419  								{
  1420  									Value: "![icon](https://example.com/icon)",
  1421  								},
  1422  							},
  1423  						},
  1424  					},
  1425  				},
  1426  			},
  1427  			Expected: []string{"https://example.com/logo", "https://example.com/icon"},
  1428  		},
  1429  		{
  1430  			Name: "non-string field",
  1431  			Post: &model.Post{
  1432  				Props: map[string]interface{}{
  1433  					"attachments": []*model.SlackAttachment{
  1434  						{
  1435  							Fields: []*model.SlackAttachmentField{
  1436  								{
  1437  									Value: 77,
  1438  								},
  1439  							},
  1440  						},
  1441  					},
  1442  				},
  1443  			},
  1444  			Expected: []string{},
  1445  		},
  1446  		{
  1447  			Name: "images in multiple locations",
  1448  			Post: &model.Post{
  1449  				Props: map[string]interface{}{
  1450  					"attachments": []*model.SlackAttachment{
  1451  						{
  1452  							Text:    "![text](https://example.com/text)",
  1453  							Pretext: "![pretext](https://example.com/pretext)",
  1454  							Fields: []*model.SlackAttachmentField{
  1455  								{
  1456  									Value: "![field1](https://example.com/field1)",
  1457  								},
  1458  								{
  1459  									Value: "![field2](https://example.com/field2)",
  1460  								},
  1461  							},
  1462  						},
  1463  					},
  1464  				},
  1465  			},
  1466  			Expected: []string{"https://example.com/text", "https://example.com/pretext", "https://example.com/field1", "https://example.com/field2"},
  1467  		},
  1468  		{
  1469  			Name: "multiple attachments",
  1470  			Post: &model.Post{
  1471  				Props: map[string]interface{}{
  1472  					"attachments": []*model.SlackAttachment{
  1473  						{
  1474  							Text: "![logo](https://example.com/logo)",
  1475  						},
  1476  						{
  1477  							Text: "![icon](https://example.com/icon)",
  1478  						},
  1479  					},
  1480  				},
  1481  			},
  1482  			Expected: []string{"https://example.com/logo", "https://example.com/icon"},
  1483  		},
  1484  	} {
  1485  		t.Run(test.Name, func(t *testing.T) {
  1486  			images := getImagesInMessageAttachments(test.Post)
  1487  
  1488  			assert.ElementsMatch(t, images, test.Expected)
  1489  		})
  1490  	}
  1491  }
  1492  
  1493  func TestGetLinkMetadata(t *testing.T) {
  1494  	setup := func(t *testing.T) *TestHelper {
  1495  		th := Setup(t)
  1496  
  1497  		th.App.UpdateConfig(func(cfg *model.Config) {
  1498  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1"
  1499  		})
  1500  
  1501  		linkCache.Purge()
  1502  
  1503  		return th
  1504  	}
  1505  
  1506  	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  1507  		params := r.URL.Query()
  1508  
  1509  		writeImage := func(height, width int) {
  1510  
  1511  			img := image.NewGray(image.Rect(0, 0, height, width))
  1512  
  1513  			var encoder png.Encoder
  1514  
  1515  			encoder.Encode(w, img)
  1516  		}
  1517  
  1518  		writeHTML := func(title string) {
  1519  			w.Header().Set("Content-Type", "text/html")
  1520  
  1521  			w.Write([]byte(`
  1522  				<html prefix="og:http://ogp.me/ns#">
  1523  				<head>
  1524  				<meta property="og:title" content="` + title + `" />
  1525  				</head>
  1526  				<body>
  1527  				</body>
  1528  				</html>`))
  1529  		}
  1530  
  1531  		if strings.HasPrefix(r.URL.Path, "/image") {
  1532  			height, _ := strconv.ParseInt(params["height"][0], 10, 0)
  1533  			width, _ := strconv.ParseInt(params["width"][0], 10, 0)
  1534  
  1535  			writeImage(int(height), int(width))
  1536  		} else if strings.HasPrefix(r.URL.Path, "/opengraph") {
  1537  			writeHTML(params["title"][0])
  1538  		} else if strings.HasPrefix(r.URL.Path, "/json") {
  1539  			w.Header().Set("Content-Type", "application/json")
  1540  
  1541  			w.Write([]byte("true"))
  1542  		} else if strings.HasPrefix(r.URL.Path, "/timeout") {
  1543  			w.Header().Set("Content-Type", "text/html")
  1544  
  1545  			w.Write([]byte("<html>"))
  1546  			select {
  1547  			case <-time.After(60 * time.Second):
  1548  			case <-r.Context().Done():
  1549  			}
  1550  			w.Write([]byte("</html>"))
  1551  		} else if strings.HasPrefix(r.URL.Path, "/mixed") {
  1552  			for _, acceptedType := range r.Header["Accept"] {
  1553  				if strings.HasPrefix(acceptedType, "image/*") || strings.HasPrefix(acceptedType, "image/png") {
  1554  					writeImage(10, 10)
  1555  				} else if strings.HasPrefix(acceptedType, "text/html") {
  1556  					writeHTML("mixed")
  1557  				}
  1558  			}
  1559  		} else {
  1560  			w.WriteHeader(http.StatusInternalServerError)
  1561  		}
  1562  	}))
  1563  	defer server.Close()
  1564  
  1565  	t.Run("in-memory cache", func(t *testing.T) {
  1566  		th := setup(t)
  1567  		defer th.TearDown()
  1568  
  1569  		requestURL := server.URL + "/cached"
  1570  		timestamp := int64(1547510400000)
  1571  		title := "from cache"
  1572  
  1573  		cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil)
  1574  
  1575  		t.Run("should use cache if cached entry exists", func(t *testing.T) {
  1576  			_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1577  			require.True(t, ok, "data should already exist in in-memory cache")
  1578  
  1579  			_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1580  			require.False(t, ok, "data should not exist in database")
  1581  
  1582  			og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1583  
  1584  			require.NotNil(t, og)
  1585  			assert.Nil(t, img)
  1586  			assert.Nil(t, err)
  1587  			assert.Equal(t, title, og.Title)
  1588  		})
  1589  
  1590  		t.Run("should use cache if cached entry exists near time", func(t *testing.T) {
  1591  			_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1592  			require.True(t, ok, "data should already exist in in-memory cache")
  1593  
  1594  			_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1595  			require.False(t, ok, "data should not exist in database")
  1596  
  1597  			og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false)
  1598  
  1599  			require.NotNil(t, og)
  1600  			assert.Nil(t, img)
  1601  			assert.Nil(t, err)
  1602  			assert.Equal(t, title, og.Title)
  1603  		})
  1604  
  1605  		t.Run("should not use cache if URL is different", func(t *testing.T) {
  1606  			differentURL := server.URL + "/other"
  1607  
  1608  			_, _, ok := getLinkMetadataFromCache(differentURL, timestamp)
  1609  			require.False(t, ok, "data should not exist in in-memory cache")
  1610  
  1611  			_, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp)
  1612  			require.False(t, ok, "data should not exist in database")
  1613  
  1614  			og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false)
  1615  
  1616  			assert.Nil(t, og)
  1617  			assert.Nil(t, img)
  1618  			assert.Nil(t, err)
  1619  		})
  1620  
  1621  		t.Run("should not use cache if timestamp is different", func(t *testing.T) {
  1622  			differentTimestamp := timestamp + 60*60*1000
  1623  
  1624  			_, _, ok := getLinkMetadataFromCache(requestURL, differentTimestamp)
  1625  			require.False(t, ok, "data should not exist in in-memory cache")
  1626  
  1627  			_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp)
  1628  			require.False(t, ok, "data should not exist in database")
  1629  
  1630  			og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false)
  1631  
  1632  			assert.Nil(t, og)
  1633  			assert.Nil(t, img)
  1634  			assert.Nil(t, err)
  1635  		})
  1636  	})
  1637  
  1638  	t.Run("database cache", func(t *testing.T) {
  1639  		th := setup(t)
  1640  		defer th.TearDown()
  1641  
  1642  		requestURL := server.URL
  1643  		timestamp := int64(1547510400000)
  1644  		title := "from database"
  1645  
  1646  		th.App.saveLinkMetadataToDatabase(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil)
  1647  
  1648  		t.Run("should use database if saved entry exists", func(t *testing.T) {
  1649  			linkCache.Purge()
  1650  
  1651  			_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1652  			require.False(t, ok, "data should not exist in in-memory cache")
  1653  
  1654  			_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1655  			require.True(t, ok, "data should already exist in database")
  1656  
  1657  			og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1658  
  1659  			require.NotNil(t, og)
  1660  			assert.Nil(t, img)
  1661  			assert.Nil(t, err)
  1662  			assert.Equal(t, title, og.Title)
  1663  		})
  1664  
  1665  		t.Run("should use database if saved entry exists near time", func(t *testing.T) {
  1666  			linkCache.Purge()
  1667  
  1668  			_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1669  			require.False(t, ok, "data should not exist in in-memory cache")
  1670  
  1671  			_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1672  			require.True(t, ok, "data should already exist in database")
  1673  
  1674  			og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false)
  1675  
  1676  			require.NotNil(t, og)
  1677  			assert.Nil(t, img)
  1678  			assert.Nil(t, err)
  1679  			assert.Equal(t, title, og.Title)
  1680  		})
  1681  
  1682  		t.Run("should not use database if URL is different", func(t *testing.T) {
  1683  			linkCache.Purge()
  1684  
  1685  			differentURL := requestURL + "/other"
  1686  
  1687  			_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1688  			require.False(t, ok, "data should not exist in in-memory cache")
  1689  
  1690  			_, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp)
  1691  			require.False(t, ok, "data should not exist in database")
  1692  
  1693  			og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false)
  1694  
  1695  			assert.Nil(t, og)
  1696  			assert.Nil(t, img)
  1697  			assert.Nil(t, err)
  1698  		})
  1699  
  1700  		t.Run("should not use database if timestamp is different", func(t *testing.T) {
  1701  			linkCache.Purge()
  1702  
  1703  			differentTimestamp := timestamp + 60*60*1000
  1704  
  1705  			_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1706  			require.False(t, ok, "data should not exist in in-memory cache")
  1707  
  1708  			_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp)
  1709  			require.False(t, ok, "data should not exist in database")
  1710  
  1711  			og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false)
  1712  
  1713  			assert.Nil(t, og)
  1714  			assert.Nil(t, img)
  1715  			assert.Nil(t, err)
  1716  		})
  1717  	})
  1718  
  1719  	t.Run("should get data from remote source", func(t *testing.T) {
  1720  		th := setup(t)
  1721  		defer th.TearDown()
  1722  
  1723  		requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name()
  1724  		timestamp := int64(1547510400000)
  1725  
  1726  		_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1727  		require.False(t, ok, "data should not exist in in-memory cache")
  1728  
  1729  		_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1730  		require.False(t, ok, "data should not exist in database")
  1731  
  1732  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1733  
  1734  		assert.NotNil(t, og)
  1735  		assert.Nil(t, img)
  1736  		assert.Nil(t, err)
  1737  	})
  1738  
  1739  	t.Run("should cache OpenGraph results", func(t *testing.T) {
  1740  		th := setup(t)
  1741  		defer th.TearDown()
  1742  
  1743  		requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name()
  1744  		timestamp := int64(1547510400000)
  1745  
  1746  		_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1747  		require.False(t, ok, "data should not exist in in-memory cache")
  1748  
  1749  		_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1750  		require.False(t, ok, "data should not exist in database")
  1751  
  1752  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1753  
  1754  		assert.NotNil(t, og)
  1755  		assert.Nil(t, img)
  1756  		assert.Nil(t, err)
  1757  
  1758  		fromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1759  		assert.True(t, ok)
  1760  		assert.Exactly(t, og, fromCache)
  1761  
  1762  		fromDatabase, _, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1763  		assert.True(t, ok)
  1764  		assert.Exactly(t, og, fromDatabase)
  1765  	})
  1766  
  1767  	t.Run("should cache image results", func(t *testing.T) {
  1768  		th := setup(t)
  1769  		defer th.TearDown()
  1770  
  1771  		requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name()
  1772  		timestamp := int64(1547510400000)
  1773  
  1774  		_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1775  		require.False(t, ok, "data should not exist in in-memory cache")
  1776  
  1777  		_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1778  		require.False(t, ok, "data should not exist in database")
  1779  
  1780  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1781  
  1782  		assert.Nil(t, og)
  1783  		assert.NotNil(t, img)
  1784  		assert.Nil(t, err)
  1785  
  1786  		_, fromCache, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1787  		assert.True(t, ok)
  1788  		assert.Exactly(t, img, fromCache)
  1789  
  1790  		_, fromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1791  		assert.True(t, ok)
  1792  		assert.Exactly(t, img, fromDatabase)
  1793  	})
  1794  
  1795  	t.Run("should cache general errors", func(t *testing.T) {
  1796  		th := setup(t)
  1797  		defer th.TearDown()
  1798  
  1799  		requestURL := server.URL + "/error"
  1800  		timestamp := int64(1547510400000)
  1801  
  1802  		_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1803  		require.False(t, ok, "data should not exist in in-memory cache")
  1804  
  1805  		_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1806  		require.False(t, ok, "data should not exist in database")
  1807  
  1808  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1809  
  1810  		assert.Nil(t, og)
  1811  		assert.Nil(t, img)
  1812  		assert.Nil(t, err)
  1813  
  1814  		ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1815  		assert.True(t, ok)
  1816  		assert.Nil(t, ogFromCache)
  1817  		assert.Nil(t, imgFromCache)
  1818  
  1819  		ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1820  		assert.True(t, ok)
  1821  		assert.Nil(t, ogFromDatabase)
  1822  		assert.Nil(t, imageFromDatabase)
  1823  	})
  1824  
  1825  	t.Run("should cache invalid URL errors", func(t *testing.T) {
  1826  		th := setup(t)
  1827  		defer th.TearDown()
  1828  
  1829  		requestURL := "http://notarealdomainthatactuallyexists.ca/?name=" + t.Name()
  1830  		timestamp := int64(1547510400000)
  1831  
  1832  		_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1833  		require.False(t, ok, "data should not exist in in-memory cache")
  1834  
  1835  		_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1836  		require.False(t, ok, "data should not exist in database")
  1837  
  1838  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1839  
  1840  		assert.Nil(t, og)
  1841  		assert.Nil(t, img)
  1842  		assert.IsType(t, &url.Error{}, err)
  1843  
  1844  		ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1845  		assert.True(t, ok)
  1846  		assert.Nil(t, ogFromCache)
  1847  		assert.Nil(t, imgFromCache)
  1848  
  1849  		ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1850  		assert.True(t, ok)
  1851  		assert.Nil(t, ogFromDatabase)
  1852  		assert.Nil(t, imageFromDatabase)
  1853  	})
  1854  
  1855  	t.Run("should cache timeout errors", func(t *testing.T) {
  1856  		th := setup(t)
  1857  		defer th.TearDown()
  1858  
  1859  		th.App.UpdateConfig(func(cfg *model.Config) {
  1860  			*cfg.ExperimentalSettings.LinkMetadataTimeoutMilliseconds = 100
  1861  		})
  1862  
  1863  		requestURL := server.URL + "/timeout?name=" + t.Name()
  1864  		timestamp := int64(1547510400000)
  1865  
  1866  		_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1867  		require.False(t, ok, "data should not exist in in-memory cache")
  1868  
  1869  		_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1870  		require.False(t, ok, "data should not exist in database")
  1871  
  1872  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1873  
  1874  		assert.Nil(t, og)
  1875  		assert.Nil(t, img)
  1876  		assert.NotNil(t, err)
  1877  		assert.True(t, os.IsTimeout(err))
  1878  
  1879  		ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1880  		assert.True(t, ok)
  1881  		assert.Nil(t, ogFromCache)
  1882  		assert.Nil(t, imgFromCache)
  1883  
  1884  		ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1885  		assert.True(t, ok)
  1886  		assert.Nil(t, ogFromDatabase)
  1887  		assert.Nil(t, imageFromDatabase)
  1888  	})
  1889  
  1890  	t.Run("should cache database results in memory", func(t *testing.T) {
  1891  		th := setup(t)
  1892  		defer th.TearDown()
  1893  
  1894  		requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name()
  1895  		timestamp := int64(1547510400000)
  1896  
  1897  		_, _, ok := getLinkMetadataFromCache(requestURL, timestamp)
  1898  		require.False(t, ok, "data should not exist in in-memory cache")
  1899  
  1900  		_, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1901  		require.False(t, ok, "data should not exist in database")
  1902  
  1903  		_, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1904  		require.Nil(t, err)
  1905  
  1906  		_, _, ok = getLinkMetadataFromCache(requestURL, timestamp)
  1907  		require.True(t, ok, "data should now exist in in-memory cache")
  1908  
  1909  		linkCache.Purge()
  1910  		_, _, ok = getLinkMetadataFromCache(requestURL, timestamp)
  1911  		require.False(t, ok, "data should no longer exist in in-memory cache")
  1912  
  1913  		_, fromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp)
  1914  		assert.True(t, ok, "data should be be in in-memory cache again")
  1915  		assert.Exactly(t, img, fromDatabase)
  1916  	})
  1917  
  1918  	t.Run("should reject non-html, non-image response", func(t *testing.T) {
  1919  		th := setup(t)
  1920  		defer th.TearDown()
  1921  
  1922  		requestURL := server.URL + "/json?name=" + t.Name()
  1923  		timestamp := int64(1547510400000)
  1924  
  1925  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1926  		assert.Nil(t, og)
  1927  		assert.Nil(t, img)
  1928  		assert.Nil(t, err)
  1929  	})
  1930  
  1931  	t.Run("should check in-memory cache for new post", func(t *testing.T) {
  1932  		th := setup(t)
  1933  		defer th.TearDown()
  1934  
  1935  		requestURL := server.URL + "/error?name=" + t.Name()
  1936  		timestamp := int64(1547510400000)
  1937  
  1938  		cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil)
  1939  
  1940  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true)
  1941  		assert.NotNil(t, og)
  1942  		assert.Nil(t, img)
  1943  		assert.Nil(t, err)
  1944  	})
  1945  
  1946  	t.Run("should skip database cache for new post", func(t *testing.T) {
  1947  		th := setup(t)
  1948  		defer th.TearDown()
  1949  
  1950  		requestURL := server.URL + "/error?name=" + t.Name()
  1951  		timestamp := int64(1547510400000)
  1952  
  1953  		th.App.saveLinkMetadataToDatabase(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil)
  1954  
  1955  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true)
  1956  		assert.Nil(t, og)
  1957  		assert.Nil(t, img)
  1958  		assert.Nil(t, err)
  1959  	})
  1960  
  1961  	t.Run("should resolve relative URL", func(t *testing.T) {
  1962  		th := setup(t)
  1963  		defer th.TearDown()
  1964  
  1965  		// Fake the SiteURL to have the relative URL resolve to the external server
  1966  		oldSiteURL := *th.App.Config().ServiceSettings.SiteURL
  1967  		defer th.App.UpdateConfig(func(cfg *model.Config) {
  1968  			*cfg.ServiceSettings.SiteURL = oldSiteURL
  1969  		})
  1970  
  1971  		th.App.UpdateConfig(func(cfg *model.Config) {
  1972  			*cfg.ServiceSettings.SiteURL = server.URL
  1973  		})
  1974  
  1975  		requestURL := "/image?height=200&width=300&name=" + t.Name()
  1976  		timestamp := int64(1547510400000)
  1977  
  1978  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  1979  		assert.Nil(t, og)
  1980  		assert.NotNil(t, img)
  1981  		assert.Nil(t, err)
  1982  	})
  1983  
  1984  	t.Run("should error on local addresses other than the image proxy", func(t *testing.T) {
  1985  		th := setup(t)
  1986  		defer th.TearDown()
  1987  
  1988  		// Disable AllowedUntrustedInternalConnections since it's turned on for the previous tests
  1989  		oldAllowUntrusted := *th.App.Config().ServiceSettings.AllowedUntrustedInternalConnections
  1990  		oldSiteURL := *th.App.Config().ServiceSettings.SiteURL
  1991  		defer th.App.UpdateConfig(func(cfg *model.Config) {
  1992  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = oldAllowUntrusted
  1993  			*cfg.ServiceSettings.SiteURL = oldSiteURL
  1994  		})
  1995  
  1996  		th.App.UpdateConfig(func(cfg *model.Config) {
  1997  			*cfg.ServiceSettings.AllowedUntrustedInternalConnections = ""
  1998  			*cfg.ServiceSettings.SiteURL = "http://mattermost.example.com"
  1999  			*cfg.ImageProxySettings.Enable = true
  2000  			*cfg.ImageProxySettings.ImageProxyType = "local"
  2001  		})
  2002  
  2003  		requestURL := server.URL + "/image?height=200&width=300&name=" + t.Name()
  2004  		timestamp := int64(1547510400000)
  2005  
  2006  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false)
  2007  		assert.Nil(t, og)
  2008  		assert.Nil(t, img)
  2009  		assert.NotNil(t, err)
  2010  		assert.IsType(t, &url.Error{}, err)
  2011  		assert.Equal(t, httpservice.AddressForbidden, err.(*url.Error).Err)
  2012  
  2013  		requestURL = th.App.GetSiteURL() + "/api/v4/image?url=" + url.QueryEscape(requestURL)
  2014  
  2015  		// Note that this request still fails while testing because the request made by the image proxy is blocked
  2016  		og, img, err = th.App.getLinkMetadata(requestURL, timestamp, false)
  2017  		assert.Nil(t, og)
  2018  		assert.Nil(t, img)
  2019  		assert.NotNil(t, err)
  2020  		assert.IsType(t, imageproxy.Error{}, err)
  2021  	})
  2022  
  2023  	t.Run("should prefer images for mixed content", func(t *testing.T) {
  2024  		th := setup(t)
  2025  		defer th.TearDown()
  2026  
  2027  		requestURL := server.URL + "/mixed?name=" + t.Name()
  2028  		timestamp := int64(1547510400000)
  2029  
  2030  		og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true)
  2031  		assert.Nil(t, og)
  2032  		assert.NotNil(t, img)
  2033  		assert.Nil(t, err)
  2034  	})
  2035  }
  2036  
  2037  func TestResolveMetadataURL(t *testing.T) {
  2038  	for _, test := range []struct {
  2039  		Name       string
  2040  		RequestURL string
  2041  		SiteURL    string
  2042  		Expected   string
  2043  	}{
  2044  		{
  2045  			Name:       "with HTTPS",
  2046  			RequestURL: "https://example.com/file?param=1",
  2047  			Expected:   "https://example.com/file?param=1",
  2048  		},
  2049  		{
  2050  			Name:       "with HTTP",
  2051  			RequestURL: "http://example.com/file?param=1",
  2052  			Expected:   "http://example.com/file?param=1",
  2053  		},
  2054  		{
  2055  			Name:       "with FTP",
  2056  			RequestURL: "ftp://example.com/file?param=1",
  2057  			Expected:   "ftp://example.com/file?param=1",
  2058  		},
  2059  		{
  2060  			Name:       "relative to root",
  2061  			RequestURL: "/file?param=1",
  2062  			SiteURL:    "https://mattermost.example.com:123",
  2063  			Expected:   "https://mattermost.example.com:123/file?param=1",
  2064  		},
  2065  		{
  2066  			Name:       "relative to root with subpath",
  2067  			RequestURL: "/file?param=1",
  2068  			SiteURL:    "https://mattermost.example.com:123/subpath",
  2069  			Expected:   "https://mattermost.example.com:123/file?param=1",
  2070  		},
  2071  	} {
  2072  		t.Run(test.Name, func(t *testing.T) {
  2073  			assert.Equal(t, resolveMetadataURL(test.RequestURL, test.SiteURL), test.Expected)
  2074  		})
  2075  	}
  2076  }
  2077  
  2078  func TestParseLinkMetadata(t *testing.T) {
  2079  	th := Setup(t)
  2080  	defer th.TearDown()
  2081  
  2082  	imageURL := "http://example.com/test.png"
  2083  	file, err := testutils.ReadTestFile("test.png")
  2084  	require.Nil(t, err)
  2085  
  2086  	ogURL := "https://example.com/hello"
  2087  	html := `
  2088  		<html>
  2089  			<head>
  2090  				<meta property="og:title" content="Hello, World!">
  2091  				<meta property="og:type" content="object">
  2092  				<meta property="og:url" content="` + ogURL + `">
  2093  			</head>
  2094  		</html>`
  2095  
  2096  	makeImageReader := func() io.Reader {
  2097  		return bytes.NewReader(file)
  2098  	}
  2099  
  2100  	makeOpenGraphReader := func() io.Reader {
  2101  		return strings.NewReader(html)
  2102  	}
  2103  
  2104  	t.Run("image", func(t *testing.T) {
  2105  		og, dimensions, err := th.App.parseLinkMetadata(imageURL, makeImageReader(), "image/png")
  2106  		assert.Nil(t, err)
  2107  
  2108  		assert.Nil(t, og)
  2109  		assert.Equal(t, &model.PostImage{
  2110  			Format: "png",
  2111  			Width:  408,
  2112  			Height: 336,
  2113  		}, dimensions)
  2114  	})
  2115  
  2116  	t.Run("malformed image", func(t *testing.T) {
  2117  		og, dimensions, err := th.App.parseLinkMetadata(imageURL, makeOpenGraphReader(), "image/png")
  2118  		assert.NotNil(t, err)
  2119  
  2120  		assert.Nil(t, og)
  2121  		assert.Nil(t, dimensions)
  2122  	})
  2123  
  2124  	t.Run("opengraph", func(t *testing.T) {
  2125  		og, dimensions, err := th.App.parseLinkMetadata(ogURL, makeOpenGraphReader(), "text/html; charset=utf-8")
  2126  		assert.Nil(t, err)
  2127  
  2128  		assert.NotNil(t, og)
  2129  		assert.Equal(t, og.Title, "Hello, World!")
  2130  		assert.Equal(t, og.Type, "object")
  2131  		assert.Equal(t, og.URL, ogURL)
  2132  		assert.Nil(t, dimensions)
  2133  	})
  2134  
  2135  	t.Run("malformed opengraph", func(t *testing.T) {
  2136  		og, dimensions, err := th.App.parseLinkMetadata(ogURL, makeImageReader(), "text/html; charset=utf-8")
  2137  		assert.Nil(t, err)
  2138  
  2139  		assert.Nil(t, og)
  2140  		assert.Nil(t, dimensions)
  2141  	})
  2142  
  2143  	t.Run("neither", func(t *testing.T) {
  2144  		og, dimensions, err := th.App.parseLinkMetadata("http://example.com/test.wad", strings.NewReader("garbage"), "application/x-doom")
  2145  		assert.Nil(t, err)
  2146  
  2147  		assert.Nil(t, og)
  2148  		assert.Nil(t, dimensions)
  2149  	})
  2150  
  2151  	t.Run("svg", func(t *testing.T) {
  2152  		og, dimensions, err := th.App.parseLinkMetadata("http://example.com/image.svg", nil, "image/svg+xml")
  2153  		assert.Nil(t, err)
  2154  
  2155  		assert.Nil(t, og)
  2156  		assert.Equal(t, &model.PostImage{
  2157  			Format: "svg",
  2158  		}, dimensions)
  2159  	})
  2160  }
  2161  
  2162  func TestParseImages(t *testing.T) {
  2163  	for name, testCase := range map[string]struct {
  2164  		FileName    string
  2165  		Expected    *model.PostImage
  2166  		ExpectError bool
  2167  	}{
  2168  		"png": {
  2169  			FileName: "test.png",
  2170  			Expected: &model.PostImage{
  2171  				Width:  408,
  2172  				Height: 336,
  2173  				Format: "png",
  2174  			},
  2175  		},
  2176  		"animated gif": {
  2177  			FileName: "testgif.gif",
  2178  			Expected: &model.PostImage{
  2179  				Width:      118,
  2180  				Height:     118,
  2181  				Format:     "gif",
  2182  				FrameCount: 4,
  2183  			},
  2184  		},
  2185  		"tiff": {
  2186  			FileName: "test.tiff",
  2187  			Expected: (*model.PostImage)(nil),
  2188  		},
  2189  		"not an image": {
  2190  			FileName:    "README.md",
  2191  			ExpectError: true,
  2192  		},
  2193  	} {
  2194  		t.Run(name, func(t *testing.T) {
  2195  			file, err := testutils.ReadTestFile(testCase.FileName)
  2196  			require.Nil(t, err)
  2197  
  2198  			result, err := parseImages(bytes.NewReader(file))
  2199  			if testCase.ExpectError {
  2200  				assert.NotNil(t, err)
  2201  			} else {
  2202  				assert.Nil(t, err)
  2203  				assert.Equal(t, testCase.Expected, result)
  2204  			}
  2205  		})
  2206  	}
  2207  }