github.com/mad-app/mattermost-server@v5.11.1+incompatible/app/post_metadata_test.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "bytes" 8 "fmt" 9 "image" 10 "image/png" 11 "io" 12 "net/http" 13 "net/http/httptest" 14 "net/url" 15 "strconv" 16 "strings" 17 "testing" 18 "time" 19 20 "github.com/dyatlov/go-opengraph/opengraph" 21 "github.com/mattermost/mattermost-server/model" 22 "github.com/mattermost/mattermost-server/services/httpservice" 23 "github.com/mattermost/mattermost-server/services/imageproxy" 24 "github.com/mattermost/mattermost-server/utils/testutils" 25 "github.com/stretchr/testify/assert" 26 "github.com/stretchr/testify/require" 27 ) 28 29 func TestPreparePostListForClient(t *testing.T) { 30 // Most of this logic is covered by TestPreparePostForClient, so this just tests handling of multiple posts 31 32 th := Setup(t).InitBasic() 33 defer th.TearDown() 34 35 th.App.UpdateConfig(func(cfg *model.Config) { 36 *cfg.ExperimentalSettings.DisablePostMetadata = false 37 }) 38 39 postList := model.NewPostList() 40 for i := 0; i < 5; i++ { 41 postList.AddPost(&model.Post{}) 42 } 43 44 clientPostList := th.App.PreparePostListForClient(postList) 45 46 t.Run("doesn't mutate provided post list", func(t *testing.T) { 47 assert.NotEqual(t, clientPostList, postList, "should've returned a new post list") 48 assert.NotEqual(t, clientPostList.Posts, postList.Posts, "should've returned a new PostList.Posts") 49 assert.Equal(t, clientPostList.Order, postList.Order, "should've returned the existing PostList.Order") 50 51 for id, originalPost := range postList.Posts { 52 assert.NotEqual(t, clientPostList.Posts[id], originalPost, "should've returned new post objects") 53 assert.Equal(t, clientPostList.Posts[id].Id, originalPost.Id, "should've returned the same posts") 54 } 55 }) 56 57 t.Run("adds metadata to each post", func(t *testing.T) { 58 for _, clientPost := range clientPostList.Posts { 59 assert.NotNil(t, clientPost.Metadata, "should've populated metadata for each post") 60 } 61 }) 62 } 63 64 func TestPreparePostForClient(t *testing.T) { 65 setup := func() *TestHelper { 66 th := Setup(t).InitBasic() 67 68 th.App.UpdateConfig(func(cfg *model.Config) { 69 *cfg.ServiceSettings.EnableLinkPreviews = true 70 *cfg.ImageProxySettings.Enable = false 71 *cfg.ExperimentalSettings.DisablePostMetadata = false 72 }) 73 74 return th 75 } 76 77 t.Run("no metadata needed", func(t *testing.T) { 78 th := setup() 79 defer th.TearDown() 80 81 message := model.NewId() 82 post := &model.Post{ 83 Message: message, 84 } 85 86 clientPost := th.App.PreparePostForClient(post, false) 87 88 t.Run("doesn't mutate provided post", func(t *testing.T) { 89 assert.NotEqual(t, clientPost, post, "should've returned a new post") 90 91 assert.Equal(t, message, post.Message, "shouldn't have mutated post.Message") 92 assert.Equal(t, (*model.PostMetadata)(nil), post.Metadata, "shouldn't have mutated post.Metadata") 93 }) 94 95 t.Run("populates all fields", func(t *testing.T) { 96 assert.Equal(t, message, clientPost.Message, "shouldn't have changed Message") 97 assert.NotEqual(t, nil, clientPost.Metadata, "should've populated Metadata") 98 assert.Len(t, clientPost.Metadata.Embeds, 0, "should've populated Embeds") 99 assert.Len(t, clientPost.Metadata.Reactions, 0, "should've populated Reactions") 100 assert.Len(t, clientPost.Metadata.Files, 0, "should've populated Files") 101 assert.Len(t, clientPost.Metadata.Emojis, 0, "should've populated Emojis") 102 assert.Len(t, clientPost.Metadata.Images, 0, "should've populated Images") 103 }) 104 }) 105 106 t.Run("metadata already set", func(t *testing.T) { 107 th := setup() 108 defer th.TearDown() 109 110 post := th.CreatePost(th.BasicChannel) 111 112 clientPost := th.App.PreparePostForClient(post, false) 113 114 assert.False(t, clientPost == post, "should've returned a new post") 115 assert.Equal(t, clientPost, post, "shouldn't have changed any metadata") 116 }) 117 118 t.Run("reactions", func(t *testing.T) { 119 th := setup() 120 defer th.TearDown() 121 122 post := th.CreatePost(th.BasicChannel) 123 reaction1 := th.AddReactionToPost(post, th.BasicUser, "smile") 124 reaction2 := th.AddReactionToPost(post, th.BasicUser2, "smile") 125 reaction3 := th.AddReactionToPost(post, th.BasicUser2, "ice_cream") 126 post.HasReactions = true 127 128 clientPost := th.App.PreparePostForClient(post, false) 129 130 assert.Len(t, clientPost.Metadata.Reactions, 3, "should've populated Reactions") 131 assert.Equal(t, reaction1, clientPost.Metadata.Reactions[0], "first reaction is incorrect") 132 assert.Equal(t, reaction2, clientPost.Metadata.Reactions[1], "second reaction is incorrect") 133 assert.Equal(t, reaction3, clientPost.Metadata.Reactions[2], "third reaction is incorrect") 134 }) 135 136 t.Run("files", func(t *testing.T) { 137 th := setup() 138 defer th.TearDown() 139 140 fileInfo, err := th.App.DoUploadFile(time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test")) 141 require.Nil(t, err) 142 143 post, err := th.App.CreatePost(&model.Post{ 144 UserId: th.BasicUser.Id, 145 ChannelId: th.BasicChannel.Id, 146 FileIds: []string{fileInfo.Id}, 147 }, th.BasicChannel, false) 148 require.Nil(t, err) 149 150 fileInfo.PostId = post.Id 151 152 clientPost := th.App.PreparePostForClient(post, false) 153 154 assert.Equal(t, []*model.FileInfo{fileInfo}, clientPost.Metadata.Files, "should've populated Files") 155 }) 156 157 t.Run("emojis without custom emojis enabled", func(t *testing.T) { 158 th := setup() 159 defer th.TearDown() 160 161 th.App.UpdateConfig(func(cfg *model.Config) { 162 *cfg.ServiceSettings.EnableCustomEmoji = false 163 }) 164 165 emoji := th.CreateEmoji() 166 167 post, err := th.App.CreatePost(&model.Post{ 168 UserId: th.BasicUser.Id, 169 ChannelId: th.BasicChannel.Id, 170 Message: ":" + emoji.Name + ": :taco:", 171 Props: map[string]interface{}{ 172 "attachments": []*model.SlackAttachment{ 173 { 174 Text: ":" + emoji.Name + ":", 175 }, 176 }, 177 }, 178 }, th.BasicChannel, false) 179 require.Nil(t, err) 180 181 th.AddReactionToPost(post, th.BasicUser, "smile") 182 th.AddReactionToPost(post, th.BasicUser, "angry") 183 th.AddReactionToPost(post, th.BasicUser2, "angry") 184 post.HasReactions = true 185 186 clientPost := th.App.PreparePostForClient(post, false) 187 188 t.Run("populates emojis", func(t *testing.T) { 189 assert.ElementsMatch(t, []*model.Emoji{}, clientPost.Metadata.Emojis, "should've populated empty Emojis") 190 }) 191 192 t.Run("populates reaction counts", func(t *testing.T) { 193 reactions := clientPost.Metadata.Reactions 194 assert.Len(t, reactions, 3, "should've populated Reactions") 195 }) 196 }) 197 198 t.Run("emojis with custom emojis enabled", func(t *testing.T) { 199 th := setup() 200 defer th.TearDown() 201 202 th.App.UpdateConfig(func(cfg *model.Config) { 203 *cfg.ServiceSettings.EnableCustomEmoji = true 204 }) 205 206 emoji1 := th.CreateEmoji() 207 emoji2 := th.CreateEmoji() 208 emoji3 := th.CreateEmoji() 209 emoji4 := th.CreateEmoji() 210 211 post, err := th.App.CreatePost(&model.Post{ 212 UserId: th.BasicUser.Id, 213 ChannelId: th.BasicChannel.Id, 214 Message: ":" + emoji3.Name + ": :taco:", 215 Props: map[string]interface{}{ 216 "attachments": []*model.SlackAttachment{ 217 { 218 Text: ":" + emoji4.Name + ":", 219 }, 220 }, 221 }, 222 }, th.BasicChannel, false) 223 require.Nil(t, err) 224 225 th.AddReactionToPost(post, th.BasicUser, emoji1.Name) 226 th.AddReactionToPost(post, th.BasicUser, emoji2.Name) 227 th.AddReactionToPost(post, th.BasicUser2, emoji2.Name) 228 th.AddReactionToPost(post, th.BasicUser2, "angry") 229 post.HasReactions = true 230 231 clientPost := th.App.PreparePostForClient(post, false) 232 233 t.Run("pupulates emojis", func(t *testing.T) { 234 assert.ElementsMatch(t, []*model.Emoji{emoji1, emoji2, emoji3, emoji4}, clientPost.Metadata.Emojis, "should've populated post.Emojis") 235 }) 236 237 t.Run("populates reaction counts", func(t *testing.T) { 238 reactions := clientPost.Metadata.Reactions 239 assert.Len(t, reactions, 4, "should've populated Reactions") 240 }) 241 }) 242 243 t.Run("markdown image dimensions", func(t *testing.T) { 244 th := setup() 245 defer th.TearDown() 246 247 post, err := th.App.CreatePost(&model.Post{ 248 UserId: th.BasicUser.Id, 249 ChannelId: th.BasicChannel.Id, 250 Message: "This is ![our logo](https://github.com/hmhealey/test-files/raw/master/logoVertical.png) and ![our icon](https://github.com/hmhealey/test-files/raw/master/icon.png)", 251 }, th.BasicChannel, false) 252 require.Nil(t, err) 253 254 clientPost := th.App.PreparePostForClient(post, false) 255 256 t.Run("populates image dimensions", func(t *testing.T) { 257 imageDimensions := clientPost.Metadata.Images 258 require.Len(t, imageDimensions, 2) 259 assert.Equal(t, &model.PostImage{ 260 Format: "png", 261 Width: 1068, 262 Height: 552, 263 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/logoVertical.png"]) 264 assert.Equal(t, &model.PostImage{ 265 Format: "png", 266 Width: 501, 267 Height: 501, 268 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/icon.png"]) 269 }) 270 }) 271 272 t.Run("proxy linked images", func(t *testing.T) { 273 th := setup() 274 defer th.TearDown() 275 276 testProxyLinkedImage(t, th, false) 277 }) 278 279 t.Run("proxy opengraph images", func(t *testing.T) { 280 th := setup() 281 defer th.TearDown() 282 283 testProxyOpenGraphImage(t, th, false) 284 }) 285 286 t.Run("image embed", func(t *testing.T) { 287 th := setup() 288 defer th.TearDown() 289 290 post, err := th.App.CreatePost(&model.Post{ 291 UserId: th.BasicUser.Id, 292 ChannelId: th.BasicChannel.Id, 293 Message: `This is our logo: https://github.com/hmhealey/test-files/raw/master/logoVertical.png 294 And this is our icon: https://github.com/hmhealey/test-files/raw/master/icon.png`, 295 }, th.BasicChannel, false) 296 require.Nil(t, err) 297 298 clientPost := th.App.PreparePostForClient(post, false) 299 300 // Reminder that only the first link gets an embed and dimensions 301 302 t.Run("populates embeds", func(t *testing.T) { 303 assert.ElementsMatch(t, []*model.PostEmbed{ 304 { 305 Type: model.POST_EMBED_IMAGE, 306 URL: "https://github.com/hmhealey/test-files/raw/master/logoVertical.png", 307 }, 308 }, clientPost.Metadata.Embeds) 309 }) 310 311 t.Run("populates image dimensions", func(t *testing.T) { 312 imageDimensions := clientPost.Metadata.Images 313 require.Len(t, imageDimensions, 1) 314 assert.Equal(t, &model.PostImage{ 315 Format: "png", 316 Width: 1068, 317 Height: 552, 318 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/logoVertical.png"]) 319 }) 320 }) 321 322 t.Run("opengraph embed", func(t *testing.T) { 323 th := setup() 324 defer th.TearDown() 325 326 post, err := th.App.CreatePost(&model.Post{ 327 UserId: th.BasicUser.Id, 328 ChannelId: th.BasicChannel.Id, 329 Message: `This is our web page: https://github.com/hmhealey/test-files`, 330 }, th.BasicChannel, false) 331 require.Nil(t, err) 332 333 clientPost := th.App.PreparePostForClient(post, false) 334 335 t.Run("populates embeds", func(t *testing.T) { 336 assert.ElementsMatch(t, []*model.PostEmbed{ 337 { 338 Type: model.POST_EMBED_OPENGRAPH, 339 URL: "https://github.com/hmhealey/test-files", 340 Data: &opengraph.OpenGraph{ 341 Description: "Contribute to hmhealey/test-files development by creating an account on GitHub.", 342 SiteName: "GitHub", 343 Title: "hmhealey/test-files", 344 Type: "object", 345 URL: "https://github.com/hmhealey/test-files", 346 Images: []*opengraph.Image{ 347 { 348 URL: "https://avatars1.githubusercontent.com/u/3277310?s=400&v=4", 349 }, 350 }, 351 }, 352 }, 353 }, clientPost.Metadata.Embeds) 354 }) 355 356 t.Run("populates image dimensions", func(t *testing.T) { 357 imageDimensions := clientPost.Metadata.Images 358 require.Len(t, imageDimensions, 1) 359 assert.Equal(t, &model.PostImage{ 360 Format: "png", 361 Width: 420, 362 Height: 420, 363 }, imageDimensions["https://avatars1.githubusercontent.com/u/3277310?s=400&v=4"]) 364 }) 365 }) 366 367 t.Run("message attachment embed", func(t *testing.T) { 368 th := setup() 369 defer th.TearDown() 370 371 post, err := th.App.CreatePost(&model.Post{ 372 UserId: th.BasicUser.Id, 373 ChannelId: th.BasicChannel.Id, 374 Props: map[string]interface{}{ 375 "attachments": []interface{}{ 376 map[string]interface{}{ 377 "text": "![icon](https://github.com/hmhealey/test-files/raw/master/icon.png)", 378 }, 379 }, 380 }, 381 }, th.BasicChannel, false) 382 require.Nil(t, err) 383 384 clientPost := th.App.PreparePostForClient(post, false) 385 386 t.Run("populates embeds", func(t *testing.T) { 387 assert.ElementsMatch(t, []*model.PostEmbed{ 388 { 389 Type: model.POST_EMBED_MESSAGE_ATTACHMENT, 390 }, 391 }, clientPost.Metadata.Embeds) 392 }) 393 394 t.Run("populates image dimensions", func(t *testing.T) { 395 imageDimensions := clientPost.Metadata.Images 396 require.Len(t, imageDimensions, 1) 397 assert.Equal(t, &model.PostImage{ 398 Format: "png", 399 Width: 501, 400 Height: 501, 401 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/icon.png"]) 402 }) 403 }) 404 405 t.Run("when disabled", func(t *testing.T) { 406 th := setup() 407 defer th.TearDown() 408 409 th.App.UpdateConfig(func(cfg *model.Config) { 410 *cfg.ExperimentalSettings.DisablePostMetadata = true 411 }) 412 413 post := th.CreatePost(th.BasicChannel) 414 post = th.App.PreparePostForClient(post, false) 415 416 assert.Nil(t, post.Metadata) 417 418 b := post.ToJson() 419 420 assert.NotContains(t, string(b), "metadata", "json shouldn't include a metadata field, not even a falsey one") 421 }) 422 } 423 424 func TestPreparePostForClientWithImageProxy(t *testing.T) { 425 setup := func() *TestHelper { 426 th := Setup(t).InitBasic() 427 428 th.App.UpdateConfig(func(cfg *model.Config) { 429 *cfg.ServiceSettings.EnableLinkPreviews = true 430 *cfg.ServiceSettings.SiteURL = "http://mymattermost.com" 431 *cfg.ImageProxySettings.Enable = true 432 *cfg.ImageProxySettings.ImageProxyType = "atmos/camo" 433 *cfg.ImageProxySettings.RemoteImageProxyURL = "https://127.0.0.1" 434 *cfg.ImageProxySettings.RemoteImageProxyOptions = "foo" 435 *cfg.ExperimentalSettings.DisablePostMetadata = false 436 }) 437 438 return th 439 } 440 441 t.Run("proxy linked images", func(t *testing.T) { 442 th := setup() 443 defer th.TearDown() 444 445 testProxyLinkedImage(t, th, true) 446 }) 447 448 t.Run("proxy opengraph images", func(t *testing.T) { 449 th := setup() 450 defer th.TearDown() 451 452 testProxyOpenGraphImage(t, th, true) 453 }) 454 } 455 456 func testProxyLinkedImage(t *testing.T, th *TestHelper, shouldProxy bool) { 457 postTemplate := "![foo](%v)" 458 imageURL := "http://mydomain.com/myimage" 459 proxiedImageURL := "https://127.0.0.1/f8dace906d23689e8d5b12c3cefbedbf7b9b72f5/687474703a2f2f6d79646f6d61696e2e636f6d2f6d79696d616765" 460 461 post := &model.Post{ 462 UserId: th.BasicUser.Id, 463 ChannelId: th.BasicChannel.Id, 464 Message: fmt.Sprintf(postTemplate, imageURL), 465 } 466 467 clientPost := th.App.PreparePostForClient(post, false) 468 469 if shouldProxy { 470 assert.Equal(t, fmt.Sprintf(postTemplate, imageURL), post.Message, "should not have mutated original post") 471 assert.Equal(t, fmt.Sprintf(postTemplate, proxiedImageURL), clientPost.Message, "should've replaced linked image URLs") 472 } else { 473 assert.Equal(t, fmt.Sprintf(postTemplate, imageURL), clientPost.Message, "shouldn't have replaced linked image URLs") 474 } 475 } 476 477 func testProxyOpenGraphImage(t *testing.T, th *TestHelper, shouldProxy bool) { 478 post, err := th.App.CreatePost(&model.Post{ 479 UserId: th.BasicUser.Id, 480 ChannelId: th.BasicChannel.Id, 481 Message: `This is our web page: https://github.com/hmhealey/test-files`, 482 }, th.BasicChannel, false) 483 require.Nil(t, err) 484 485 embeds := th.App.PreparePostForClient(post, false).Metadata.Embeds 486 require.Len(t, embeds, 1, "should have one embed") 487 488 embed := embeds[0] 489 assert.Equal(t, model.POST_EMBED_OPENGRAPH, embed.Type, "embed type should be OpenGraph") 490 assert.Equal(t, "https://github.com/hmhealey/test-files", embed.URL, "embed URL should be correct") 491 492 og, ok := embed.Data.(*opengraph.OpenGraph) 493 assert.Equal(t, true, ok, "data should be non-nil OpenGraph data") 494 assert.Equal(t, "GitHub", og.SiteName, "OpenGraph data should be correctly populated") 495 496 require.Len(t, og.Images, 1, "OpenGraph data should have one image") 497 498 image := og.Images[0] 499 if shouldProxy { 500 assert.Equal(t, "", image.URL, "image URL should not be set with proxy") 501 assert.Equal(t, "https://127.0.0.1/b2ef6ef4890a0107aa80ba33b3011fd51f668303/68747470733a2f2f61766174617273312e67697468756275736572636f6e74656e742e636f6d2f752f333237373331303f733d34303026763d34", image.SecureURL, "secure image URL should be sent through proxy") 502 } else { 503 assert.Equal(t, "https://avatars1.githubusercontent.com/u/3277310?s=400&v=4", image.URL, "image URL should be set") 504 assert.Equal(t, "", image.SecureURL, "secure image URL should not be set") 505 } 506 } 507 508 func TestGetEmbedForPost(t *testing.T) { 509 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 510 if r.URL.Path == "/index.html" { 511 w.Header().Set("Content-Type", "text/html") 512 w.Write([]byte(` 513 <html> 514 <head> 515 <meta property="og:title" content="Title" /> 516 </head> 517 </html>`)) 518 } else if r.URL.Path == "/image.png" { 519 file, err := testutils.ReadTestFile("test.png") 520 require.Nil(t, err) 521 522 w.Header().Set("Content-Type", "image/png") 523 w.Write(file) 524 } else { 525 t.Fatal("Invalid path", r.URL.Path) 526 } 527 })) 528 defer server.Close() 529 530 ogURL := server.URL + "/index.html" 531 imageURL := server.URL + "/image.png" 532 533 t.Run("with link previews enabled", func(t *testing.T) { 534 th := Setup(t) 535 defer th.TearDown() 536 537 th.App.UpdateConfig(func(cfg *model.Config) { 538 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 539 *cfg.ServiceSettings.EnableLinkPreviews = true 540 }) 541 542 t.Run("should return a message attachment when the post has one", func(t *testing.T) { 543 embed, err := th.App.getEmbedForPost(&model.Post{ 544 Props: model.StringInterface{ 545 "attachments": []*model.SlackAttachment{ 546 { 547 Text: "test", 548 }, 549 }, 550 }, 551 }, "", false) 552 553 assert.Equal(t, &model.PostEmbed{ 554 Type: model.POST_EMBED_MESSAGE_ATTACHMENT, 555 }, embed) 556 assert.Nil(t, err) 557 }) 558 559 t.Run("should return an image embed when the first link is an image", func(t *testing.T) { 560 embed, err := th.App.getEmbedForPost(&model.Post{}, imageURL, false) 561 562 assert.Equal(t, &model.PostEmbed{ 563 Type: model.POST_EMBED_IMAGE, 564 URL: imageURL, 565 }, embed) 566 assert.Nil(t, err) 567 }) 568 569 t.Run("should return an image embed when the first link is an image", func(t *testing.T) { 570 embed, err := th.App.getEmbedForPost(&model.Post{}, ogURL, false) 571 572 assert.Equal(t, &model.PostEmbed{ 573 Type: model.POST_EMBED_OPENGRAPH, 574 URL: ogURL, 575 Data: &opengraph.OpenGraph{ 576 Title: "Title", 577 }, 578 }, embed) 579 assert.Nil(t, err) 580 }) 581 }) 582 583 t.Run("with link previews disabled", func(t *testing.T) { 584 th := Setup(t) 585 defer th.TearDown() 586 587 th.App.UpdateConfig(func(cfg *model.Config) { 588 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 589 *cfg.ServiceSettings.EnableLinkPreviews = false 590 }) 591 592 t.Run("should return an embedded message attachment", func(t *testing.T) { 593 embed, err := th.App.getEmbedForPost(&model.Post{ 594 Props: model.StringInterface{ 595 "attachments": []*model.SlackAttachment{ 596 { 597 Text: "test", 598 }, 599 }, 600 }, 601 }, "", false) 602 603 assert.Equal(t, &model.PostEmbed{ 604 Type: model.POST_EMBED_MESSAGE_ATTACHMENT, 605 }, embed) 606 assert.Nil(t, err) 607 }) 608 609 t.Run("should not return an opengraph embed", func(t *testing.T) { 610 embed, err := th.App.getEmbedForPost(&model.Post{}, ogURL, false) 611 612 assert.Nil(t, embed) 613 assert.Nil(t, err) 614 }) 615 616 t.Run("should not return an image embed", func(t *testing.T) { 617 embed, err := th.App.getEmbedForPost(&model.Post{}, imageURL, false) 618 619 assert.Nil(t, embed) 620 assert.Nil(t, err) 621 }) 622 }) 623 } 624 625 func TestGetImagesForPost(t *testing.T) { 626 t.Run("with an image link", func(t *testing.T) { 627 th := Setup(t) 628 defer th.TearDown() 629 630 th.App.UpdateConfig(func(cfg *model.Config) { 631 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 632 }) 633 634 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 635 file, err := testutils.ReadTestFile("test.png") 636 require.Nil(t, err) 637 638 w.Header().Set("Content-Type", "image/png") 639 w.Write(file) 640 })) 641 642 post := &model.Post{ 643 Metadata: &model.PostMetadata{}, 644 } 645 imageURL := server.URL + "/image.png" 646 647 images := th.App.getImagesForPost(post, []string{imageURL}, false) 648 649 assert.Equal(t, images, map[string]*model.PostImage{ 650 imageURL: { 651 Format: "png", 652 Width: 408, 653 Height: 336, 654 }, 655 }) 656 }) 657 658 t.Run("with an invalid image link", func(t *testing.T) { 659 th := Setup(t) 660 defer th.TearDown() 661 662 th.App.UpdateConfig(func(cfg *model.Config) { 663 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 664 }) 665 666 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 667 w.WriteHeader(http.StatusInternalServerError) 668 })) 669 670 post := &model.Post{ 671 Metadata: &model.PostMetadata{}, 672 } 673 imageURL := server.URL + "/bad_image.png" 674 675 images := th.App.getImagesForPost(post, []string{imageURL}, false) 676 677 assert.Equal(t, images, map[string]*model.PostImage{}) 678 }) 679 680 t.Run("for an OpenGraph image", func(t *testing.T) { 681 th := Setup(t) 682 defer th.TearDown() 683 684 th.App.UpdateConfig(func(cfg *model.Config) { 685 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 686 }) 687 688 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 689 if r.URL.Path == "/image.png" { 690 w.Header().Set("Content-Type", "image/png") 691 692 img := image.NewGray(image.Rect(0, 0, 200, 300)) 693 694 var encoder png.Encoder 695 encoder.Encode(w, img) 696 } else { 697 w.WriteHeader(http.StatusNotFound) 698 } 699 })) 700 defer server.Close() 701 702 ogURL := server.URL + "/index.html" 703 imageURL := server.URL + "/image.png" 704 705 post := &model.Post{ 706 Metadata: &model.PostMetadata{ 707 Embeds: []*model.PostEmbed{ 708 { 709 Type: model.POST_EMBED_OPENGRAPH, 710 URL: ogURL, 711 Data: &opengraph.OpenGraph{ 712 Images: []*opengraph.Image{ 713 { 714 URL: imageURL, 715 }, 716 }, 717 }, 718 }, 719 }, 720 }, 721 } 722 723 images := th.App.getImagesForPost(post, []string{}, false) 724 725 assert.Equal(t, images, map[string]*model.PostImage{ 726 imageURL: { 727 Format: "png", 728 Width: 200, 729 Height: 300, 730 }, 731 }) 732 }) 733 734 t.Run("with an OpenGraph image with a secure_url", func(t *testing.T) { 735 th := Setup(t) 736 defer th.TearDown() 737 738 th.App.UpdateConfig(func(cfg *model.Config) { 739 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 740 }) 741 742 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 743 if r.URL.Path == "/secure_image.png" { 744 w.Header().Set("Content-Type", "image/png") 745 746 img := image.NewGray(image.Rect(0, 0, 300, 400)) 747 748 var encoder png.Encoder 749 encoder.Encode(w, img) 750 } else { 751 w.WriteHeader(http.StatusNotFound) 752 } 753 })) 754 defer server.Close() 755 756 ogURL := server.URL + "/index.html" 757 imageURL := server.URL + "/secure_image.png" 758 759 post := &model.Post{ 760 Metadata: &model.PostMetadata{ 761 Embeds: []*model.PostEmbed{ 762 { 763 Type: model.POST_EMBED_OPENGRAPH, 764 URL: ogURL, 765 Data: &opengraph.OpenGraph{ 766 Images: []*opengraph.Image{ 767 { 768 SecureURL: imageURL, 769 }, 770 }, 771 }, 772 }, 773 }, 774 }, 775 } 776 777 images := th.App.getImagesForPost(post, []string{}, false) 778 779 assert.Equal(t, images, map[string]*model.PostImage{ 780 imageURL: { 781 Format: "png", 782 Width: 300, 783 Height: 400, 784 }, 785 }) 786 }) 787 788 t.Run("with an OpenGraph image with a secure_url and no dimensions", func(t *testing.T) { 789 th := Setup(t) 790 defer th.TearDown() 791 792 th.App.UpdateConfig(func(cfg *model.Config) { 793 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 794 }) 795 796 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 797 if r.URL.Path == "/secure_image.png" { 798 w.Header().Set("Content-Type", "image/png") 799 800 img := image.NewGray(image.Rect(0, 0, 400, 500)) 801 802 var encoder png.Encoder 803 encoder.Encode(w, img) 804 } else { 805 w.WriteHeader(http.StatusNotFound) 806 } 807 })) 808 809 ogURL := server.URL + "/index.html" 810 imageURL := server.URL + "/secure_image.png" 811 812 post := &model.Post{ 813 Metadata: &model.PostMetadata{ 814 Embeds: []*model.PostEmbed{ 815 { 816 Type: model.POST_EMBED_OPENGRAPH, 817 URL: ogURL, 818 Data: &opengraph.OpenGraph{ 819 Images: []*opengraph.Image{ 820 { 821 URL: server.URL + "/image.png", 822 SecureURL: imageURL, 823 }, 824 }, 825 }, 826 }, 827 }, 828 }, 829 } 830 831 images := th.App.getImagesForPost(post, []string{}, false) 832 833 assert.Equal(t, images, map[string]*model.PostImage{ 834 imageURL: { 835 Format: "png", 836 Width: 400, 837 Height: 500, 838 }, 839 }) 840 }) 841 } 842 843 func TestGetEmojiNamesForString(t *testing.T) { 844 testCases := []struct { 845 Description string 846 Input string 847 Expected []string 848 }{ 849 { 850 Description: "no emojis", 851 Input: "this is a string", 852 Expected: []string{}, 853 }, 854 { 855 Description: "one emoji", 856 Input: "this is an :emoji1: string", 857 Expected: []string{"emoji1"}, 858 }, 859 { 860 Description: "two emojis", 861 Input: "this is a :emoji3: :emoji2: string", 862 Expected: []string{"emoji3", "emoji2"}, 863 }, 864 { 865 Description: "punctuation around emojis", 866 Input: ":emoji3:/:emoji1: (:emoji2:)", 867 Expected: []string{"emoji3", "emoji1", "emoji2"}, 868 }, 869 { 870 Description: "adjacent emojis", 871 Input: ":emoji3::emoji1:", 872 Expected: []string{"emoji3", "emoji1"}, 873 }, 874 { 875 Description: "duplicate emojis", 876 Input: ":emoji1: :emoji1: :emoji1::emoji2::emoji2: :emoji1:", 877 Expected: []string{"emoji1", "emoji1", "emoji1", "emoji2", "emoji2", "emoji1"}, 878 }, 879 { 880 Description: "fake emojis", 881 Input: "these don't exist :tomato: :potato: :rotato:", 882 Expected: []string{"tomato", "potato", "rotato"}, 883 }, 884 } 885 886 for _, testCase := range testCases { 887 testCase := testCase 888 t.Run(testCase.Description, func(t *testing.T) { 889 emojis := getEmojiNamesForString(testCase.Input) 890 assert.ElementsMatch(t, emojis, testCase.Expected, "received incorrect emoji names") 891 }) 892 } 893 } 894 895 func TestGetEmojiNamesForPost(t *testing.T) { 896 testCases := []struct { 897 Description string 898 Post *model.Post 899 Reactions []*model.Reaction 900 Expected []string 901 }{ 902 { 903 Description: "no emojis", 904 Post: &model.Post{ 905 Message: "this is a post", 906 }, 907 Expected: []string{}, 908 }, 909 { 910 Description: "in post message", 911 Post: &model.Post{ 912 Message: "this is :emoji:", 913 }, 914 Expected: []string{"emoji"}, 915 }, 916 { 917 Description: "in reactions", 918 Post: &model.Post{}, 919 Reactions: []*model.Reaction{ 920 { 921 EmojiName: "emoji1", 922 }, 923 { 924 EmojiName: "emoji2", 925 }, 926 }, 927 Expected: []string{"emoji1", "emoji2"}, 928 }, 929 { 930 Description: "in message attachments", 931 Post: &model.Post{ 932 Message: "this is a post", 933 Props: map[string]interface{}{ 934 "attachments": []*model.SlackAttachment{ 935 { 936 Text: ":emoji1:", 937 Pretext: ":emoji2:", 938 }, 939 { 940 Fields: []*model.SlackAttachmentField{ 941 { 942 Value: ":emoji3:", 943 }, 944 { 945 Value: ":emoji4:", 946 }, 947 }, 948 }, 949 }, 950 }, 951 }, 952 Expected: []string{"emoji1", "emoji2", "emoji3", "emoji4"}, 953 }, 954 { 955 Description: "with duplicates", 956 Post: &model.Post{ 957 Message: "this is :emoji1", 958 Props: map[string]interface{}{ 959 "attachments": []*model.SlackAttachment{ 960 { 961 Text: ":emoji2:", 962 Pretext: ":emoji2:", 963 Fields: []*model.SlackAttachmentField{ 964 { 965 Value: ":emoji3:", 966 }, 967 { 968 Value: ":emoji1:", 969 }, 970 }, 971 }, 972 }, 973 }, 974 }, 975 Expected: []string{"emoji1", "emoji2", "emoji3"}, 976 }, 977 } 978 979 for _, testCase := range testCases { 980 testCase := testCase 981 t.Run(testCase.Description, func(t *testing.T) { 982 emojis := getEmojiNamesForPost(testCase.Post, testCase.Reactions) 983 assert.ElementsMatch(t, emojis, testCase.Expected, "received incorrect emoji names") 984 }) 985 } 986 } 987 988 func TestGetCustomEmojisForPost(t *testing.T) { 989 th := Setup(t).InitBasic() 990 defer th.TearDown() 991 992 th.App.UpdateConfig(func(cfg *model.Config) { 993 *cfg.ServiceSettings.EnableCustomEmoji = true 994 }) 995 996 emojis := []*model.Emoji{ 997 th.CreateEmoji(), 998 th.CreateEmoji(), 999 th.CreateEmoji(), 1000 th.CreateEmoji(), 1001 th.CreateEmoji(), 1002 th.CreateEmoji(), 1003 } 1004 1005 t.Run("from different parts of the post", func(t *testing.T) { 1006 reactions := []*model.Reaction{ 1007 { 1008 UserId: th.BasicUser.Id, 1009 EmojiName: emojis[0].Name, 1010 }, 1011 } 1012 1013 post := &model.Post{ 1014 Message: ":" + emojis[1].Name + ":", 1015 Props: map[string]interface{}{ 1016 "attachments": []*model.SlackAttachment{ 1017 { 1018 Pretext: ":" + emojis[2].Name + ":", 1019 Text: ":" + emojis[3].Name + ":", 1020 Fields: []*model.SlackAttachmentField{ 1021 { 1022 Value: ":" + emojis[4].Name + ":", 1023 }, 1024 { 1025 Value: ":" + emojis[5].Name + ":", 1026 }, 1027 }, 1028 }, 1029 }, 1030 }, 1031 } 1032 1033 emojisForPost, err := th.App.getCustomEmojisForPost(post, reactions) 1034 assert.Nil(t, err, "failed to get emojis for post") 1035 assert.ElementsMatch(t, emojisForPost, emojis, "received incorrect emojis") 1036 }) 1037 1038 t.Run("with emojis that don't exist", func(t *testing.T) { 1039 post := &model.Post{ 1040 Message: ":secret: :" + emojis[0].Name + ":", 1041 Props: map[string]interface{}{ 1042 "attachments": []*model.SlackAttachment{ 1043 { 1044 Text: ":imaginary:", 1045 }, 1046 }, 1047 }, 1048 } 1049 1050 emojisForPost, err := th.App.getCustomEmojisForPost(post, nil) 1051 assert.Nil(t, err, "failed to get emojis for post") 1052 assert.ElementsMatch(t, emojisForPost, []*model.Emoji{emojis[0]}, "received incorrect emojis") 1053 }) 1054 1055 t.Run("with no emojis", func(t *testing.T) { 1056 post := &model.Post{ 1057 Message: "this post is boring", 1058 Props: map[string]interface{}{}, 1059 } 1060 1061 emojisForPost, err := th.App.getCustomEmojisForPost(post, nil) 1062 assert.Nil(t, err, "failed to get emojis for post") 1063 assert.ElementsMatch(t, emojisForPost, []*model.Emoji{}, "should have received no emojis") 1064 }) 1065 } 1066 1067 func TestGetFirstLinkAndImages(t *testing.T) { 1068 for name, testCase := range map[string]struct { 1069 Input string 1070 ExpectedFirstLink string 1071 ExpectedImages []string 1072 }{ 1073 "no links or images": { 1074 Input: "this is a string", 1075 ExpectedFirstLink: "", 1076 ExpectedImages: []string{}, 1077 }, 1078 "http link": { 1079 Input: "this is a http://example.com", 1080 ExpectedFirstLink: "http://example.com", 1081 ExpectedImages: []string{}, 1082 }, 1083 "www link": { 1084 Input: "this is a www.example.com", 1085 ExpectedFirstLink: "http://www.example.com", 1086 ExpectedImages: []string{}, 1087 }, 1088 "image": { 1089 Input: "this is a ![our logo](http://example.com/logo)", 1090 ExpectedFirstLink: "", 1091 ExpectedImages: []string{"http://example.com/logo"}, 1092 }, 1093 "multiple images": { 1094 Input: "this is a ![our logo](http://example.com/logo) and ![their logo](http://example.com/logo2) and ![my logo](http://example.com/logo3)", 1095 ExpectedFirstLink: "", 1096 ExpectedImages: []string{"http://example.com/logo", "http://example.com/logo2", "http://example.com/logo3"}, 1097 }, 1098 "multiple images with duplicate": { 1099 Input: "this is a ![our logo](http://example.com/logo) and ![their logo](http://example.com/logo2) and ![my logo which is their logo](http://example.com/logo2)", 1100 ExpectedFirstLink: "", 1101 ExpectedImages: []string{"http://example.com/logo", "http://example.com/logo2", "http://example.com/logo2"}, 1102 }, 1103 "reference image": { 1104 Input: `this is a ![our logo][logo] 1105 1106 [logo]: http://example.com/logo`, 1107 ExpectedFirstLink: "", 1108 ExpectedImages: []string{"http://example.com/logo"}, 1109 }, 1110 "image and link": { 1111 Input: "this is a https://example.com and ![our logo](https://example.com/logo)", 1112 ExpectedFirstLink: "https://example.com", 1113 ExpectedImages: []string{"https://example.com/logo"}, 1114 }, 1115 "markdown links (not returned)": { 1116 Input: `this is a [our page](http://example.com) and [another page][] 1117 1118 [another page]: http://www.exaple.com/another_page`, 1119 ExpectedFirstLink: "", 1120 ExpectedImages: []string{}, 1121 }, 1122 } { 1123 t.Run(name, func(t *testing.T) { 1124 firstLink, images := getFirstLinkAndImages(testCase.Input) 1125 1126 assert.Equal(t, firstLink, testCase.ExpectedFirstLink) 1127 assert.Equal(t, images, testCase.ExpectedImages) 1128 }) 1129 } 1130 } 1131 1132 func TestGetImagesInMessageAttachments(t *testing.T) { 1133 for _, test := range []struct { 1134 Name string 1135 Post *model.Post 1136 Expected []string 1137 }{ 1138 { 1139 Name: "no attachments", 1140 Post: &model.Post{}, 1141 Expected: []string{}, 1142 }, 1143 { 1144 Name: "empty attachments", 1145 Post: &model.Post{ 1146 Props: map[string]interface{}{ 1147 "attachments": []*model.SlackAttachment{}, 1148 }, 1149 }, 1150 Expected: []string{}, 1151 }, 1152 { 1153 Name: "attachment with no fields that can contain images", 1154 Post: &model.Post{ 1155 Props: map[string]interface{}{ 1156 "attachments": []*model.SlackAttachment{ 1157 { 1158 Title: "This is the title", 1159 }, 1160 }, 1161 }, 1162 }, 1163 Expected: []string{}, 1164 }, 1165 { 1166 Name: "images in text", 1167 Post: &model.Post{ 1168 Props: map[string]interface{}{ 1169 "attachments": []*model.SlackAttachment{ 1170 { 1171 Text: "![logo](https://example.com/logo) and ![icon](https://example.com/icon)", 1172 }, 1173 }, 1174 }, 1175 }, 1176 Expected: []string{"https://example.com/logo", "https://example.com/icon"}, 1177 }, 1178 { 1179 Name: "images in pretext", 1180 Post: &model.Post{ 1181 Props: map[string]interface{}{ 1182 "attachments": []*model.SlackAttachment{ 1183 { 1184 Pretext: "![logo](https://example.com/logo1) and ![icon](https://example.com/icon1)", 1185 }, 1186 }, 1187 }, 1188 }, 1189 Expected: []string{"https://example.com/logo1", "https://example.com/icon1"}, 1190 }, 1191 { 1192 Name: "images in fields", 1193 Post: &model.Post{ 1194 Props: map[string]interface{}{ 1195 "attachments": []*model.SlackAttachment{ 1196 { 1197 Fields: []*model.SlackAttachmentField{ 1198 { 1199 Value: "![logo](https://example.com/logo2) and ![icon](https://example.com/icon2)", 1200 }, 1201 }, 1202 }, 1203 }, 1204 }, 1205 }, 1206 Expected: []string{"https://example.com/logo2", "https://example.com/icon2"}, 1207 }, 1208 { 1209 Name: "image in author_icon", 1210 Post: &model.Post{ 1211 Props: map[string]interface{}{ 1212 "attachments": []*model.SlackAttachment{ 1213 { 1214 AuthorIcon: "https://example.com/icon2", 1215 }, 1216 }, 1217 }, 1218 }, 1219 Expected: []string{"https://example.com/icon2"}, 1220 }, 1221 { 1222 Name: "image in image_url", 1223 Post: &model.Post{ 1224 Props: map[string]interface{}{ 1225 "attachments": []*model.SlackAttachment{ 1226 { 1227 ImageURL: "https://example.com/image", 1228 }, 1229 }, 1230 }, 1231 }, 1232 Expected: []string{"https://example.com/image"}, 1233 }, 1234 { 1235 Name: "image in thumb_url", 1236 Post: &model.Post{ 1237 Props: map[string]interface{}{ 1238 "attachments": []*model.SlackAttachment{ 1239 { 1240 ThumbURL: "https://example.com/image", 1241 }, 1242 }, 1243 }, 1244 }, 1245 Expected: []string{"https://example.com/image"}, 1246 }, 1247 { 1248 Name: "image in footer_icon", 1249 Post: &model.Post{ 1250 Props: map[string]interface{}{ 1251 "attachments": []*model.SlackAttachment{ 1252 { 1253 FooterIcon: "https://example.com/image", 1254 }, 1255 }, 1256 }, 1257 }, 1258 Expected: []string{"https://example.com/image"}, 1259 }, 1260 { 1261 Name: "images in multiple fields", 1262 Post: &model.Post{ 1263 Props: map[string]interface{}{ 1264 "attachments": []*model.SlackAttachment{ 1265 { 1266 Fields: []*model.SlackAttachmentField{ 1267 { 1268 Value: "![logo](https://example.com/logo)", 1269 }, 1270 { 1271 Value: "![icon](https://example.com/icon)", 1272 }, 1273 }, 1274 }, 1275 }, 1276 }, 1277 }, 1278 Expected: []string{"https://example.com/logo", "https://example.com/icon"}, 1279 }, 1280 { 1281 Name: "non-string field", 1282 Post: &model.Post{ 1283 Props: map[string]interface{}{ 1284 "attachments": []*model.SlackAttachment{ 1285 { 1286 Fields: []*model.SlackAttachmentField{ 1287 { 1288 Value: 77, 1289 }, 1290 }, 1291 }, 1292 }, 1293 }, 1294 }, 1295 Expected: []string{}, 1296 }, 1297 { 1298 Name: "images in multiple locations", 1299 Post: &model.Post{ 1300 Props: map[string]interface{}{ 1301 "attachments": []*model.SlackAttachment{ 1302 { 1303 Text: "![text](https://example.com/text)", 1304 Pretext: "![pretext](https://example.com/pretext)", 1305 Fields: []*model.SlackAttachmentField{ 1306 { 1307 Value: "![field1](https://example.com/field1)", 1308 }, 1309 { 1310 Value: "![field2](https://example.com/field2)", 1311 }, 1312 }, 1313 }, 1314 }, 1315 }, 1316 }, 1317 Expected: []string{"https://example.com/text", "https://example.com/pretext", "https://example.com/field1", "https://example.com/field2"}, 1318 }, 1319 { 1320 Name: "multiple attachments", 1321 Post: &model.Post{ 1322 Props: map[string]interface{}{ 1323 "attachments": []*model.SlackAttachment{ 1324 { 1325 Text: "![logo](https://example.com/logo)", 1326 }, 1327 { 1328 Text: "![icon](https://example.com/icon)", 1329 }, 1330 }, 1331 }, 1332 }, 1333 Expected: []string{"https://example.com/logo", "https://example.com/icon"}, 1334 }, 1335 } { 1336 t.Run(test.Name, func(t *testing.T) { 1337 images := getImagesInMessageAttachments(test.Post) 1338 1339 assert.ElementsMatch(t, images, test.Expected) 1340 }) 1341 } 1342 } 1343 1344 func TestGetLinkMetadata(t *testing.T) { 1345 setup := func() *TestHelper { 1346 th := Setup(t).InitBasic() 1347 1348 th.App.UpdateConfig(func(cfg *model.Config) { 1349 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 1350 }) 1351 1352 linkCache.Purge() 1353 1354 return th 1355 } 1356 1357 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 1358 params := r.URL.Query() 1359 1360 if strings.HasPrefix(r.URL.Path, "/image") { 1361 height, _ := strconv.ParseInt(params["height"][0], 10, 0) 1362 width, _ := strconv.ParseInt(params["width"][0], 10, 0) 1363 1364 img := image.NewGray(image.Rect(0, 0, int(width), int(height))) 1365 1366 var encoder png.Encoder 1367 1368 encoder.Encode(w, img) 1369 } else if strings.HasPrefix(r.URL.Path, "/opengraph") { 1370 w.Header().Set("Content-Type", "text/html") 1371 1372 w.Write([]byte(` 1373 <html prefix="og:http://ogp.me/ns#"> 1374 <head> 1375 <meta property="og:title" content="` + params["title"][0] + `" /> 1376 </head> 1377 <body> 1378 </body> 1379 </html>`)) 1380 } else if strings.HasPrefix(r.URL.Path, "/json") { 1381 w.Header().Set("Content-Type", "application/json") 1382 1383 w.Write([]byte("true")) 1384 } else if strings.HasPrefix(r.URL.Path, "/timeout") { 1385 w.Header().Set("Content-Type", "text/html") 1386 1387 w.Write([]byte("<html>")) 1388 select { 1389 case <-time.After(60 * time.Second): 1390 case <-r.Context().Done(): 1391 } 1392 w.Write([]byte("</html>")) 1393 } else { 1394 w.WriteHeader(http.StatusInternalServerError) 1395 } 1396 })) 1397 defer server.Close() 1398 1399 t.Run("in-memory cache", func(t *testing.T) { 1400 th := setup() 1401 defer th.TearDown() 1402 1403 requestURL := server.URL + "/cached" 1404 timestamp := int64(1547510400000) 1405 title := "from cache" 1406 1407 cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil) 1408 1409 t.Run("should use cache if cached entry exists", func(t *testing.T) { 1410 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1411 require.True(t, ok, "data should already exist in in-memory cache") 1412 1413 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1414 require.False(t, ok, "data should not exist in database") 1415 1416 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1417 1418 require.NotNil(t, og) 1419 assert.Nil(t, img) 1420 assert.Nil(t, err) 1421 assert.Equal(t, title, og.Title) 1422 }) 1423 1424 t.Run("should use cache if cached entry exists near time", func(t *testing.T) { 1425 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1426 require.True(t, ok, "data should already exist in in-memory cache") 1427 1428 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1429 require.False(t, ok, "data should not exist in database") 1430 1431 og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) 1432 1433 require.NotNil(t, og) 1434 assert.Nil(t, img) 1435 assert.Nil(t, err) 1436 assert.Equal(t, title, og.Title) 1437 }) 1438 1439 t.Run("should not use cache if URL is different", func(t *testing.T) { 1440 differentURL := server.URL + "/other" 1441 1442 _, _, ok := getLinkMetadataFromCache(differentURL, timestamp) 1443 require.False(t, ok, "data should not exist in in-memory cache") 1444 1445 _, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp) 1446 require.False(t, ok, "data should not exist in database") 1447 1448 og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false) 1449 1450 assert.Nil(t, og) 1451 assert.Nil(t, img) 1452 assert.Nil(t, err) 1453 }) 1454 1455 t.Run("should not use cache if timestamp is different", func(t *testing.T) { 1456 differentTimestamp := timestamp + 60*60*1000 1457 1458 _, _, ok := getLinkMetadataFromCache(requestURL, differentTimestamp) 1459 require.False(t, ok, "data should not exist in in-memory cache") 1460 1461 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp) 1462 require.False(t, ok, "data should not exist in database") 1463 1464 og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) 1465 1466 assert.Nil(t, og) 1467 assert.Nil(t, img) 1468 assert.Nil(t, err) 1469 }) 1470 }) 1471 1472 t.Run("database cache", func(t *testing.T) { 1473 th := setup() 1474 defer th.TearDown() 1475 1476 requestURL := server.URL 1477 timestamp := int64(1547510400000) 1478 title := "from database" 1479 1480 th.App.saveLinkMetadataToDatabase(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil) 1481 1482 t.Run("should use database if saved entry exists", func(t *testing.T) { 1483 linkCache.Purge() 1484 1485 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1486 require.False(t, ok, "data should not exist in in-memory cache") 1487 1488 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1489 require.True(t, ok, "data should already exist in database") 1490 1491 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1492 1493 require.NotNil(t, og) 1494 assert.Nil(t, img) 1495 assert.Nil(t, err) 1496 assert.Equal(t, title, og.Title) 1497 }) 1498 1499 t.Run("should use database if saved entry exists near time", func(t *testing.T) { 1500 linkCache.Purge() 1501 1502 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1503 require.False(t, ok, "data should not exist in in-memory cache") 1504 1505 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1506 require.True(t, ok, "data should already exist in database") 1507 1508 og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) 1509 1510 require.NotNil(t, og) 1511 assert.Nil(t, img) 1512 assert.Nil(t, err) 1513 assert.Equal(t, title, og.Title) 1514 }) 1515 1516 t.Run("should not use database if URL is different", func(t *testing.T) { 1517 linkCache.Purge() 1518 1519 differentURL := requestURL + "/other" 1520 1521 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1522 require.False(t, ok, "data should not exist in in-memory cache") 1523 1524 _, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp) 1525 require.False(t, ok, "data should not exist in database") 1526 1527 og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false) 1528 1529 assert.Nil(t, og) 1530 assert.Nil(t, img) 1531 assert.Nil(t, err) 1532 }) 1533 1534 t.Run("should not use database if timestamp is different", func(t *testing.T) { 1535 linkCache.Purge() 1536 1537 differentTimestamp := timestamp + 60*60*1000 1538 1539 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1540 require.False(t, ok, "data should not exist in in-memory cache") 1541 1542 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp) 1543 require.False(t, ok, "data should not exist in database") 1544 1545 og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) 1546 1547 assert.Nil(t, og) 1548 assert.Nil(t, img) 1549 assert.Nil(t, err) 1550 }) 1551 }) 1552 1553 t.Run("should get data from remote source", func(t *testing.T) { 1554 th := setup() 1555 defer th.TearDown() 1556 1557 requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name() 1558 timestamp := int64(1547510400000) 1559 1560 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1561 require.False(t, ok, "data should not exist in in-memory cache") 1562 1563 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1564 require.False(t, ok, "data should not exist in database") 1565 1566 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1567 1568 assert.NotNil(t, og) 1569 assert.Nil(t, img) 1570 assert.Nil(t, err) 1571 }) 1572 1573 t.Run("should cache OpenGraph results", func(t *testing.T) { 1574 th := setup() 1575 defer th.TearDown() 1576 1577 requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name() 1578 timestamp := int64(1547510400000) 1579 1580 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1581 require.False(t, ok, "data should not exist in in-memory cache") 1582 1583 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1584 require.False(t, ok, "data should not exist in database") 1585 1586 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1587 1588 assert.NotNil(t, og) 1589 assert.Nil(t, img) 1590 assert.Nil(t, err) 1591 1592 fromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1593 assert.True(t, ok) 1594 assert.Exactly(t, og, fromCache) 1595 1596 fromDatabase, _, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1597 assert.True(t, ok) 1598 assert.Exactly(t, og, fromDatabase) 1599 }) 1600 1601 t.Run("should cache image results", func(t *testing.T) { 1602 th := setup() 1603 defer th.TearDown() 1604 1605 requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name() 1606 timestamp := int64(1547510400000) 1607 1608 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1609 require.False(t, ok, "data should not exist in in-memory cache") 1610 1611 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1612 require.False(t, ok, "data should not exist in database") 1613 1614 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1615 1616 assert.Nil(t, og) 1617 assert.NotNil(t, img) 1618 assert.Nil(t, err) 1619 1620 _, fromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1621 assert.True(t, ok) 1622 assert.Exactly(t, img, fromCache) 1623 1624 _, fromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1625 assert.True(t, ok) 1626 assert.Exactly(t, img, fromDatabase) 1627 }) 1628 1629 t.Run("should cache general errors", func(t *testing.T) { 1630 th := setup() 1631 defer th.TearDown() 1632 1633 requestURL := server.URL + "/error" 1634 timestamp := int64(1547510400000) 1635 1636 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1637 require.False(t, ok, "data should not exist in in-memory cache") 1638 1639 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1640 require.False(t, ok, "data should not exist in database") 1641 1642 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1643 1644 assert.Nil(t, og) 1645 assert.Nil(t, img) 1646 assert.Nil(t, err) 1647 1648 ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1649 assert.True(t, ok) 1650 assert.Nil(t, ogFromCache) 1651 assert.Nil(t, imgFromCache) 1652 1653 ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1654 assert.True(t, ok) 1655 assert.Nil(t, ogFromDatabase) 1656 assert.Nil(t, imageFromDatabase) 1657 }) 1658 1659 t.Run("should cache invalid URL errors", func(t *testing.T) { 1660 th := setup() 1661 defer th.TearDown() 1662 1663 requestURL := "http://notarealdomainthatactuallyexists.ca/?name=" + t.Name() 1664 timestamp := int64(1547510400000) 1665 1666 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1667 require.False(t, ok, "data should not exist in in-memory cache") 1668 1669 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1670 require.False(t, ok, "data should not exist in database") 1671 1672 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1673 1674 assert.Nil(t, og) 1675 assert.Nil(t, img) 1676 assert.IsType(t, &url.Error{}, err) 1677 1678 ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1679 assert.True(t, ok) 1680 assert.Nil(t, ogFromCache) 1681 assert.Nil(t, imgFromCache) 1682 1683 ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1684 assert.True(t, ok) 1685 assert.Nil(t, ogFromDatabase) 1686 assert.Nil(t, imageFromDatabase) 1687 }) 1688 1689 t.Run("should cache timeout errors", func(t *testing.T) { 1690 th := setup() 1691 defer th.TearDown() 1692 1693 th.App.UpdateConfig(func(cfg *model.Config) { 1694 *cfg.ExperimentalSettings.LinkMetadataTimeoutMilliseconds = 100 1695 }) 1696 1697 requestURL := server.URL + "/timeout?name=" + t.Name() 1698 timestamp := int64(1547510400000) 1699 1700 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1701 require.False(t, ok, "data should not exist in in-memory cache") 1702 1703 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1704 require.False(t, ok, "data should not exist in database") 1705 1706 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1707 1708 assert.Nil(t, og) 1709 assert.Nil(t, img) 1710 assert.NotNil(t, err) 1711 assert.Contains(t, err.Error(), "Client.Timeout") 1712 1713 ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1714 assert.True(t, ok) 1715 assert.Nil(t, ogFromCache) 1716 assert.Nil(t, imgFromCache) 1717 1718 ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1719 assert.True(t, ok) 1720 assert.Nil(t, ogFromDatabase) 1721 assert.Nil(t, imageFromDatabase) 1722 }) 1723 1724 t.Run("should cache database results in memory", func(t *testing.T) { 1725 th := setup() 1726 defer th.TearDown() 1727 1728 requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name() 1729 timestamp := int64(1547510400000) 1730 1731 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1732 require.False(t, ok, "data should not exist in in-memory cache") 1733 1734 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1735 require.False(t, ok, "data should not exist in database") 1736 1737 _, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1738 require.Nil(t, err) 1739 1740 _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) 1741 require.True(t, ok, "data should now exist in in-memory cache") 1742 1743 linkCache.Purge() 1744 _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) 1745 require.False(t, ok, "data should no longer exist in in-memory cache") 1746 1747 _, fromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1748 assert.True(t, ok, "data should be be in in-memory cache again") 1749 assert.Exactly(t, img, fromDatabase) 1750 }) 1751 1752 t.Run("should reject non-html, non-image response", func(t *testing.T) { 1753 th := setup() 1754 defer th.TearDown() 1755 1756 requestURL := server.URL + "/json?name=" + t.Name() 1757 timestamp := int64(1547510400000) 1758 1759 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1760 assert.Nil(t, og) 1761 assert.Nil(t, img) 1762 assert.Nil(t, err) 1763 }) 1764 1765 t.Run("should check in-memory cache for new post", func(t *testing.T) { 1766 th := setup() 1767 defer th.TearDown() 1768 1769 requestURL := server.URL + "/error?name=" + t.Name() 1770 timestamp := int64(1547510400000) 1771 1772 cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil) 1773 1774 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true) 1775 assert.NotNil(t, og) 1776 assert.Nil(t, img) 1777 assert.Nil(t, err) 1778 }) 1779 1780 t.Run("should skip database cache for new post", func(t *testing.T) { 1781 th := setup() 1782 defer th.TearDown() 1783 1784 requestURL := server.URL + "/error?name=" + t.Name() 1785 timestamp := int64(1547510400000) 1786 1787 th.App.saveLinkMetadataToDatabase(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil) 1788 1789 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true) 1790 assert.Nil(t, og) 1791 assert.Nil(t, img) 1792 assert.Nil(t, err) 1793 }) 1794 1795 t.Run("should resolve relative URL", func(t *testing.T) { 1796 th := setup() 1797 defer th.TearDown() 1798 1799 // Fake the SiteURL to have the relative URL resolve to the external server 1800 oldSiteURL := *th.App.Config().ServiceSettings.SiteURL 1801 defer th.App.UpdateConfig(func(cfg *model.Config) { 1802 *cfg.ServiceSettings.SiteURL = oldSiteURL 1803 }) 1804 1805 th.App.UpdateConfig(func(cfg *model.Config) { 1806 *cfg.ServiceSettings.SiteURL = server.URL 1807 }) 1808 1809 requestURL := "/image?height=200&width=300&name=" + t.Name() 1810 timestamp := int64(1547510400000) 1811 1812 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1813 assert.Nil(t, og) 1814 assert.NotNil(t, img) 1815 assert.Nil(t, err) 1816 }) 1817 1818 t.Run("should error on local addresses other than the image proxy", func(t *testing.T) { 1819 th := setup() 1820 defer th.TearDown() 1821 1822 // Disable AllowedUntrustedInternalConnections since it's turned on for the previous tests 1823 oldAllowUntrusted := *th.App.Config().ServiceSettings.AllowedUntrustedInternalConnections 1824 oldSiteURL := *th.App.Config().ServiceSettings.SiteURL 1825 defer th.App.UpdateConfig(func(cfg *model.Config) { 1826 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = oldAllowUntrusted 1827 *cfg.ServiceSettings.SiteURL = oldSiteURL 1828 }) 1829 1830 th.App.UpdateConfig(func(cfg *model.Config) { 1831 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "" 1832 *cfg.ServiceSettings.SiteURL = "http://mattermost.example.com" 1833 *cfg.ImageProxySettings.Enable = true 1834 *cfg.ImageProxySettings.ImageProxyType = "local" 1835 }) 1836 1837 requestURL := server.URL + "/image?height=200&width=300&name=" + t.Name() 1838 timestamp := int64(1547510400000) 1839 1840 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1841 assert.Nil(t, og) 1842 assert.Nil(t, img) 1843 assert.NotNil(t, err) 1844 assert.IsType(t, &url.Error{}, err) 1845 assert.Equal(t, httpservice.AddressForbidden, err.(*url.Error).Err) 1846 1847 requestURL = th.App.GetSiteURL() + "/api/v4/image?url=" + url.QueryEscape(requestURL) 1848 1849 // Note that this request still fails while testing because the request made by the image proxy is blocked 1850 og, img, err = th.App.getLinkMetadata(requestURL, timestamp, false) 1851 assert.Nil(t, og) 1852 assert.Nil(t, img) 1853 assert.NotNil(t, err) 1854 assert.IsType(t, imageproxy.Error{}, err) 1855 }) 1856 } 1857 1858 func TestResolveMetadataURL(t *testing.T) { 1859 for _, test := range []struct { 1860 Name string 1861 RequestURL string 1862 SiteURL string 1863 Expected string 1864 }{ 1865 { 1866 Name: "with HTTPS", 1867 RequestURL: "https://example.com/file?param=1", 1868 Expected: "https://example.com/file?param=1", 1869 }, 1870 { 1871 Name: "with HTTP", 1872 RequestURL: "http://example.com/file?param=1", 1873 Expected: "http://example.com/file?param=1", 1874 }, 1875 { 1876 Name: "with FTP", 1877 RequestURL: "ftp://example.com/file?param=1", 1878 Expected: "ftp://example.com/file?param=1", 1879 }, 1880 { 1881 Name: "relative to root", 1882 RequestURL: "/file?param=1", 1883 SiteURL: "https://mattermost.example.com:123", 1884 Expected: "https://mattermost.example.com:123/file?param=1", 1885 }, 1886 { 1887 Name: "relative to root with subpath", 1888 RequestURL: "/file?param=1", 1889 SiteURL: "https://mattermost.example.com:123/subpath", 1890 Expected: "https://mattermost.example.com:123/file?param=1", 1891 }, 1892 } { 1893 t.Run(test.Name, func(t *testing.T) { 1894 assert.Equal(t, resolveMetadataURL(test.RequestURL, test.SiteURL), test.Expected) 1895 }) 1896 } 1897 } 1898 1899 func TestParseLinkMetadata(t *testing.T) { 1900 th := Setup(t).InitBasic() 1901 defer th.TearDown() 1902 1903 imageURL := "http://example.com/test.png" 1904 file, err := testutils.ReadTestFile("test.png") 1905 require.Nil(t, err) 1906 1907 ogURL := "https://example.com/hello" 1908 html := ` 1909 <html> 1910 <head> 1911 <meta property="og:title" content="Hello, World!"> 1912 <meta property="og:type" content="object"> 1913 <meta property="og:url" content="` + ogURL + `"> 1914 </head> 1915 </html>` 1916 1917 makeImageReader := func() io.Reader { 1918 return bytes.NewReader(file) 1919 } 1920 1921 makeOpenGraphReader := func() io.Reader { 1922 return strings.NewReader(html) 1923 } 1924 1925 t.Run("image", func(t *testing.T) { 1926 og, dimensions, err := th.App.parseLinkMetadata(imageURL, makeImageReader(), "image/png") 1927 assert.Nil(t, err) 1928 1929 assert.Nil(t, og) 1930 assert.Equal(t, &model.PostImage{ 1931 Format: "png", 1932 Width: 408, 1933 Height: 336, 1934 }, dimensions) 1935 }) 1936 1937 t.Run("malformed image", func(t *testing.T) { 1938 og, dimensions, err := th.App.parseLinkMetadata(imageURL, makeOpenGraphReader(), "image/png") 1939 assert.NotNil(t, err) 1940 1941 assert.Nil(t, og) 1942 assert.Nil(t, dimensions) 1943 }) 1944 1945 t.Run("opengraph", func(t *testing.T) { 1946 og, dimensions, err := th.App.parseLinkMetadata(ogURL, makeOpenGraphReader(), "text/html; charset=utf-8") 1947 assert.Nil(t, err) 1948 1949 assert.NotNil(t, og) 1950 assert.Equal(t, og.Title, "Hello, World!") 1951 assert.Equal(t, og.Type, "object") 1952 assert.Equal(t, og.URL, ogURL) 1953 assert.Nil(t, dimensions) 1954 }) 1955 1956 t.Run("malformed opengraph", func(t *testing.T) { 1957 og, dimensions, err := th.App.parseLinkMetadata(ogURL, makeImageReader(), "text/html; charset=utf-8") 1958 assert.Nil(t, err) 1959 1960 assert.Nil(t, og) 1961 assert.Nil(t, dimensions) 1962 }) 1963 1964 t.Run("neither", func(t *testing.T) { 1965 og, dimensions, err := th.App.parseLinkMetadata("http://example.com/test.wad", strings.NewReader("garbage"), "application/x-doom") 1966 assert.Nil(t, err) 1967 1968 assert.Nil(t, og) 1969 assert.Nil(t, dimensions) 1970 }) 1971 } 1972 1973 func TestParseImages(t *testing.T) { 1974 for name, testCase := range map[string]struct { 1975 FileName string 1976 Expected *model.PostImage 1977 ExpectError bool 1978 }{ 1979 "png": { 1980 FileName: "test.png", 1981 Expected: &model.PostImage{ 1982 Width: 408, 1983 Height: 336, 1984 Format: "png", 1985 }, 1986 }, 1987 "animated gif": { 1988 FileName: "testgif.gif", 1989 Expected: &model.PostImage{ 1990 Width: 118, 1991 Height: 118, 1992 Format: "gif", 1993 FrameCount: 4, 1994 }, 1995 }, 1996 "tiff": { 1997 FileName: "test.tiff", 1998 Expected: &model.PostImage{ 1999 Width: 701, 2000 Height: 701, 2001 Format: "tiff", 2002 }, 2003 }, 2004 "not an image": { 2005 FileName: "README.md", 2006 ExpectError: true, 2007 }, 2008 } { 2009 t.Run(name, func(t *testing.T) { 2010 file, err := testutils.ReadTestFile(testCase.FileName) 2011 require.Nil(t, err) 2012 2013 result, err := parseImages(bytes.NewReader(file)) 2014 if testCase.ExpectError { 2015 assert.NotNil(t, err) 2016 } else { 2017 assert.Nil(t, err) 2018 assert.Equal(t, testCase.Expected, result) 2019 } 2020 }) 2021 } 2022 }