github.com/crspeller/mattermost-server@v0.0.0-20190328001957-a200beb3d111/app/post_metadata_test.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "bytes" 8 "fmt" 9 "image" 10 "image/png" 11 "io" 12 "net/http" 13 "net/http/httptest" 14 "net/url" 15 "strconv" 16 "strings" 17 "testing" 18 "time" 19 20 "github.com/dyatlov/go-opengraph/opengraph" 21 "github.com/crspeller/mattermost-server/model" 22 "github.com/crspeller/mattermost-server/services/httpservice" 23 "github.com/crspeller/mattermost-server/services/imageproxy" 24 "github.com/crspeller/mattermost-server/utils/testutils" 25 "github.com/stretchr/testify/assert" 26 "github.com/stretchr/testify/require" 27 ) 28 29 func TestPreparePostListForClient(t *testing.T) { 30 // Most of this logic is covered by TestPreparePostForClient, so this just tests handling of multiple posts 31 32 th := Setup(t).InitBasic() 33 defer th.TearDown() 34 35 th.App.UpdateConfig(func(cfg *model.Config) { 36 *cfg.ExperimentalSettings.DisablePostMetadata = false 37 }) 38 39 postList := model.NewPostList() 40 for i := 0; i < 5; i++ { 41 postList.AddPost(&model.Post{}) 42 } 43 44 clientPostList := th.App.PreparePostListForClient(postList) 45 46 t.Run("doesn't mutate provided post list", func(t *testing.T) { 47 assert.NotEqual(t, clientPostList, postList, "should've returned a new post list") 48 assert.NotEqual(t, clientPostList.Posts, postList.Posts, "should've returned a new PostList.Posts") 49 assert.Equal(t, clientPostList.Order, postList.Order, "should've returned the existing PostList.Order") 50 51 for id, originalPost := range postList.Posts { 52 assert.NotEqual(t, clientPostList.Posts[id], originalPost, "should've returned new post objects") 53 assert.Equal(t, clientPostList.Posts[id].Id, originalPost.Id, "should've returned the same posts") 54 } 55 }) 56 57 t.Run("adds metadata to each post", func(t *testing.T) { 58 for _, clientPost := range clientPostList.Posts { 59 assert.NotNil(t, clientPost.Metadata, "should've populated metadata for each post") 60 } 61 }) 62 } 63 64 func TestPreparePostForClient(t *testing.T) { 65 setup := func() *TestHelper { 66 th := Setup(t).InitBasic() 67 68 th.App.UpdateConfig(func(cfg *model.Config) { 69 *cfg.ImageProxySettings.Enable = false 70 *cfg.ExperimentalSettings.DisablePostMetadata = false 71 }) 72 73 return th 74 } 75 76 t.Run("no metadata needed", func(t *testing.T) { 77 th := setup() 78 defer th.TearDown() 79 80 message := model.NewId() 81 post := &model.Post{ 82 Message: message, 83 } 84 85 clientPost := th.App.PreparePostForClient(post, false) 86 87 t.Run("doesn't mutate provided post", func(t *testing.T) { 88 assert.NotEqual(t, clientPost, post, "should've returned a new post") 89 90 assert.Equal(t, message, post.Message, "shouldn't have mutated post.Message") 91 assert.Equal(t, (*model.PostMetadata)(nil), post.Metadata, "shouldn't have mutated post.Metadata") 92 }) 93 94 t.Run("populates all fields", func(t *testing.T) { 95 assert.Equal(t, message, clientPost.Message, "shouldn't have changed Message") 96 assert.NotEqual(t, nil, clientPost.Metadata, "should've populated Metadata") 97 assert.Len(t, clientPost.Metadata.Embeds, 0, "should've populated Embeds") 98 assert.Len(t, clientPost.Metadata.Reactions, 0, "should've populated Reactions") 99 assert.Len(t, clientPost.Metadata.Files, 0, "should've populated Files") 100 assert.Len(t, clientPost.Metadata.Emojis, 0, "should've populated Emojis") 101 assert.Len(t, clientPost.Metadata.Images, 0, "should've populated Images") 102 }) 103 }) 104 105 t.Run("metadata already set", func(t *testing.T) { 106 th := setup() 107 defer th.TearDown() 108 109 post := th.CreatePost(th.BasicChannel) 110 111 clientPost := th.App.PreparePostForClient(post, false) 112 113 assert.False(t, clientPost == post, "should've returned a new post") 114 assert.Equal(t, clientPost, post, "shouldn't have changed any metadata") 115 }) 116 117 t.Run("reactions", func(t *testing.T) { 118 th := setup() 119 defer th.TearDown() 120 121 post := th.CreatePost(th.BasicChannel) 122 reaction1 := th.AddReactionToPost(post, th.BasicUser, "smile") 123 reaction2 := th.AddReactionToPost(post, th.BasicUser2, "smile") 124 reaction3 := th.AddReactionToPost(post, th.BasicUser2, "ice_cream") 125 post.HasReactions = true 126 127 clientPost := th.App.PreparePostForClient(post, false) 128 129 assert.Len(t, clientPost.Metadata.Reactions, 3, "should've populated Reactions") 130 assert.Equal(t, reaction1, clientPost.Metadata.Reactions[0], "first reaction is incorrect") 131 assert.Equal(t, reaction2, clientPost.Metadata.Reactions[1], "second reaction is incorrect") 132 assert.Equal(t, reaction3, clientPost.Metadata.Reactions[2], "third reaction is incorrect") 133 }) 134 135 t.Run("files", func(t *testing.T) { 136 th := setup() 137 defer th.TearDown() 138 139 fileInfo, err := th.App.DoUploadFile(time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test")) 140 require.Nil(t, err) 141 142 post, err := th.App.CreatePost(&model.Post{ 143 UserId: th.BasicUser.Id, 144 ChannelId: th.BasicChannel.Id, 145 FileIds: []string{fileInfo.Id}, 146 }, th.BasicChannel, false) 147 require.Nil(t, err) 148 149 fileInfo.PostId = post.Id 150 151 clientPost := th.App.PreparePostForClient(post, false) 152 153 assert.Equal(t, []*model.FileInfo{fileInfo}, clientPost.Metadata.Files, "should've populated Files") 154 }) 155 156 t.Run("emojis without custom emojis enabled", func(t *testing.T) { 157 th := setup() 158 defer th.TearDown() 159 160 th.App.UpdateConfig(func(cfg *model.Config) { 161 *cfg.ServiceSettings.EnableCustomEmoji = false 162 }) 163 164 emoji := th.CreateEmoji() 165 166 post, err := th.App.CreatePost(&model.Post{ 167 UserId: th.BasicUser.Id, 168 ChannelId: th.BasicChannel.Id, 169 Message: ":" + emoji.Name + ": :taco:", 170 Props: map[string]interface{}{ 171 "attachments": []*model.SlackAttachment{ 172 { 173 Text: ":" + emoji.Name + ":", 174 }, 175 }, 176 }, 177 }, th.BasicChannel, false) 178 require.Nil(t, err) 179 180 th.AddReactionToPost(post, th.BasicUser, "smile") 181 th.AddReactionToPost(post, th.BasicUser, "angry") 182 th.AddReactionToPost(post, th.BasicUser2, "angry") 183 post.HasReactions = true 184 185 clientPost := th.App.PreparePostForClient(post, false) 186 187 t.Run("populates emojis", func(t *testing.T) { 188 assert.ElementsMatch(t, []*model.Emoji{}, clientPost.Metadata.Emojis, "should've populated empty Emojis") 189 }) 190 191 t.Run("populates reaction counts", func(t *testing.T) { 192 reactions := clientPost.Metadata.Reactions 193 assert.Len(t, reactions, 3, "should've populated Reactions") 194 }) 195 }) 196 197 t.Run("emojis with custom emojis enabled", func(t *testing.T) { 198 th := setup() 199 defer th.TearDown() 200 201 th.App.UpdateConfig(func(cfg *model.Config) { 202 *cfg.ServiceSettings.EnableCustomEmoji = true 203 }) 204 205 emoji1 := th.CreateEmoji() 206 emoji2 := th.CreateEmoji() 207 emoji3 := th.CreateEmoji() 208 emoji4 := th.CreateEmoji() 209 210 post, err := th.App.CreatePost(&model.Post{ 211 UserId: th.BasicUser.Id, 212 ChannelId: th.BasicChannel.Id, 213 Message: ":" + emoji3.Name + ": :taco:", 214 Props: map[string]interface{}{ 215 "attachments": []*model.SlackAttachment{ 216 { 217 Text: ":" + emoji4.Name + ":", 218 }, 219 }, 220 }, 221 }, th.BasicChannel, false) 222 require.Nil(t, err) 223 224 th.AddReactionToPost(post, th.BasicUser, emoji1.Name) 225 th.AddReactionToPost(post, th.BasicUser, emoji2.Name) 226 th.AddReactionToPost(post, th.BasicUser2, emoji2.Name) 227 th.AddReactionToPost(post, th.BasicUser2, "angry") 228 post.HasReactions = true 229 230 clientPost := th.App.PreparePostForClient(post, false) 231 232 t.Run("pupulates emojis", func(t *testing.T) { 233 assert.ElementsMatch(t, []*model.Emoji{emoji1, emoji2, emoji3, emoji4}, clientPost.Metadata.Emojis, "should've populated post.Emojis") 234 }) 235 236 t.Run("populates reaction counts", func(t *testing.T) { 237 reactions := clientPost.Metadata.Reactions 238 assert.Len(t, reactions, 4, "should've populated Reactions") 239 }) 240 }) 241 242 t.Run("markdown image dimensions", func(t *testing.T) { 243 th := setup() 244 defer th.TearDown() 245 246 post, err := th.App.CreatePost(&model.Post{ 247 UserId: th.BasicUser.Id, 248 ChannelId: th.BasicChannel.Id, 249 Message: "This is ![our logo](https://github.com/hmhealey/test-files/raw/master/logoVertical.png) and ![our icon](https://github.com/hmhealey/test-files/raw/master/icon.png)", 250 }, th.BasicChannel, false) 251 require.Nil(t, err) 252 253 clientPost := th.App.PreparePostForClient(post, false) 254 255 t.Run("populates image dimensions", func(t *testing.T) { 256 imageDimensions := clientPost.Metadata.Images 257 require.Len(t, imageDimensions, 2) 258 assert.Equal(t, &model.PostImage{ 259 Width: 1068, 260 Height: 552, 261 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/logoVertical.png"]) 262 assert.Equal(t, &model.PostImage{ 263 Width: 501, 264 Height: 501, 265 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/icon.png"]) 266 }) 267 }) 268 269 t.Run("proxy linked images", func(t *testing.T) { 270 th := setup() 271 defer th.TearDown() 272 273 testProxyLinkedImage(t, th, false) 274 }) 275 276 t.Run("proxy opengraph images", func(t *testing.T) { 277 th := setup() 278 defer th.TearDown() 279 280 testProxyOpenGraphImage(t, th, false) 281 }) 282 283 t.Run("image embed", func(t *testing.T) { 284 th := setup() 285 defer th.TearDown() 286 287 post, err := th.App.CreatePost(&model.Post{ 288 UserId: th.BasicUser.Id, 289 ChannelId: th.BasicChannel.Id, 290 Message: `This is our logo: https://github.com/hmhealey/test-files/raw/master/logoVertical.png 291 And this is our icon: https://github.com/hmhealey/test-files/raw/master/icon.png`, 292 }, th.BasicChannel, false) 293 require.Nil(t, err) 294 295 clientPost := th.App.PreparePostForClient(post, false) 296 297 // Reminder that only the first link gets an embed and dimensions 298 299 t.Run("populates embeds", func(t *testing.T) { 300 assert.ElementsMatch(t, []*model.PostEmbed{ 301 { 302 Type: model.POST_EMBED_IMAGE, 303 URL: "https://github.com/hmhealey/test-files/raw/master/logoVertical.png", 304 }, 305 }, clientPost.Metadata.Embeds) 306 }) 307 308 t.Run("populates image dimensions", func(t *testing.T) { 309 imageDimensions := clientPost.Metadata.Images 310 require.Len(t, imageDimensions, 1) 311 assert.Equal(t, &model.PostImage{ 312 Width: 1068, 313 Height: 552, 314 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/logoVertical.png"]) 315 }) 316 }) 317 318 t.Run("opengraph embed", func(t *testing.T) { 319 th := setup() 320 defer th.TearDown() 321 322 post, err := th.App.CreatePost(&model.Post{ 323 UserId: th.BasicUser.Id, 324 ChannelId: th.BasicChannel.Id, 325 Message: `This is our web page: https://github.com/hmhealey/test-files`, 326 }, th.BasicChannel, false) 327 require.Nil(t, err) 328 329 clientPost := th.App.PreparePostForClient(post, false) 330 331 t.Run("populates embeds", func(t *testing.T) { 332 assert.ElementsMatch(t, []*model.PostEmbed{ 333 { 334 Type: model.POST_EMBED_OPENGRAPH, 335 URL: "https://github.com/hmhealey/test-files", 336 Data: &opengraph.OpenGraph{ 337 Description: "Contribute to hmhealey/test-files development by creating an account on GitHub.", 338 SiteName: "GitHub", 339 Title: "hmhealey/test-files", 340 Type: "object", 341 URL: "https://github.com/hmhealey/test-files", 342 Images: []*opengraph.Image{ 343 { 344 URL: "https://avatars1.githubusercontent.com/u/3277310?s=400&v=4", 345 }, 346 }, 347 }, 348 }, 349 }, clientPost.Metadata.Embeds) 350 }) 351 352 t.Run("populates image dimensions", func(t *testing.T) { 353 imageDimensions := clientPost.Metadata.Images 354 require.Len(t, imageDimensions, 1) 355 assert.Equal(t, &model.PostImage{ 356 Width: 420, 357 Height: 420, 358 }, imageDimensions["https://avatars1.githubusercontent.com/u/3277310?s=400&v=4"]) 359 }) 360 }) 361 362 t.Run("message attachment embed", func(t *testing.T) { 363 th := setup() 364 defer th.TearDown() 365 366 post, err := th.App.CreatePost(&model.Post{ 367 UserId: th.BasicUser.Id, 368 ChannelId: th.BasicChannel.Id, 369 Props: map[string]interface{}{ 370 "attachments": []interface{}{ 371 map[string]interface{}{ 372 "text": "![icon](https://github.com/hmhealey/test-files/raw/master/icon.png)", 373 }, 374 }, 375 }, 376 }, th.BasicChannel, false) 377 require.Nil(t, err) 378 379 clientPost := th.App.PreparePostForClient(post, false) 380 381 t.Run("populates embeds", func(t *testing.T) { 382 assert.ElementsMatch(t, []*model.PostEmbed{ 383 { 384 Type: model.POST_EMBED_MESSAGE_ATTACHMENT, 385 }, 386 }, clientPost.Metadata.Embeds) 387 }) 388 389 t.Run("populates image dimensions", func(t *testing.T) { 390 imageDimensions := clientPost.Metadata.Images 391 require.Len(t, imageDimensions, 1) 392 assert.Equal(t, &model.PostImage{ 393 Width: 501, 394 Height: 501, 395 }, imageDimensions["https://github.com/hmhealey/test-files/raw/master/icon.png"]) 396 }) 397 }) 398 399 t.Run("when disabled", func(t *testing.T) { 400 th := setup() 401 defer th.TearDown() 402 403 th.App.UpdateConfig(func(cfg *model.Config) { 404 *cfg.ExperimentalSettings.DisablePostMetadata = true 405 }) 406 407 post := th.CreatePost(th.BasicChannel) 408 post = th.App.PreparePostForClient(post, false) 409 410 assert.Nil(t, post.Metadata) 411 412 b := post.ToJson() 413 414 assert.NotContains(t, string(b), "metadata", "json shouldn't include a metadata field, not even a falsey one") 415 }) 416 } 417 418 func TestPreparePostForClientWithImageProxy(t *testing.T) { 419 setup := func() *TestHelper { 420 th := Setup(t).InitBasic() 421 422 th.App.UpdateConfig(func(cfg *model.Config) { 423 *cfg.ServiceSettings.SiteURL = "http://mymattermost.com" 424 *cfg.ImageProxySettings.Enable = true 425 *cfg.ImageProxySettings.ImageProxyType = "atmos/camo" 426 *cfg.ImageProxySettings.RemoteImageProxyURL = "https://127.0.0.1" 427 *cfg.ImageProxySettings.RemoteImageProxyOptions = "foo" 428 *cfg.ExperimentalSettings.DisablePostMetadata = false 429 }) 430 431 return th 432 } 433 434 t.Run("proxy linked images", func(t *testing.T) { 435 th := setup() 436 defer th.TearDown() 437 438 testProxyLinkedImage(t, th, true) 439 }) 440 441 t.Run("proxy opengraph images", func(t *testing.T) { 442 th := setup() 443 defer th.TearDown() 444 445 testProxyOpenGraphImage(t, th, true) 446 }) 447 } 448 449 func testProxyLinkedImage(t *testing.T, th *TestHelper, shouldProxy bool) { 450 postTemplate := "![foo](%v)" 451 imageURL := "http://mydomain.com/myimage" 452 proxiedImageURL := "https://127.0.0.1/f8dace906d23689e8d5b12c3cefbedbf7b9b72f5/687474703a2f2f6d79646f6d61696e2e636f6d2f6d79696d616765" 453 454 post := &model.Post{ 455 UserId: th.BasicUser.Id, 456 ChannelId: th.BasicChannel.Id, 457 Message: fmt.Sprintf(postTemplate, imageURL), 458 } 459 460 clientPost := th.App.PreparePostForClient(post, false) 461 462 if shouldProxy { 463 assert.Equal(t, fmt.Sprintf(postTemplate, imageURL), post.Message, "should not have mutated original post") 464 assert.Equal(t, fmt.Sprintf(postTemplate, proxiedImageURL), clientPost.Message, "should've replaced linked image URLs") 465 } else { 466 assert.Equal(t, fmt.Sprintf(postTemplate, imageURL), clientPost.Message, "shouldn't have replaced linked image URLs") 467 } 468 } 469 470 func testProxyOpenGraphImage(t *testing.T, th *TestHelper, shouldProxy bool) { 471 post, err := th.App.CreatePost(&model.Post{ 472 UserId: th.BasicUser.Id, 473 ChannelId: th.BasicChannel.Id, 474 Message: `This is our web page: https://github.com/hmhealey/test-files`, 475 }, th.BasicChannel, false) 476 require.Nil(t, err) 477 478 embeds := th.App.PreparePostForClient(post, false).Metadata.Embeds 479 require.Len(t, embeds, 1, "should have one embed") 480 481 embed := embeds[0] 482 assert.Equal(t, model.POST_EMBED_OPENGRAPH, embed.Type, "embed type should be OpenGraph") 483 assert.Equal(t, "https://github.com/hmhealey/test-files", embed.URL, "embed URL should be correct") 484 485 og, ok := embed.Data.(*opengraph.OpenGraph) 486 assert.Equal(t, true, ok, "data should be non-nil OpenGraph data") 487 assert.Equal(t, "GitHub", og.SiteName, "OpenGraph data should be correctly populated") 488 489 require.Len(t, og.Images, 1, "OpenGraph data should have one image") 490 491 image := og.Images[0] 492 if shouldProxy { 493 assert.Equal(t, "", image.URL, "image URL should not be set with proxy") 494 assert.Equal(t, "https://127.0.0.1/b2ef6ef4890a0107aa80ba33b3011fd51f668303/68747470733a2f2f61766174617273312e67697468756275736572636f6e74656e742e636f6d2f752f333237373331303f733d34303026763d34", image.SecureURL, "secure image URL should be sent through proxy") 495 } else { 496 assert.Equal(t, "https://avatars1.githubusercontent.com/u/3277310?s=400&v=4", image.URL, "image URL should be set") 497 assert.Equal(t, "", image.SecureURL, "secure image URL should not be set") 498 } 499 } 500 501 func TestGetImagesForPost(t *testing.T) { 502 t.Run("with an image link", func(t *testing.T) { 503 th := Setup(t) 504 defer th.TearDown() 505 506 th.App.UpdateConfig(func(cfg *model.Config) { 507 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 508 }) 509 510 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 511 file, err := testutils.ReadTestFile("test.png") 512 require.Nil(t, err) 513 514 w.Header().Set("Content-Type", "image/png") 515 w.Write(file) 516 })) 517 518 post := &model.Post{ 519 Metadata: &model.PostMetadata{}, 520 } 521 imageURL := server.URL + "/image.png" 522 523 images := th.App.getImagesForPost(post, []string{imageURL}, false) 524 525 assert.Equal(t, images, map[string]*model.PostImage{ 526 imageURL: { 527 Width: 408, 528 Height: 336, 529 }, 530 }) 531 }) 532 533 t.Run("with an invalid image link", func(t *testing.T) { 534 th := Setup(t) 535 defer th.TearDown() 536 537 th.App.UpdateConfig(func(cfg *model.Config) { 538 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 539 }) 540 541 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 542 w.WriteHeader(http.StatusInternalServerError) 543 })) 544 545 post := &model.Post{ 546 Metadata: &model.PostMetadata{}, 547 } 548 imageURL := server.URL + "/bad_image.png" 549 550 images := th.App.getImagesForPost(post, []string{imageURL}, false) 551 552 assert.Equal(t, images, map[string]*model.PostImage{}) 553 }) 554 555 t.Run("for an OpenGraph image with dimensions", func(t *testing.T) { 556 th := Setup(t) 557 defer th.TearDown() 558 559 th.App.UpdateConfig(func(cfg *model.Config) { 560 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 561 }) 562 563 ogURL := "https://example.com/index.html" 564 imageURL := "https://example.com/image.png" 565 566 post := &model.Post{ 567 Metadata: &model.PostMetadata{ 568 Embeds: []*model.PostEmbed{ 569 { 570 Type: model.POST_EMBED_OPENGRAPH, 571 URL: ogURL, 572 Data: &opengraph.OpenGraph{ 573 Images: []*opengraph.Image{ 574 { 575 URL: imageURL, 576 Width: 100, 577 Height: 200, 578 }, 579 }, 580 }, 581 }, 582 }, 583 }, 584 } 585 586 images := th.App.getImagesForPost(post, []string{}, false) 587 588 assert.Equal(t, images, map[string]*model.PostImage{ 589 imageURL: { 590 Width: 100, 591 Height: 200, 592 }, 593 }) 594 }) 595 596 t.Run("for an OpenGraph image without dimensions", func(t *testing.T) { 597 th := Setup(t) 598 defer th.TearDown() 599 600 th.App.UpdateConfig(func(cfg *model.Config) { 601 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 602 }) 603 604 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 605 if r.URL.Path == "/image.png" { 606 w.Header().Set("Content-Type", "image/png") 607 608 img := image.NewGray(image.Rect(0, 0, 200, 300)) 609 610 var encoder png.Encoder 611 encoder.Encode(w, img) 612 } else { 613 w.WriteHeader(http.StatusNotFound) 614 } 615 })) 616 defer server.Close() 617 618 ogURL := server.URL + "/index.html" 619 imageURL := server.URL + "/image.png" 620 621 post := &model.Post{ 622 Metadata: &model.PostMetadata{ 623 Embeds: []*model.PostEmbed{ 624 { 625 Type: model.POST_EMBED_OPENGRAPH, 626 URL: ogURL, 627 Data: &opengraph.OpenGraph{ 628 Images: []*opengraph.Image{ 629 { 630 URL: imageURL, 631 }, 632 }, 633 }, 634 }, 635 }, 636 }, 637 } 638 639 images := th.App.getImagesForPost(post, []string{}, false) 640 641 assert.Equal(t, images, map[string]*model.PostImage{ 642 imageURL: { 643 Width: 200, 644 Height: 300, 645 }, 646 }) 647 }) 648 649 t.Run("with an OpenGraph image with a secure_url and dimensions", func(t *testing.T) { 650 th := Setup(t) 651 defer th.TearDown() 652 653 th.App.UpdateConfig(func(cfg *model.Config) { 654 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 655 }) 656 657 ogURL := "https://example.com/index.html" 658 imageURL := "https://example.com/secure_image.png" 659 660 post := &model.Post{ 661 Metadata: &model.PostMetadata{ 662 Embeds: []*model.PostEmbed{ 663 { 664 Type: model.POST_EMBED_OPENGRAPH, 665 URL: ogURL, 666 Data: &opengraph.OpenGraph{ 667 Images: []*opengraph.Image{ 668 { 669 URL: imageURL, 670 Width: 300, 671 Height: 400, 672 }, 673 }, 674 }, 675 }, 676 }, 677 }, 678 } 679 680 images := th.App.getImagesForPost(post, []string{}, false) 681 682 assert.Equal(t, images, map[string]*model.PostImage{ 683 imageURL: { 684 Width: 300, 685 Height: 400, 686 }, 687 }) 688 }) 689 690 t.Run("with an OpenGraph image with a secure_url and no dimensions", func(t *testing.T) { 691 th := Setup(t) 692 defer th.TearDown() 693 694 th.App.UpdateConfig(func(cfg *model.Config) { 695 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 696 }) 697 698 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 699 if r.URL.Path == "/secure_image.png" { 700 w.Header().Set("Content-Type", "image/png") 701 702 img := image.NewGray(image.Rect(0, 0, 400, 500)) 703 704 var encoder png.Encoder 705 encoder.Encode(w, img) 706 } else { 707 w.WriteHeader(http.StatusNotFound) 708 } 709 })) 710 711 ogURL := server.URL + "/index.html" 712 imageURL := server.URL + "/secure_image.png" 713 714 post := &model.Post{ 715 Metadata: &model.PostMetadata{ 716 Embeds: []*model.PostEmbed{ 717 { 718 Type: model.POST_EMBED_OPENGRAPH, 719 URL: ogURL, 720 Data: &opengraph.OpenGraph{ 721 Images: []*opengraph.Image{ 722 { 723 URL: server.URL + "/image.png", 724 SecureURL: imageURL, 725 }, 726 }, 727 }, 728 }, 729 }, 730 }, 731 } 732 733 images := th.App.getImagesForPost(post, []string{}, false) 734 735 assert.Equal(t, images, map[string]*model.PostImage{ 736 imageURL: { 737 Width: 400, 738 Height: 500, 739 }, 740 }) 741 }) 742 } 743 744 func TestGetEmojiNamesForString(t *testing.T) { 745 testCases := []struct { 746 Description string 747 Input string 748 Expected []string 749 }{ 750 { 751 Description: "no emojis", 752 Input: "this is a string", 753 Expected: []string{}, 754 }, 755 { 756 Description: "one emoji", 757 Input: "this is an :emoji1: string", 758 Expected: []string{"emoji1"}, 759 }, 760 { 761 Description: "two emojis", 762 Input: "this is a :emoji3: :emoji2: string", 763 Expected: []string{"emoji3", "emoji2"}, 764 }, 765 { 766 Description: "punctuation around emojis", 767 Input: ":emoji3:/:emoji1: (:emoji2:)", 768 Expected: []string{"emoji3", "emoji1", "emoji2"}, 769 }, 770 { 771 Description: "adjacent emojis", 772 Input: ":emoji3::emoji1:", 773 Expected: []string{"emoji3", "emoji1"}, 774 }, 775 { 776 Description: "duplicate emojis", 777 Input: ":emoji1: :emoji1: :emoji1::emoji2::emoji2: :emoji1:", 778 Expected: []string{"emoji1", "emoji1", "emoji1", "emoji2", "emoji2", "emoji1"}, 779 }, 780 { 781 Description: "fake emojis", 782 Input: "these don't exist :tomato: :potato: :rotato:", 783 Expected: []string{"tomato", "potato", "rotato"}, 784 }, 785 } 786 787 for _, testCase := range testCases { 788 testCase := testCase 789 t.Run(testCase.Description, func(t *testing.T) { 790 emojis := getEmojiNamesForString(testCase.Input) 791 assert.ElementsMatch(t, emojis, testCase.Expected, "received incorrect emoji names") 792 }) 793 } 794 } 795 796 func TestGetEmojiNamesForPost(t *testing.T) { 797 testCases := []struct { 798 Description string 799 Post *model.Post 800 Reactions []*model.Reaction 801 Expected []string 802 }{ 803 { 804 Description: "no emojis", 805 Post: &model.Post{ 806 Message: "this is a post", 807 }, 808 Expected: []string{}, 809 }, 810 { 811 Description: "in post message", 812 Post: &model.Post{ 813 Message: "this is :emoji:", 814 }, 815 Expected: []string{"emoji"}, 816 }, 817 { 818 Description: "in reactions", 819 Post: &model.Post{}, 820 Reactions: []*model.Reaction{ 821 { 822 EmojiName: "emoji1", 823 }, 824 { 825 EmojiName: "emoji2", 826 }, 827 }, 828 Expected: []string{"emoji1", "emoji2"}, 829 }, 830 { 831 Description: "in message attachments", 832 Post: &model.Post{ 833 Message: "this is a post", 834 Props: map[string]interface{}{ 835 "attachments": []*model.SlackAttachment{ 836 { 837 Text: ":emoji1:", 838 Pretext: ":emoji2:", 839 }, 840 { 841 Fields: []*model.SlackAttachmentField{ 842 { 843 Value: ":emoji3:", 844 }, 845 { 846 Value: ":emoji4:", 847 }, 848 }, 849 }, 850 }, 851 }, 852 }, 853 Expected: []string{"emoji1", "emoji2", "emoji3", "emoji4"}, 854 }, 855 { 856 Description: "with duplicates", 857 Post: &model.Post{ 858 Message: "this is :emoji1", 859 Props: map[string]interface{}{ 860 "attachments": []*model.SlackAttachment{ 861 { 862 Text: ":emoji2:", 863 Pretext: ":emoji2:", 864 Fields: []*model.SlackAttachmentField{ 865 { 866 Value: ":emoji3:", 867 }, 868 { 869 Value: ":emoji1:", 870 }, 871 }, 872 }, 873 }, 874 }, 875 }, 876 Expected: []string{"emoji1", "emoji2", "emoji3"}, 877 }, 878 } 879 880 for _, testCase := range testCases { 881 testCase := testCase 882 t.Run(testCase.Description, func(t *testing.T) { 883 emojis := getEmojiNamesForPost(testCase.Post, testCase.Reactions) 884 assert.ElementsMatch(t, emojis, testCase.Expected, "received incorrect emoji names") 885 }) 886 } 887 } 888 889 func TestGetCustomEmojisForPost(t *testing.T) { 890 th := Setup(t).InitBasic() 891 defer th.TearDown() 892 893 th.App.UpdateConfig(func(cfg *model.Config) { 894 *cfg.ServiceSettings.EnableCustomEmoji = true 895 }) 896 897 emojis := []*model.Emoji{ 898 th.CreateEmoji(), 899 th.CreateEmoji(), 900 th.CreateEmoji(), 901 th.CreateEmoji(), 902 th.CreateEmoji(), 903 th.CreateEmoji(), 904 } 905 906 t.Run("from different parts of the post", func(t *testing.T) { 907 reactions := []*model.Reaction{ 908 { 909 UserId: th.BasicUser.Id, 910 EmojiName: emojis[0].Name, 911 }, 912 } 913 914 post := &model.Post{ 915 Message: ":" + emojis[1].Name + ":", 916 Props: map[string]interface{}{ 917 "attachments": []*model.SlackAttachment{ 918 { 919 Pretext: ":" + emojis[2].Name + ":", 920 Text: ":" + emojis[3].Name + ":", 921 Fields: []*model.SlackAttachmentField{ 922 { 923 Value: ":" + emojis[4].Name + ":", 924 }, 925 { 926 Value: ":" + emojis[5].Name + ":", 927 }, 928 }, 929 }, 930 }, 931 }, 932 } 933 934 emojisForPost, err := th.App.getCustomEmojisForPost(post, reactions) 935 assert.Nil(t, err, "failed to get emojis for post") 936 assert.ElementsMatch(t, emojisForPost, emojis, "received incorrect emojis") 937 }) 938 939 t.Run("with emojis that don't exist", func(t *testing.T) { 940 post := &model.Post{ 941 Message: ":secret: :" + emojis[0].Name + ":", 942 Props: map[string]interface{}{ 943 "attachments": []*model.SlackAttachment{ 944 { 945 Text: ":imaginary:", 946 }, 947 }, 948 }, 949 } 950 951 emojisForPost, err := th.App.getCustomEmojisForPost(post, nil) 952 assert.Nil(t, err, "failed to get emojis for post") 953 assert.ElementsMatch(t, emojisForPost, []*model.Emoji{emojis[0]}, "received incorrect emojis") 954 }) 955 956 t.Run("with no emojis", func(t *testing.T) { 957 post := &model.Post{ 958 Message: "this post is boring", 959 Props: map[string]interface{}{}, 960 } 961 962 emojisForPost, err := th.App.getCustomEmojisForPost(post, nil) 963 assert.Nil(t, err, "failed to get emojis for post") 964 assert.ElementsMatch(t, emojisForPost, []*model.Emoji{}, "should have received no emojis") 965 }) 966 } 967 968 func TestGetFirstLinkAndImages(t *testing.T) { 969 for name, testCase := range map[string]struct { 970 Input string 971 ExpectedFirstLink string 972 ExpectedImages []string 973 }{ 974 "no links or images": { 975 Input: "this is a string", 976 ExpectedFirstLink: "", 977 ExpectedImages: []string{}, 978 }, 979 "http link": { 980 Input: "this is a http://example.com", 981 ExpectedFirstLink: "http://example.com", 982 ExpectedImages: []string{}, 983 }, 984 "www link": { 985 Input: "this is a www.example.com", 986 ExpectedFirstLink: "http://www.example.com", 987 ExpectedImages: []string{}, 988 }, 989 "image": { 990 Input: "this is a ![our logo](http://example.com/logo)", 991 ExpectedFirstLink: "", 992 ExpectedImages: []string{"http://example.com/logo"}, 993 }, 994 "multiple images": { 995 Input: "this is a ![our logo](http://example.com/logo) and ![their logo](http://example.com/logo2) and ![my logo](http://example.com/logo3)", 996 ExpectedFirstLink: "", 997 ExpectedImages: []string{"http://example.com/logo", "http://example.com/logo2", "http://example.com/logo3"}, 998 }, 999 "multiple images with duplicate": { 1000 Input: "this is a ![our logo](http://example.com/logo) and ![their logo](http://example.com/logo2) and ![my logo which is their logo](http://example.com/logo2)", 1001 ExpectedFirstLink: "", 1002 ExpectedImages: []string{"http://example.com/logo", "http://example.com/logo2", "http://example.com/logo2"}, 1003 }, 1004 "reference image": { 1005 Input: `this is a ![our logo][logo] 1006 1007 [logo]: http://example.com/logo`, 1008 ExpectedFirstLink: "", 1009 ExpectedImages: []string{"http://example.com/logo"}, 1010 }, 1011 "image and link": { 1012 Input: "this is a https://example.com and ![our logo](https://example.com/logo)", 1013 ExpectedFirstLink: "https://example.com", 1014 ExpectedImages: []string{"https://example.com/logo"}, 1015 }, 1016 "markdown links (not returned)": { 1017 Input: `this is a [our page](http://example.com) and [another page][] 1018 1019 [another page]: http://www.exaple.com/another_page`, 1020 ExpectedFirstLink: "", 1021 ExpectedImages: []string{}, 1022 }, 1023 } { 1024 t.Run(name, func(t *testing.T) { 1025 firstLink, images := getFirstLinkAndImages(testCase.Input) 1026 1027 assert.Equal(t, firstLink, testCase.ExpectedFirstLink) 1028 assert.Equal(t, images, testCase.ExpectedImages) 1029 }) 1030 } 1031 } 1032 1033 func TestGetImagesInMessageAttachments(t *testing.T) { 1034 for _, test := range []struct { 1035 Name string 1036 Post *model.Post 1037 Expected []string 1038 }{ 1039 { 1040 Name: "no attachments", 1041 Post: &model.Post{}, 1042 Expected: []string{}, 1043 }, 1044 { 1045 Name: "empty attachments", 1046 Post: &model.Post{ 1047 Props: map[string]interface{}{ 1048 "attachments": []*model.SlackAttachment{}, 1049 }, 1050 }, 1051 Expected: []string{}, 1052 }, 1053 { 1054 Name: "attachment with no fields that can contain images", 1055 Post: &model.Post{ 1056 Props: map[string]interface{}{ 1057 "attachments": []*model.SlackAttachment{ 1058 { 1059 Title: "This is the title", 1060 }, 1061 }, 1062 }, 1063 }, 1064 Expected: []string{}, 1065 }, 1066 { 1067 Name: "images in text", 1068 Post: &model.Post{ 1069 Props: map[string]interface{}{ 1070 "attachments": []*model.SlackAttachment{ 1071 { 1072 Text: "![logo](https://example.com/logo) and ![icon](https://example.com/icon)", 1073 }, 1074 }, 1075 }, 1076 }, 1077 Expected: []string{"https://example.com/logo", "https://example.com/icon"}, 1078 }, 1079 { 1080 Name: "images in pretext", 1081 Post: &model.Post{ 1082 Props: map[string]interface{}{ 1083 "attachments": []*model.SlackAttachment{ 1084 { 1085 Pretext: "![logo](https://example.com/logo1) and ![icon](https://example.com/icon1)", 1086 }, 1087 }, 1088 }, 1089 }, 1090 Expected: []string{"https://example.com/logo1", "https://example.com/icon1"}, 1091 }, 1092 { 1093 Name: "images in fields", 1094 Post: &model.Post{ 1095 Props: map[string]interface{}{ 1096 "attachments": []*model.SlackAttachment{ 1097 { 1098 Fields: []*model.SlackAttachmentField{ 1099 { 1100 Value: "![logo](https://example.com/logo2) and ![icon](https://example.com/icon2)", 1101 }, 1102 }, 1103 }, 1104 }, 1105 }, 1106 }, 1107 Expected: []string{"https://example.com/logo2", "https://example.com/icon2"}, 1108 }, 1109 { 1110 Name: "image in author_icon", 1111 Post: &model.Post{ 1112 Props: map[string]interface{}{ 1113 "attachments": []*model.SlackAttachment{ 1114 { 1115 AuthorIcon: "https://example.com/icon2", 1116 }, 1117 }, 1118 }, 1119 }, 1120 Expected: []string{"https://example.com/icon2"}, 1121 }, 1122 { 1123 Name: "image in image_url", 1124 Post: &model.Post{ 1125 Props: map[string]interface{}{ 1126 "attachments": []*model.SlackAttachment{ 1127 { 1128 ImageURL: "https://example.com/image", 1129 }, 1130 }, 1131 }, 1132 }, 1133 Expected: []string{"https://example.com/image"}, 1134 }, 1135 { 1136 Name: "image in thumb_url", 1137 Post: &model.Post{ 1138 Props: map[string]interface{}{ 1139 "attachments": []*model.SlackAttachment{ 1140 { 1141 ThumbURL: "https://example.com/image", 1142 }, 1143 }, 1144 }, 1145 }, 1146 Expected: []string{"https://example.com/image"}, 1147 }, 1148 { 1149 Name: "image in footer_icon", 1150 Post: &model.Post{ 1151 Props: map[string]interface{}{ 1152 "attachments": []*model.SlackAttachment{ 1153 { 1154 FooterIcon: "https://example.com/image", 1155 }, 1156 }, 1157 }, 1158 }, 1159 Expected: []string{"https://example.com/image"}, 1160 }, 1161 { 1162 Name: "images in multiple fields", 1163 Post: &model.Post{ 1164 Props: map[string]interface{}{ 1165 "attachments": []*model.SlackAttachment{ 1166 { 1167 Fields: []*model.SlackAttachmentField{ 1168 { 1169 Value: "![logo](https://example.com/logo)", 1170 }, 1171 { 1172 Value: "![icon](https://example.com/icon)", 1173 }, 1174 }, 1175 }, 1176 }, 1177 }, 1178 }, 1179 Expected: []string{"https://example.com/logo", "https://example.com/icon"}, 1180 }, 1181 { 1182 Name: "non-string field", 1183 Post: &model.Post{ 1184 Props: map[string]interface{}{ 1185 "attachments": []*model.SlackAttachment{ 1186 { 1187 Fields: []*model.SlackAttachmentField{ 1188 { 1189 Value: 77, 1190 }, 1191 }, 1192 }, 1193 }, 1194 }, 1195 }, 1196 Expected: []string{}, 1197 }, 1198 { 1199 Name: "images in multiple locations", 1200 Post: &model.Post{ 1201 Props: map[string]interface{}{ 1202 "attachments": []*model.SlackAttachment{ 1203 { 1204 Text: "![text](https://example.com/text)", 1205 Pretext: "![pretext](https://example.com/pretext)", 1206 Fields: []*model.SlackAttachmentField{ 1207 { 1208 Value: "![field1](https://example.com/field1)", 1209 }, 1210 { 1211 Value: "![field2](https://example.com/field2)", 1212 }, 1213 }, 1214 }, 1215 }, 1216 }, 1217 }, 1218 Expected: []string{"https://example.com/text", "https://example.com/pretext", "https://example.com/field1", "https://example.com/field2"}, 1219 }, 1220 { 1221 Name: "multiple attachments", 1222 Post: &model.Post{ 1223 Props: map[string]interface{}{ 1224 "attachments": []*model.SlackAttachment{ 1225 { 1226 Text: "![logo](https://example.com/logo)", 1227 }, 1228 { 1229 Text: "![icon](https://example.com/icon)", 1230 }, 1231 }, 1232 }, 1233 }, 1234 Expected: []string{"https://example.com/logo", "https://example.com/icon"}, 1235 }, 1236 } { 1237 t.Run(test.Name, func(t *testing.T) { 1238 images := getImagesInMessageAttachments(test.Post) 1239 1240 assert.ElementsMatch(t, images, test.Expected) 1241 }) 1242 } 1243 } 1244 1245 func TestGetLinkMetadata(t *testing.T) { 1246 setup := func() *TestHelper { 1247 th := Setup(t).InitBasic() 1248 1249 th.App.UpdateConfig(func(cfg *model.Config) { 1250 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "127.0.0.1" 1251 }) 1252 1253 linkCache.Purge() 1254 1255 return th 1256 } 1257 1258 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 1259 params := r.URL.Query() 1260 1261 if strings.HasPrefix(r.URL.Path, "/image") { 1262 height, _ := strconv.ParseInt(params["height"][0], 10, 0) 1263 width, _ := strconv.ParseInt(params["width"][0], 10, 0) 1264 1265 img := image.NewGray(image.Rect(0, 0, int(width), int(height))) 1266 1267 var encoder png.Encoder 1268 1269 encoder.Encode(w, img) 1270 } else if strings.HasPrefix(r.URL.Path, "/opengraph") { 1271 w.Header().Set("Content-Type", "text/html") 1272 1273 w.Write([]byte(` 1274 <html prefix="og:http://ogp.me/ns#"> 1275 <head> 1276 <meta property="og:title" content="` + params["title"][0] + `" /> 1277 </head> 1278 <body> 1279 </body> 1280 </html>`)) 1281 } else if strings.HasPrefix(r.URL.Path, "/json") { 1282 w.Header().Set("Content-Type", "application/json") 1283 1284 w.Write([]byte("true")) 1285 } else if strings.HasPrefix(r.URL.Path, "/timeout") { 1286 w.Header().Set("Content-Type", "text/html") 1287 1288 w.Write([]byte("<html>")) 1289 select { 1290 case <-time.After(60 * time.Second): 1291 case <-r.Context().Done(): 1292 } 1293 w.Write([]byte("</html>")) 1294 } else { 1295 w.WriteHeader(http.StatusInternalServerError) 1296 } 1297 })) 1298 defer server.Close() 1299 1300 t.Run("in-memory cache", func(t *testing.T) { 1301 th := setup() 1302 defer th.TearDown() 1303 1304 requestURL := server.URL + "/cached" 1305 timestamp := int64(1547510400000) 1306 title := "from cache" 1307 1308 cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil) 1309 1310 t.Run("should use cache if cached entry exists", func(t *testing.T) { 1311 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1312 require.True(t, ok, "data should already exist in in-memory cache") 1313 1314 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1315 require.False(t, ok, "data should not exist in database") 1316 1317 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1318 1319 require.NotNil(t, og) 1320 assert.Nil(t, img) 1321 assert.Nil(t, err) 1322 assert.Equal(t, title, og.Title) 1323 }) 1324 1325 t.Run("should use cache if cached entry exists near time", func(t *testing.T) { 1326 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1327 require.True(t, ok, "data should already exist in in-memory cache") 1328 1329 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1330 require.False(t, ok, "data should not exist in database") 1331 1332 og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) 1333 1334 require.NotNil(t, og) 1335 assert.Nil(t, img) 1336 assert.Nil(t, err) 1337 assert.Equal(t, title, og.Title) 1338 }) 1339 1340 t.Run("should not use cache if URL is different", func(t *testing.T) { 1341 differentURL := server.URL + "/other" 1342 1343 _, _, ok := getLinkMetadataFromCache(differentURL, timestamp) 1344 require.False(t, ok, "data should not exist in in-memory cache") 1345 1346 _, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp) 1347 require.False(t, ok, "data should not exist in database") 1348 1349 og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false) 1350 1351 assert.Nil(t, og) 1352 assert.Nil(t, img) 1353 assert.Nil(t, err) 1354 }) 1355 1356 t.Run("should not use cache if timestamp is different", func(t *testing.T) { 1357 differentTimestamp := timestamp + 60*60*1000 1358 1359 _, _, ok := getLinkMetadataFromCache(requestURL, differentTimestamp) 1360 require.False(t, ok, "data should not exist in in-memory cache") 1361 1362 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp) 1363 require.False(t, ok, "data should not exist in database") 1364 1365 og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) 1366 1367 assert.Nil(t, og) 1368 assert.Nil(t, img) 1369 assert.Nil(t, err) 1370 }) 1371 }) 1372 1373 t.Run("database cache", func(t *testing.T) { 1374 th := setup() 1375 defer th.TearDown() 1376 1377 requestURL := server.URL 1378 timestamp := int64(1547510400000) 1379 title := "from database" 1380 1381 th.App.saveLinkMetadataToDatabase(requestURL, timestamp, &opengraph.OpenGraph{Title: title}, nil) 1382 1383 t.Run("should use database if saved entry exists", func(t *testing.T) { 1384 linkCache.Purge() 1385 1386 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1387 require.False(t, ok, "data should not exist in in-memory cache") 1388 1389 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1390 require.True(t, ok, "data should already exist in database") 1391 1392 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1393 1394 require.NotNil(t, og) 1395 assert.Nil(t, img) 1396 assert.Nil(t, err) 1397 assert.Equal(t, title, og.Title) 1398 }) 1399 1400 t.Run("should use database if saved entry exists near time", func(t *testing.T) { 1401 linkCache.Purge() 1402 1403 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1404 require.False(t, ok, "data should not exist in in-memory cache") 1405 1406 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1407 require.True(t, ok, "data should already exist in database") 1408 1409 og, img, err := th.App.getLinkMetadata(requestURL, timestamp+60*1000, false) 1410 1411 require.NotNil(t, og) 1412 assert.Nil(t, img) 1413 assert.Nil(t, err) 1414 assert.Equal(t, title, og.Title) 1415 }) 1416 1417 t.Run("should not use database if URL is different", func(t *testing.T) { 1418 linkCache.Purge() 1419 1420 differentURL := requestURL + "/other" 1421 1422 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1423 require.False(t, ok, "data should not exist in in-memory cache") 1424 1425 _, _, ok = th.App.getLinkMetadataFromDatabase(differentURL, timestamp) 1426 require.False(t, ok, "data should not exist in database") 1427 1428 og, img, err := th.App.getLinkMetadata(differentURL, timestamp, false) 1429 1430 assert.Nil(t, og) 1431 assert.Nil(t, img) 1432 assert.Nil(t, err) 1433 }) 1434 1435 t.Run("should not use database if timestamp is different", func(t *testing.T) { 1436 linkCache.Purge() 1437 1438 differentTimestamp := timestamp + 60*60*1000 1439 1440 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1441 require.False(t, ok, "data should not exist in in-memory cache") 1442 1443 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, differentTimestamp) 1444 require.False(t, ok, "data should not exist in database") 1445 1446 og, img, err := th.App.getLinkMetadata(requestURL, differentTimestamp, false) 1447 1448 assert.Nil(t, og) 1449 assert.Nil(t, img) 1450 assert.Nil(t, err) 1451 }) 1452 }) 1453 1454 t.Run("should get data from remote source", func(t *testing.T) { 1455 th := setup() 1456 defer th.TearDown() 1457 1458 requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name() 1459 timestamp := int64(1547510400000) 1460 1461 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1462 require.False(t, ok, "data should not exist in in-memory cache") 1463 1464 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1465 require.False(t, ok, "data should not exist in database") 1466 1467 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1468 1469 assert.NotNil(t, og) 1470 assert.Nil(t, img) 1471 assert.Nil(t, err) 1472 }) 1473 1474 t.Run("should cache OpenGraph results", func(t *testing.T) { 1475 th := setup() 1476 defer th.TearDown() 1477 1478 requestURL := server.URL + "/opengraph?title=Remote&name=" + t.Name() 1479 timestamp := int64(1547510400000) 1480 1481 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1482 require.False(t, ok, "data should not exist in in-memory cache") 1483 1484 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1485 require.False(t, ok, "data should not exist in database") 1486 1487 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1488 1489 assert.NotNil(t, og) 1490 assert.Nil(t, img) 1491 assert.Nil(t, err) 1492 1493 fromCache, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1494 assert.True(t, ok) 1495 assert.Exactly(t, og, fromCache) 1496 1497 fromDatabase, _, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1498 assert.True(t, ok) 1499 assert.Exactly(t, og, fromDatabase) 1500 }) 1501 1502 t.Run("should cache image results", func(t *testing.T) { 1503 th := setup() 1504 defer th.TearDown() 1505 1506 requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name() 1507 timestamp := int64(1547510400000) 1508 1509 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1510 require.False(t, ok, "data should not exist in in-memory cache") 1511 1512 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1513 require.False(t, ok, "data should not exist in database") 1514 1515 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1516 1517 assert.Nil(t, og) 1518 assert.NotNil(t, img) 1519 assert.Nil(t, err) 1520 1521 _, fromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1522 assert.True(t, ok) 1523 assert.Exactly(t, img, fromCache) 1524 1525 _, fromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1526 assert.True(t, ok) 1527 assert.Exactly(t, img, fromDatabase) 1528 }) 1529 1530 t.Run("should cache general errors", func(t *testing.T) { 1531 th := setup() 1532 defer th.TearDown() 1533 1534 requestURL := server.URL + "/error" 1535 timestamp := int64(1547510400000) 1536 1537 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1538 require.False(t, ok, "data should not exist in in-memory cache") 1539 1540 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1541 require.False(t, ok, "data should not exist in database") 1542 1543 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1544 1545 assert.Nil(t, og) 1546 assert.Nil(t, img) 1547 assert.Nil(t, err) 1548 1549 ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1550 assert.True(t, ok) 1551 assert.Nil(t, ogFromCache) 1552 assert.Nil(t, imgFromCache) 1553 1554 ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1555 assert.True(t, ok) 1556 assert.Nil(t, ogFromDatabase) 1557 assert.Nil(t, imageFromDatabase) 1558 }) 1559 1560 t.Run("should cache invalid URL errors", func(t *testing.T) { 1561 th := setup() 1562 defer th.TearDown() 1563 1564 requestURL := "http://notarealdomainthatactuallyexists.ca/?name=" + t.Name() 1565 timestamp := int64(1547510400000) 1566 1567 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1568 require.False(t, ok, "data should not exist in in-memory cache") 1569 1570 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1571 require.False(t, ok, "data should not exist in database") 1572 1573 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1574 1575 assert.Nil(t, og) 1576 assert.Nil(t, img) 1577 assert.IsType(t, &url.Error{}, err) 1578 1579 ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1580 assert.True(t, ok) 1581 assert.Nil(t, ogFromCache) 1582 assert.Nil(t, imgFromCache) 1583 1584 ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1585 assert.True(t, ok) 1586 assert.Nil(t, ogFromDatabase) 1587 assert.Nil(t, imageFromDatabase) 1588 }) 1589 1590 t.Run("should cache timeout errors", func(t *testing.T) { 1591 th := setup() 1592 defer th.TearDown() 1593 1594 th.App.UpdateConfig(func(cfg *model.Config) { 1595 *cfg.ExperimentalSettings.LinkMetadataTimeoutMilliseconds = 100 1596 }) 1597 1598 requestURL := server.URL + "/timeout?name=" + t.Name() 1599 timestamp := int64(1547510400000) 1600 1601 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1602 require.False(t, ok, "data should not exist in in-memory cache") 1603 1604 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1605 require.False(t, ok, "data should not exist in database") 1606 1607 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1608 1609 assert.Nil(t, og) 1610 assert.Nil(t, img) 1611 assert.NotNil(t, err) 1612 assert.Contains(t, err.Error(), "Client.Timeout") 1613 1614 ogFromCache, imgFromCache, ok := getLinkMetadataFromCache(requestURL, timestamp) 1615 assert.True(t, ok) 1616 assert.Nil(t, ogFromCache) 1617 assert.Nil(t, imgFromCache) 1618 1619 ogFromDatabase, imageFromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1620 assert.True(t, ok) 1621 assert.Nil(t, ogFromDatabase) 1622 assert.Nil(t, imageFromDatabase) 1623 }) 1624 1625 t.Run("should cache database results in memory", func(t *testing.T) { 1626 th := setup() 1627 defer th.TearDown() 1628 1629 requestURL := server.URL + "/image?height=300&width=400&name=" + t.Name() 1630 timestamp := int64(1547510400000) 1631 1632 _, _, ok := getLinkMetadataFromCache(requestURL, timestamp) 1633 require.False(t, ok, "data should not exist in in-memory cache") 1634 1635 _, _, ok = th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1636 require.False(t, ok, "data should not exist in database") 1637 1638 _, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1639 require.Nil(t, err) 1640 1641 _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) 1642 require.True(t, ok, "data should now exist in in-memory cache") 1643 1644 linkCache.Purge() 1645 _, _, ok = getLinkMetadataFromCache(requestURL, timestamp) 1646 require.False(t, ok, "data should no longer exist in in-memory cache") 1647 1648 _, fromDatabase, ok := th.App.getLinkMetadataFromDatabase(requestURL, timestamp) 1649 assert.True(t, ok, "data should be be in in-memory cache again") 1650 assert.Exactly(t, img, fromDatabase) 1651 }) 1652 1653 t.Run("should reject non-html, non-image response", func(t *testing.T) { 1654 th := setup() 1655 defer th.TearDown() 1656 1657 requestURL := server.URL + "/json?name=" + t.Name() 1658 timestamp := int64(1547510400000) 1659 1660 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1661 assert.Nil(t, og) 1662 assert.Nil(t, img) 1663 assert.Nil(t, err) 1664 }) 1665 1666 t.Run("should check in-memory cache for new post", func(t *testing.T) { 1667 th := setup() 1668 defer th.TearDown() 1669 1670 requestURL := server.URL + "/error?name=" + t.Name() 1671 timestamp := int64(1547510400000) 1672 1673 cacheLinkMetadata(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil) 1674 1675 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true) 1676 assert.NotNil(t, og) 1677 assert.Nil(t, img) 1678 assert.Nil(t, err) 1679 }) 1680 1681 t.Run("should skip database cache for new post", func(t *testing.T) { 1682 th := setup() 1683 defer th.TearDown() 1684 1685 requestURL := server.URL + "/error?name=" + t.Name() 1686 timestamp := int64(1547510400000) 1687 1688 th.App.saveLinkMetadataToDatabase(requestURL, timestamp, &opengraph.OpenGraph{Title: "cached"}, nil) 1689 1690 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, true) 1691 assert.Nil(t, og) 1692 assert.Nil(t, img) 1693 assert.Nil(t, err) 1694 }) 1695 1696 t.Run("should resolve relative URL", func(t *testing.T) { 1697 th := setup() 1698 defer th.TearDown() 1699 1700 // Fake the SiteURL to have the relative URL resolve to the external server 1701 oldSiteURL := *th.App.Config().ServiceSettings.SiteURL 1702 defer th.App.UpdateConfig(func(cfg *model.Config) { 1703 *cfg.ServiceSettings.SiteURL = oldSiteURL 1704 }) 1705 1706 th.App.UpdateConfig(func(cfg *model.Config) { 1707 *cfg.ServiceSettings.SiteURL = server.URL 1708 }) 1709 1710 requestURL := "/image?height=200&width=300&name=" + t.Name() 1711 timestamp := int64(1547510400000) 1712 1713 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1714 assert.Nil(t, og) 1715 assert.NotNil(t, img) 1716 assert.Nil(t, err) 1717 }) 1718 1719 t.Run("should error on local addresses other than the image proxy", func(t *testing.T) { 1720 th := setup() 1721 defer th.TearDown() 1722 1723 // Disable AllowedUntrustedInternalConnections since it's turned on for the previous tests 1724 oldAllowUntrusted := *th.App.Config().ServiceSettings.AllowedUntrustedInternalConnections 1725 oldSiteURL := *th.App.Config().ServiceSettings.SiteURL 1726 defer th.App.UpdateConfig(func(cfg *model.Config) { 1727 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = oldAllowUntrusted 1728 *cfg.ServiceSettings.SiteURL = oldSiteURL 1729 }) 1730 1731 th.App.UpdateConfig(func(cfg *model.Config) { 1732 *cfg.ServiceSettings.AllowedUntrustedInternalConnections = "" 1733 *cfg.ServiceSettings.SiteURL = "http://mattermost.example.com" 1734 *cfg.ImageProxySettings.Enable = true 1735 *cfg.ImageProxySettings.ImageProxyType = "local" 1736 }) 1737 1738 requestURL := server.URL + "/image?height=200&width=300&name=" + t.Name() 1739 timestamp := int64(1547510400000) 1740 1741 og, img, err := th.App.getLinkMetadata(requestURL, timestamp, false) 1742 assert.Nil(t, og) 1743 assert.Nil(t, img) 1744 assert.NotNil(t, err) 1745 assert.IsType(t, &url.Error{}, err) 1746 assert.Equal(t, httpservice.AddressForbidden, err.(*url.Error).Err) 1747 1748 requestURL = th.App.GetSiteURL() + "/api/v4/image?url=" + url.QueryEscape(requestURL) 1749 1750 // Note that this request still fails while testing because the request made by the image proxy is blocked 1751 og, img, err = th.App.getLinkMetadata(requestURL, timestamp, false) 1752 assert.Nil(t, og) 1753 assert.Nil(t, img) 1754 assert.NotNil(t, err) 1755 assert.IsType(t, imageproxy.Error{}, err) 1756 }) 1757 } 1758 1759 func TestResolveMetadataURL(t *testing.T) { 1760 for _, test := range []struct { 1761 Name string 1762 RequestURL string 1763 SiteURL string 1764 Expected string 1765 }{ 1766 { 1767 Name: "with HTTPS", 1768 RequestURL: "https://example.com/file?param=1", 1769 Expected: "https://example.com/file?param=1", 1770 }, 1771 { 1772 Name: "with HTTP", 1773 RequestURL: "http://example.com/file?param=1", 1774 Expected: "http://example.com/file?param=1", 1775 }, 1776 { 1777 Name: "with FTP", 1778 RequestURL: "ftp://example.com/file?param=1", 1779 Expected: "ftp://example.com/file?param=1", 1780 }, 1781 { 1782 Name: "relative to root", 1783 RequestURL: "/file?param=1", 1784 SiteURL: "https://mattermost.example.com:123", 1785 Expected: "https://mattermost.example.com:123/file?param=1", 1786 }, 1787 { 1788 Name: "relative to root with subpath", 1789 RequestURL: "/file?param=1", 1790 SiteURL: "https://mattermost.example.com:123/subpath", 1791 Expected: "https://mattermost.example.com:123/file?param=1", 1792 }, 1793 } { 1794 t.Run(test.Name, func(t *testing.T) { 1795 assert.Equal(t, resolveMetadataURL(test.RequestURL, test.SiteURL), test.Expected) 1796 }) 1797 } 1798 } 1799 1800 func TestParseLinkMetadata(t *testing.T) { 1801 th := Setup(t).InitBasic() 1802 defer th.TearDown() 1803 1804 imageURL := "http://example.com/test.png" 1805 file, err := testutils.ReadTestFile("test.png") 1806 require.Nil(t, err) 1807 1808 ogURL := "https://example.com/hello" 1809 html := ` 1810 <html> 1811 <head> 1812 <meta property="og:title" content="Hello, World!"> 1813 <meta property="og:type" content="object"> 1814 <meta property="og:url" content="` + ogURL + `"> 1815 </head> 1816 </html>` 1817 1818 makeImageReader := func() io.Reader { 1819 return bytes.NewReader(file) 1820 } 1821 1822 makeOpenGraphReader := func() io.Reader { 1823 return strings.NewReader(html) 1824 } 1825 1826 t.Run("image", func(t *testing.T) { 1827 og, dimensions, err := th.App.parseLinkMetadata(imageURL, makeImageReader(), "image/png") 1828 assert.Nil(t, err) 1829 1830 assert.Nil(t, og) 1831 assert.Equal(t, &model.PostImage{ 1832 Width: 408, 1833 Height: 336, 1834 }, dimensions) 1835 }) 1836 1837 t.Run("malformed image", func(t *testing.T) { 1838 og, dimensions, err := th.App.parseLinkMetadata(imageURL, makeOpenGraphReader(), "image/png") 1839 assert.NotNil(t, err) 1840 1841 assert.Nil(t, og) 1842 assert.Nil(t, dimensions) 1843 }) 1844 1845 t.Run("opengraph", func(t *testing.T) { 1846 og, dimensions, err := th.App.parseLinkMetadata(ogURL, makeOpenGraphReader(), "text/html; charset=utf-8") 1847 assert.Nil(t, err) 1848 1849 assert.NotNil(t, og) 1850 assert.Equal(t, og.Title, "Hello, World!") 1851 assert.Equal(t, og.Type, "object") 1852 assert.Equal(t, og.URL, ogURL) 1853 assert.Nil(t, dimensions) 1854 }) 1855 1856 t.Run("malformed opengraph", func(t *testing.T) { 1857 og, dimensions, err := th.App.parseLinkMetadata(ogURL, makeImageReader(), "text/html; charset=utf-8") 1858 assert.Nil(t, err) 1859 1860 assert.Nil(t, og) 1861 assert.Nil(t, dimensions) 1862 }) 1863 1864 t.Run("neither", func(t *testing.T) { 1865 og, dimensions, err := th.App.parseLinkMetadata("http://example.com/test.wad", strings.NewReader("garbage"), "application/x-doom") 1866 assert.Nil(t, err) 1867 1868 assert.Nil(t, og) 1869 assert.Nil(t, dimensions) 1870 }) 1871 } 1872 1873 func TestParseImages(t *testing.T) { 1874 for name, testCase := range map[string]struct { 1875 FileName string 1876 ExpectedWidth int 1877 ExpectedHeight int 1878 ExpectError bool 1879 }{ 1880 "png": { 1881 FileName: "test.png", 1882 ExpectedWidth: 408, 1883 ExpectedHeight: 336, 1884 }, 1885 "animated gif": { 1886 FileName: "testgif.gif", 1887 ExpectedWidth: 118, 1888 ExpectedHeight: 118, 1889 }, 1890 "not an image": { 1891 FileName: "README.md", 1892 ExpectError: true, 1893 }, 1894 } { 1895 t.Run(name, func(t *testing.T) { 1896 file, err := testutils.ReadTestFile(testCase.FileName) 1897 require.Nil(t, err) 1898 1899 dimensions, err := parseImages(bytes.NewReader(file)) 1900 if testCase.ExpectError { 1901 require.NotNil(t, err) 1902 } else { 1903 require.Nil(t, err) 1904 1905 require.NotNil(t, dimensions) 1906 require.Equal(t, testCase.ExpectedWidth, dimensions.Width) 1907 require.Equal(t, testCase.ExpectedHeight, dimensions.Height) 1908 } 1909 }) 1910 } 1911 }