github.com/adacta-ru/mattermost-server/v6@v6.0.0/app/import_test.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See LICENSE.txt for license information. 3 4 package app 5 6 import ( 7 "io/ioutil" 8 "net/http" 9 "os" 10 "path/filepath" 11 "runtime" 12 "strings" 13 "testing" 14 15 "github.com/stretchr/testify/assert" 16 "github.com/stretchr/testify/require" 17 18 "github.com/adacta-ru/mattermost-server/v6/model" 19 "github.com/adacta-ru/mattermost-server/v6/utils" 20 "github.com/adacta-ru/mattermost-server/v6/utils/fileutils" 21 ) 22 23 func ptrStr(s string) *string { 24 return &s 25 } 26 27 func ptrInt64(i int64) *int64 { 28 return &i 29 } 30 31 func ptrInt(i int) *int { 32 return &i 33 } 34 35 func ptrBool(b bool) *bool { 36 return &b 37 } 38 39 func checkPreference(t *testing.T, a *App, userId string, category string, name string, value string) { 40 preferences, err := a.Srv().Store.Preference().GetCategory(userId, category) 41 require.Nilf(t, err, "Failed to get preferences for user %v with category %v", userId, category) 42 found := false 43 for _, preference := range preferences { 44 if preference.Name == name { 45 found = true 46 require.Equal(t, preference.Value, value, "Preference for user %v in category %v with name %v has value %v, expected %v", userId, category, name, preference.Value, value) 47 break 48 } 49 } 50 require.Truef(t, found, "Did not find preference for user %v in category %v with name %v", userId, category, name) 51 } 52 53 func checkNotifyProp(t *testing.T, user *model.User, key string, value string) { 54 actual, ok := user.NotifyProps[key] 55 require.True(t, ok, "Notify prop %v not found. User: %v", key, user.Id) 56 require.Equalf(t, actual, value, "Notify Prop %v was %v but expected %v. User: %v", key, actual, value, user.Id) 57 } 58 59 func checkError(t *testing.T, err *model.AppError) { 60 require.NotNil(t, err, "Should have returned an error.") 61 } 62 63 func checkNoError(t *testing.T, err *model.AppError) { 64 require.Nil(t, err, "Unexpected Error: %v", err) 65 } 66 67 func AssertAllPostsCount(t *testing.T, a *App, initialCount int64, change int64, teamName string) { 68 result, err := a.Srv().Store.Post().AnalyticsPostCount(teamName, false, false) 69 require.Nil(t, err) 70 require.Equal(t, initialCount+change, result, "Did not find the expected number of posts.") 71 } 72 73 func AssertChannelCount(t *testing.T, a *App, channelType string, expectedCount int64) { 74 count, err := a.Srv().Store.Channel().AnalyticsTypeCount("", channelType) 75 require.Equalf(t, expectedCount, count, "Channel count of type: %v. Expected: %v, Got: %v", channelType, expectedCount, count) 76 require.Nil(t, err, "Failed to get channel count.") 77 } 78 79 func TestImportImportLine(t *testing.T) { 80 th := Setup(t) 81 defer th.TearDown() 82 83 // Try import line with an invalid type. 84 line := LineImportData{ 85 Type: "gibberish", 86 } 87 88 err := th.App.importLine(line, false) 89 require.NotNil(t, err, "Expected an error when importing a line with invalid type.") 90 91 // Try import line with team type but nil team. 92 line.Type = "team" 93 err = th.App.importLine(line, false) 94 require.NotNil(t, err, "Expected an error when importing a line of type team with a nil team.") 95 96 // Try import line with channel type but nil channel. 97 line.Type = "channel" 98 err = th.App.importLine(line, false) 99 require.NotNil(t, err, "Expected an error when importing a line with type channel with a nil channel.") 100 101 // Try import line with user type but nil user. 102 line.Type = "user" 103 err = th.App.importLine(line, false) 104 require.NotNil(t, err, "Expected an error when importing a line with type user with a nil user.") 105 106 // Try import line with post type but nil post. 107 line.Type = "post" 108 err = th.App.importLine(line, false) 109 require.NotNil(t, err, "Expected an error when importing a line with type post with a nil post.") 110 111 // Try import line with direct_channel type but nil direct_channel. 112 line.Type = "direct_channel" 113 err = th.App.importLine(line, false) 114 require.NotNil(t, err, "Expected an error when importing a line with type direct_channel with a nil direct_channel.") 115 116 // Try import line with direct_post type but nil direct_post. 117 line.Type = "direct_post" 118 err = th.App.importLine(line, false) 119 require.NotNil(t, err, "Expected an error when importing a line with type direct_post with a nil direct_post.") 120 121 // Try import line with scheme type but nil scheme. 122 line.Type = "scheme" 123 err = th.App.importLine(line, false) 124 require.NotNil(t, err, "Expected an error when importing a line with type scheme with a nil scheme.") 125 } 126 127 func TestStopOnError(t *testing.T) { 128 assert.True(t, stopOnError(LineImportWorkerError{ 129 model.NewAppError("test", "app.import.attachment.bad_file.error", nil, "", http.StatusBadRequest), 130 1, 131 })) 132 133 assert.True(t, stopOnError(LineImportWorkerError{ 134 model.NewAppError("test", "app.import.attachment.file_upload.error", nil, "", http.StatusBadRequest), 135 1, 136 })) 137 138 assert.False(t, stopOnError(LineImportWorkerError{ 139 model.NewAppError("test", "api.file.upload_file.large_image.app_error", nil, "", http.StatusBadRequest), 140 1, 141 })) 142 } 143 144 func TestImportBulkImport(t *testing.T) { 145 th := Setup(t) 146 defer th.TearDown() 147 148 th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.EnableCustomEmoji = true }) 149 150 teamName := model.NewRandomTeamName() 151 channelName := model.NewId() 152 username := model.NewId() 153 username2 := model.NewId() 154 username3 := model.NewId() 155 emojiName := model.NewId() 156 testsDir, _ := fileutils.FindDir("tests") 157 testImage := filepath.Join(testsDir, "test.png") 158 teamTheme1 := `{\"awayIndicator\":\"#DBBD4E\",\"buttonBg\":\"#23A1FF\",\"buttonColor\":\"#FFFFFF\",\"centerChannelBg\":\"#ffffff\",\"centerChannelColor\":\"#333333\",\"codeTheme\":\"github\",\"image\":\"/static/files/a4a388b38b32678e83823ef1b3e17766.png\",\"linkColor\":\"#2389d7\",\"mentionBg\":\"#2389d7\",\"mentionColor\":\"#ffffff\",\"mentionHighlightBg\":\"#fff2bb\",\"mentionHighlightLink\":\"#2f81b7\",\"newMessageSeparator\":\"#FF8800\",\"onlineIndicator\":\"#7DBE00\",\"sidebarBg\":\"#fafafa\",\"sidebarHeaderBg\":\"#3481B9\",\"sidebarHeaderTextColor\":\"#ffffff\",\"sidebarText\":\"#333333\",\"sidebarTextActiveBorder\":\"#378FD2\",\"sidebarTextActiveColor\":\"#111111\",\"sidebarTextHoverBg\":\"#e6f2fa\",\"sidebarUnreadText\":\"#333333\",\"type\":\"Mattermost\"}` 159 teamTheme2 := `{\"awayIndicator\":\"#DBBD4E\",\"buttonBg\":\"#23A100\",\"buttonColor\":\"#EEEEEE\",\"centerChannelBg\":\"#ffffff\",\"centerChannelColor\":\"#333333\",\"codeTheme\":\"github\",\"image\":\"/static/files/a4a388b38b32678e83823ef1b3e17766.png\",\"linkColor\":\"#2389d7\",\"mentionBg\":\"#2389d7\",\"mentionColor\":\"#ffffff\",\"mentionHighlightBg\":\"#fff2bb\",\"mentionHighlightLink\":\"#2f81b7\",\"newMessageSeparator\":\"#FF8800\",\"onlineIndicator\":\"#7DBE00\",\"sidebarBg\":\"#fafafa\",\"sidebarHeaderBg\":\"#3481B9\",\"sidebarHeaderTextColor\":\"#ffffff\",\"sidebarText\":\"#333333\",\"sidebarTextActiveBorder\":\"#378FD2\",\"sidebarTextActiveColor\":\"#222222\",\"sidebarTextHoverBg\":\"#e6f2fa\",\"sidebarUnreadText\":\"#444444\",\"type\":\"Mattermost\"}` 160 161 // Run bulk import with a valid 1 of everything. 162 data1 := `{"type": "version", "version": 1} 163 {"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} 164 {"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}} 165 {"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme1 + `", "channels": [{"name": "` + channelName + `"}]}]}} 166 {"type": "user", "user": {"username": "` + username2 + `", "email": "` + username2 + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme2 + `", "channels": [{"name": "` + channelName + `"}]}]}} 167 {"type": "user", "user": {"username": "` + username3 + `", "email": "` + username3 + `@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}], "delete_at": 123456789016}]}} 168 {"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012, "attachments":[{"path": "` + testImage + `"}]}} 169 {"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username3 + `", "message": "Hey Everyone!", "create_at": 123456789013, "attachments":[{"path": "` + testImage + `"}]}} 170 {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username + `"]}} 171 {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username2 + `"]}} 172 {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username2 + `", "` + username3 + `"]}} 173 {"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username + `"], "user": "` + username + `", "message": "Hello Direct Channel to myself", "create_at": 123456789014}} 174 {"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username2 + `"], "user": "` + username + `", "message": "Hello Direct Channel", "create_at": 123456789014}} 175 {"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username2 + `", "` + username3 + `"], "user": "` + username + `", "message": "Hello Group Channel", "create_at": 123456789015}} 176 {"type": "emoji", "emoji": {"name": "` + emojiName + `", "image": "` + testImage + `"}}` 177 178 err, line := th.App.BulkImport(strings.NewReader(data1), false, 2) 179 require.Nil(t, err, "BulkImport should have succeeded") 180 require.Equal(t, 0, line, "BulkImport line should be 0") 181 182 // Run bulk import using a string that contains a line with invalid json. 183 data2 := `{"type": "version", "version": 1` 184 err, line = th.App.BulkImport(strings.NewReader(data2), false, 2) 185 require.NotNil(t, err, "Should have failed due to invalid JSON on line 1.") 186 require.Equal(t, 1, line, "Should have failed due to invalid JSON on line 1.") 187 188 // Run bulk import using valid JSON but missing version line at the start. 189 data3 := `{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} 190 {"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}} 191 {"type": "user", "user": {"username": "kufjgnkxkrhhfgbrip6qxkfsaa", "email": "kufjgnkxkrhhfgbrip6qxkfsaa@example.com"}} 192 {"type": "user", "user": {"username": "bwshaim6qnc2ne7oqkd5b2s2rq", "email": "bwshaim6qnc2ne7oqkd5b2s2rq@example.com", "teams": [{"name": "` + teamName + `", "channels": [{"name": "` + channelName + `"}]}]}}` 193 err, line = th.App.BulkImport(strings.NewReader(data3), false, 2) 194 require.NotNil(t, err, "Should have failed due to missing version line on line 1.") 195 require.Equal(t, 1, line, "Should have failed due to missing version line on line 1.") 196 197 // Run bulk import using a valid and large input and a \r\n line break. 198 t.Run("", func(t *testing.T) { 199 posts := `{"type": "post"` + strings.Repeat(`, "post": {"team": "`+teamName+`", "channel": "`+channelName+`", "user": "`+username+`", "message": "Repeat after me", "create_at": 193456789012}`, 1e4) + "}" 200 data4 := `{"type": "version", "version": 1} 201 {"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} 202 {"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}} 203 {"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme1 + `", "channels": [{"name": "` + channelName + `"}]}]}} 204 {"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}}` 205 err, line = th.App.BulkImport(strings.NewReader(data4+"\r\n"+posts), false, 2) 206 require.Nil(t, err, "BulkImport should have succeeded") 207 require.Equal(t, 0, line, "BulkImport line should be 0") 208 }) 209 210 t.Run("First item after version without type", func(t *testing.T) { 211 data := `{"type": "version", "version": 1} 212 {"name": "custom-emoji-troll", "image": "bulkdata/emoji/trollolol.png"}` 213 err, line := th.App.BulkImport(strings.NewReader(data), false, 2) 214 require.NotNil(t, err, "Should have failed due to invalid type on line 2.") 215 require.Equal(t, 2, line, "Should have failed due to invalid type on line 2.") 216 }) 217 218 t.Run("Posts with prop information", func(t *testing.T) { 219 data6 := `{"type": "version", "version": 1} 220 {"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}} 221 {"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}} 222 {"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme1 + `", "channels": [{"name": "` + channelName + `"}]}]}} 223 {"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012, "attachments":[{"path": "` + testImage + `"}], "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}} 224 {"type": "direct_channel", "direct_channel": {"members": ["` + username + `", "` + username + `"]}} 225 {"type": "direct_post", "direct_post": {"channel_members": ["` + username + `", "` + username + `"], "user": "` + username + `", "message": "Hello Direct Channel to myself", "create_at": 123456789014, "props":{"attachments":[{"id":0,"fallback":"[February 4th, 2020 2:46 PM] author: fallback","color":"D0D0D0","pretext":"","author_name":"author","author_link":"","title":"","title_link":"","text":"this post has props","fields":null,"image_url":"","thumb_url":"","footer":"Posted in #general","footer_icon":"","ts":"1580823992.000100"}]}}}}` 226 227 err, line := th.App.BulkImport(strings.NewReader(data6), false, 2) 228 require.Nil(t, err, "BulkImport should have succeeded") 229 require.Equal(t, 0, line, "BulkImport line should be 0") 230 }) 231 } 232 233 func TestImportProcessImportDataFileVersionLine(t *testing.T) { 234 data := LineImportData{ 235 Type: "version", 236 Version: ptrInt(1), 237 } 238 version, err := processImportDataFileVersionLine(data) 239 require.Nil(t, err, "Expected no error") 240 require.Equal(t, 1, version, "Expected version 1") 241 242 data.Type = "NotVersion" 243 _, err = processImportDataFileVersionLine(data) 244 require.NotNil(t, err, "Expected error on invalid version line.") 245 246 data.Type = "version" 247 data.Version = nil 248 _, err = processImportDataFileVersionLine(data) 249 require.NotNil(t, err, "Expected error on invalid version line.") 250 } 251 252 func GetAttachments(userId string, th *TestHelper, t *testing.T) []*model.FileInfo { 253 fileInfos, err := th.App.Srv().Store.FileInfo().GetForUser(userId) 254 require.Nil(t, err) 255 return fileInfos 256 } 257 258 func AssertFileIdsInPost(files []*model.FileInfo, th *TestHelper, t *testing.T) { 259 postId := files[0].PostId 260 require.NotNil(t, postId) 261 262 posts, err := th.App.Srv().Store.Post().GetPostsByIds([]string{postId}) 263 require.Nil(t, err) 264 265 require.Len(t, posts, 1) 266 for _, file := range files { 267 assert.Contains(t, posts[0].FileIds, file.Id) 268 } 269 } 270 271 func TestRewriteFilePaths(t *testing.T) { 272 genAttachments := func() *[]AttachmentImportData { 273 return &[]AttachmentImportData{ 274 { 275 Path: model.NewString("file.jpg"), 276 }, 277 { 278 Path: model.NewString("somedir/file.jpg"), 279 }, 280 } 281 } 282 283 line := LineImportData{ 284 Type: "post", 285 Post: &PostImportData{ 286 Attachments: genAttachments(), 287 }, 288 } 289 290 line2 := LineImportData{ 291 Type: "direct_post", 292 DirectPost: &DirectPostImportData{ 293 Attachments: genAttachments(), 294 }, 295 } 296 297 userLine := LineImportData{ 298 Type: "user", 299 User: &UserImportData{ 300 ProfileImage: model.NewString("profile.jpg"), 301 }, 302 } 303 304 emojiLine := LineImportData{ 305 Type: "emoji", 306 Emoji: &EmojiImportData{ 307 Image: model.NewString("emoji.png"), 308 }, 309 } 310 311 t.Run("empty path", func(t *testing.T) { 312 expected := &[]AttachmentImportData{ 313 { 314 Path: model.NewString("file.jpg"), 315 }, 316 { 317 Path: model.NewString("somedir/file.jpg"), 318 }, 319 } 320 rewriteFilePaths(&line, "") 321 require.Equal(t, expected, line.Post.Attachments) 322 rewriteFilePaths(&line2, "") 323 require.Equal(t, expected, line2.DirectPost.Attachments) 324 }) 325 326 t.Run("valid path", func(t *testing.T) { 327 expected := &[]AttachmentImportData{ 328 { 329 Path: model.NewString("/tmp/file.jpg"), 330 }, 331 { 332 Path: model.NewString("/tmp/somedir/file.jpg"), 333 }, 334 } 335 336 t.Run("post attachments", func(t *testing.T) { 337 rewriteFilePaths(&line, "/tmp") 338 require.Equal(t, expected, line.Post.Attachments) 339 }) 340 341 t.Run("direct post attachments", func(t *testing.T) { 342 rewriteFilePaths(&line2, "/tmp") 343 require.Equal(t, expected, line2.DirectPost.Attachments) 344 }) 345 346 t.Run("profile image", func(t *testing.T) { 347 expected := "/tmp/profile.jpg" 348 rewriteFilePaths(&userLine, "/tmp") 349 require.Equal(t, expected, *userLine.User.ProfileImage) 350 }) 351 352 t.Run("emoji", func(t *testing.T) { 353 expected := "/tmp/emoji.png" 354 rewriteFilePaths(&emojiLine, "/tmp") 355 require.Equal(t, expected, *emojiLine.Emoji.Image) 356 }) 357 }) 358 } 359 360 func BenchmarkBulkImport(b *testing.B) { 361 th := Setup(b) 362 defer th.TearDown() 363 364 testsDir, _ := fileutils.FindDir("tests") 365 366 importFile, err := os.Open(testsDir + "/import_test.zip") 367 require.Nil(b, err) 368 defer importFile.Close() 369 370 info, err := importFile.Stat() 371 require.Nil(b, err) 372 373 dir, err := ioutil.TempDir("", "testimport") 374 require.Nil(b, err) 375 defer os.RemoveAll(dir) 376 377 _, err = utils.UnzipToPath(importFile, info.Size(), dir) 378 require.Nil(b, err) 379 380 jsonFile, err := os.Open(dir + "/import.jsonl") 381 require.Nil(b, err) 382 defer jsonFile.Close() 383 384 b.ResetTimer() 385 for i := 0; i < b.N; i++ { 386 err, _ := th.App.BulkImportWithPath(jsonFile, false, runtime.NumCPU(), dir) 387 require.Nil(b, err) 388 } 389 b.StopTimer() 390 }