github.com/crspeller/mattermost-server@v0.0.0-20190328001957-a200beb3d111/app/export.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "encoding/json" 8 "io" 9 "net/http" 10 "os" 11 "path/filepath" 12 "strings" 13 14 "github.com/crspeller/mattermost-server/mlog" 15 "github.com/crspeller/mattermost-server/model" 16 "github.com/pkg/errors" 17 ) 18 19 // We use this map to identify the exportable preferences. 20 // Here we link the preference category and name, to the name of the relevant filed in the import struct. 21 var exportablePreferences = map[ComparablePreference]string{{ 22 Category: model.PREFERENCE_CATEGORY_THEME, 23 Name: "", 24 }: "Theme", { 25 Category: model.PREFERENCE_CATEGORY_ADVANCED_SETTINGS, 26 Name: "feature_enabled_markdown_preview", 27 }: "UseMarkdownPreview", { 28 Category: model.PREFERENCE_CATEGORY_ADVANCED_SETTINGS, 29 Name: "formatting", 30 }: "UseFormatting", { 31 Category: model.PREFERENCE_CATEGORY_SIDEBAR_SETTINGS, 32 Name: "show_unread_section", 33 }: "ShowUnreadSection", { 34 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 35 Name: model.PREFERENCE_NAME_USE_MILITARY_TIME, 36 }: "UseMilitaryTime", { 37 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 38 Name: model.PREFERENCE_NAME_COLLAPSE_SETTING, 39 }: "CollapsePreviews", { 40 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 41 Name: model.PREFERENCE_NAME_MESSAGE_DISPLAY, 42 }: "MessageDisplay", { 43 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 44 Name: "channel_display_mode", 45 }: "ChannelDisplayMode", { 46 Category: model.PREFERENCE_CATEGORY_TUTORIAL_STEPS, 47 Name: "", 48 }: "TutorialStep", { 49 Category: model.PREFERENCE_CATEGORY_NOTIFICATIONS, 50 Name: model.PREFERENCE_NAME_EMAIL_INTERVAL, 51 }: "EmailInterval", 52 } 53 54 func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { 55 if err := a.ExportVersion(writer); err != nil { 56 return err 57 } 58 59 if err := a.ExportAllTeams(writer); err != nil { 60 return err 61 } 62 63 if err := a.ExportAllChannels(writer); err != nil { 64 return err 65 } 66 67 if err := a.ExportAllUsers(writer); err != nil { 68 return err 69 } 70 71 if err := a.ExportAllPosts(writer); err != nil { 72 return err 73 } 74 75 if err := a.ExportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil { 76 return err 77 } 78 79 if err := a.ExportAllDirectChannels(writer); err != nil { 80 return err 81 } 82 83 if err := a.ExportAllDirectPosts(writer); err != nil { 84 return err 85 } 86 87 return nil 88 } 89 90 func (a *App) ExportWriteLine(writer io.Writer, line *LineImportData) *model.AppError { 91 b, err := json.Marshal(line) 92 if err != nil { 93 return model.NewAppError("BulkExport", "app.export.export_write_line.json_marshall.error", nil, "err="+err.Error(), http.StatusBadRequest) 94 } 95 96 if _, err := writer.Write(append(b, '\n')); err != nil { 97 return model.NewAppError("BulkExport", "app.export.export_write_line.io_writer.error", nil, "err="+err.Error(), http.StatusBadRequest) 98 } 99 100 return nil 101 } 102 103 func (a *App) ExportVersion(writer io.Writer) *model.AppError { 104 version := 1 105 versionLine := &LineImportData{ 106 Type: "version", 107 Version: &version, 108 } 109 110 return a.ExportWriteLine(writer, versionLine) 111 } 112 113 func (a *App) ExportAllTeams(writer io.Writer) *model.AppError { 114 afterId := strings.Repeat("0", 26) 115 for { 116 result := <-a.Srv.Store.Team().GetAllForExportAfter(1000, afterId) 117 118 if result.Err != nil { 119 return result.Err 120 } 121 122 teams := result.Data.([]*model.TeamForExport) 123 124 if len(teams) == 0 { 125 break 126 } 127 128 for _, team := range teams { 129 afterId = team.Id 130 131 // Skip deleted. 132 if team.DeleteAt != 0 { 133 continue 134 } 135 136 teamLine := ImportLineFromTeam(team) 137 if err := a.ExportWriteLine(writer, teamLine); err != nil { 138 return err 139 } 140 } 141 } 142 143 return nil 144 } 145 146 func (a *App) ExportAllChannels(writer io.Writer) *model.AppError { 147 afterId := strings.Repeat("0", 26) 148 for { 149 result := <-a.Srv.Store.Channel().GetAllChannelsForExportAfter(1000, afterId) 150 151 if result.Err != nil { 152 return result.Err 153 } 154 155 channels := result.Data.([]*model.ChannelForExport) 156 157 if len(channels) == 0 { 158 break 159 } 160 161 for _, channel := range channels { 162 afterId = channel.Id 163 164 // Skip deleted. 165 if channel.DeleteAt != 0 { 166 continue 167 } 168 169 channelLine := ImportLineFromChannel(channel) 170 if err := a.ExportWriteLine(writer, channelLine); err != nil { 171 return err 172 } 173 } 174 } 175 176 return nil 177 } 178 179 func (a *App) ExportAllUsers(writer io.Writer) *model.AppError { 180 afterId := strings.Repeat("0", 26) 181 for { 182 result := <-a.Srv.Store.User().GetAllAfter(1000, afterId) 183 184 if result.Err != nil { 185 return result.Err 186 } 187 188 users := result.Data.([]*model.User) 189 190 if len(users) == 0 { 191 break 192 } 193 194 for _, user := range users { 195 afterId = user.Id 196 197 // Gathering here the exportable preferences to pass them on to ImportLineFromUser 198 exportedPrefs := make(map[string]*string) 199 allPrefs, err := a.GetPreferencesForUser(user.Id) 200 if err != nil { 201 return err 202 } 203 for _, pref := range allPrefs { 204 // We need to manage the special cases 205 // Here we manage Tutorial steps 206 if pref.Category == model.PREFERENCE_CATEGORY_TUTORIAL_STEPS { 207 pref.Name = "" 208 // Then the email interval 209 } else if pref.Category == model.PREFERENCE_CATEGORY_NOTIFICATIONS && pref.Name == model.PREFERENCE_NAME_EMAIL_INTERVAL { 210 switch pref.Value { 211 case model.PREFERENCE_EMAIL_INTERVAL_NO_BATCHING_SECONDS: 212 pref.Value = model.PREFERENCE_EMAIL_INTERVAL_IMMEDIATELY 213 case model.PREFERENCE_EMAIL_INTERVAL_FIFTEEN_AS_SECONDS: 214 pref.Value = model.PREFERENCE_EMAIL_INTERVAL_FIFTEEN 215 case model.PREFERENCE_EMAIL_INTERVAL_HOUR_AS_SECONDS: 216 pref.Value = model.PREFERENCE_EMAIL_INTERVAL_HOUR 217 case "0": 218 pref.Value = "" 219 } 220 } 221 id, ok := exportablePreferences[ComparablePreference{ 222 Category: pref.Category, 223 Name: pref.Name, 224 }] 225 if ok { 226 prefPtr := pref.Value 227 if prefPtr != "" { 228 exportedPrefs[id] = &prefPtr 229 } else { 230 exportedPrefs[id] = nil 231 } 232 } 233 } 234 235 userLine := ImportLineFromUser(user, exportedPrefs) 236 237 userLine.User.NotifyProps = a.buildUserNotifyProps(user.NotifyProps) 238 239 // Do the Team Memberships. 240 members, err := a.buildUserTeamAndChannelMemberships(user.Id) 241 if err != nil { 242 return err 243 } 244 245 userLine.User.Teams = members 246 247 if err := a.ExportWriteLine(writer, userLine); err != nil { 248 return err 249 } 250 } 251 } 252 253 return nil 254 } 255 256 func (a *App) buildUserTeamAndChannelMemberships(userId string) (*[]UserTeamImportData, *model.AppError) { 257 var memberships []UserTeamImportData 258 259 result := <-a.Srv.Store.Team().GetTeamMembersForExport(userId) 260 261 if result.Err != nil { 262 return nil, result.Err 263 } 264 265 members := result.Data.([]*model.TeamMemberForExport) 266 267 for _, member := range members { 268 // Skip deleted. 269 if member.DeleteAt != 0 { 270 continue 271 } 272 273 memberData := ImportUserTeamDataFromTeamMember(member) 274 275 // Do the Channel Memberships. 276 channelMembers, err := a.buildUserChannelMemberships(userId, member.TeamId) 277 if err != nil { 278 return nil, err 279 } 280 281 memberData.Channels = channelMembers 282 283 memberships = append(memberships, *memberData) 284 } 285 286 return &memberships, nil 287 } 288 289 func (a *App) buildUserChannelMemberships(userId string, teamId string) (*[]UserChannelImportData, *model.AppError) { 290 var memberships []UserChannelImportData 291 292 result := <-a.Srv.Store.Channel().GetChannelMembersForExport(userId, teamId) 293 if result.Err != nil { 294 return nil, result.Err 295 } 296 297 members := result.Data.([]*model.ChannelMemberForExport) 298 299 category := model.PREFERENCE_CATEGORY_FAVORITE_CHANNEL 300 preferences, err := a.GetPreferenceByCategoryForUser(userId, category) 301 if err != nil && err.StatusCode != http.StatusNotFound { 302 return nil, err 303 } 304 305 for _, member := range members { 306 memberships = append(memberships, *ImportUserChannelDataFromChannelMemberAndPreferences(member, &preferences)) 307 } 308 return &memberships, nil 309 } 310 311 func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *UserNotifyPropsImportData { 312 313 getProp := func(key string) *string { 314 if v, ok := notifyProps[key]; ok { 315 return &v 316 } 317 return nil 318 } 319 320 return &UserNotifyPropsImportData{ 321 Desktop: getProp(model.DESKTOP_NOTIFY_PROP), 322 DesktopSound: getProp(model.DESKTOP_SOUND_NOTIFY_PROP), 323 Email: getProp(model.EMAIL_NOTIFY_PROP), 324 Mobile: getProp(model.PUSH_NOTIFY_PROP), 325 MobilePushStatus: getProp(model.PUSH_STATUS_NOTIFY_PROP), 326 ChannelTrigger: getProp(model.CHANNEL_MENTIONS_NOTIFY_PROP), 327 CommentsTrigger: getProp(model.COMMENTS_NOTIFY_PROP), 328 MentionKeys: getProp(model.MENTION_KEYS_NOTIFY_PROP), 329 } 330 } 331 332 func (a *App) ExportAllPosts(writer io.Writer) *model.AppError { 333 afterId := strings.Repeat("0", 26) 334 for { 335 result := <-a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId) 336 337 if result.Err != nil { 338 return result.Err 339 } 340 341 posts := result.Data.([]*model.PostForExport) 342 343 if len(posts) == 0 { 344 break 345 } 346 347 for _, post := range posts { 348 afterId = post.Id 349 350 // Skip deleted. 351 if post.DeleteAt != 0 { 352 continue 353 } 354 355 postLine := ImportLineForPost(post) 356 357 // Do the Replies. 358 replies, err := a.buildPostReplies(post.Id) 359 if err != nil { 360 return err 361 } 362 363 reactions, err := a.BuildPostReactions(post.Id) 364 if err != nil { 365 return err 366 } 367 368 postLine.Post.Replies = replies 369 370 postLine.Post.Reactions = reactions 371 372 if err := a.ExportWriteLine(writer, postLine); err != nil { 373 return err 374 } 375 } 376 } 377 378 return nil 379 } 380 381 func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) { 382 var replies []ReplyImportData 383 384 result := <-a.Srv.Store.Post().GetRepliesForExport(postId) 385 386 if result.Err != nil { 387 return nil, result.Err 388 } 389 390 replyPosts := result.Data.([]*model.ReplyForExport) 391 392 for _, reply := range replyPosts { 393 replyImportObject := ImportReplyFromPost(reply) 394 if reply.HasReactions == true { 395 reactionsOfReply, err := a.BuildPostReactions(reply.Id) 396 if err != nil { 397 return nil, err 398 } 399 replyImportObject.Reactions = reactionsOfReply 400 } 401 replies = append(replies, *replyImportObject) 402 } 403 404 return &replies, nil 405 } 406 407 func (a *App) BuildPostReactions(postId string) (*[]ReactionImportData, *model.AppError) { 408 var reactionsOfPost []ReactionImportData 409 410 result := <-a.Srv.Store.Reaction().GetForPost(postId, true) 411 if result.Err != nil { 412 return nil, result.Err 413 } 414 415 reactions := result.Data.([]*model.Reaction) 416 417 for _, reaction := range reactions { 418 result := <-a.Srv.Store.User().Get(reaction.UserId) 419 if result.Err != nil { 420 return nil, result.Err 421 } 422 user := result.Data.(*model.User) 423 reactionsOfPost = append(reactionsOfPost, *ImportReactionFromPost(user, reaction)) 424 } 425 426 return &reactionsOfPost, nil 427 428 } 429 430 func (a *App) ExportCustomEmoji(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { 431 pageNumber := 0 432 for { 433 customEmojiList, err := a.GetEmojiList(pageNumber, 100, model.EMOJI_SORT_BY_NAME) 434 435 if err != nil { 436 return err 437 } 438 439 if len(customEmojiList) == 0 { 440 break 441 } 442 443 pageNumber++ 444 445 pathToDir := a.createDirForEmoji(file, dirNameToExportEmoji) 446 447 for _, emoji := range customEmojiList { 448 emojiImagePath := pathToEmojiDir + emoji.Id + "/image" 449 err := a.copyEmojiImages(emoji.Id, emojiImagePath, pathToDir) 450 if err != nil { 451 return model.NewAppError("BulkExport", "app.export.export_custom_emoji.copy_emoji_images.error", nil, "err="+err.Error(), http.StatusBadRequest) 452 } 453 454 filePath := dirNameToExportEmoji + "/" + emoji.Id + "/image" 455 456 emojiImportObject := ImportLineFromEmoji(emoji, filePath) 457 458 if err := a.ExportWriteLine(writer, emojiImportObject); err != nil { 459 return err 460 } 461 } 462 } 463 464 return nil 465 } 466 467 // Creates directory named 'exported_emoji' to copy the emoji files 468 // Directory and the file specified by admin share the same path 469 func (a *App) createDirForEmoji(file string, dirName string) string { 470 pathToFile, _ := filepath.Abs(file) 471 pathSlice := strings.Split(pathToFile, "/") 472 if len(pathSlice) > 0 { 473 pathSlice = pathSlice[:len(pathSlice)-1] 474 } 475 pathToDir := strings.Join(pathSlice, "/") + "/" + dirName 476 477 if _, err := os.Stat(pathToDir); os.IsNotExist(err) { 478 os.Mkdir(pathToDir, os.ModePerm) 479 } 480 return pathToDir 481 } 482 483 // Copies emoji files from 'data/emoji' dir to 'exported_emoji' dir 484 func (a *App) copyEmojiImages(emojiId string, emojiImagePath string, pathToDir string) error { 485 fromPath, err := os.Open(emojiImagePath) 486 if fromPath == nil || err != nil { 487 return errors.New("Error reading " + emojiImagePath + "file") 488 } 489 defer fromPath.Close() 490 491 emojiDir := pathToDir + "/" + emojiId 492 493 if _, err = os.Stat(emojiDir); err != nil { 494 if !os.IsNotExist(err) { 495 return errors.Wrapf(err, "Error fetching file info of emoji directory %v", emojiDir) 496 } 497 498 if err = os.Mkdir(emojiDir, os.ModePerm); err != nil { 499 return errors.Wrapf(err, "Error creating emoji directory %v", emojiDir) 500 } 501 } 502 503 toPath, err := os.OpenFile(emojiDir+"/image", os.O_RDWR|os.O_CREATE, 0666) 504 if err != nil { 505 return errors.New("Error creating the image file " + err.Error()) 506 } 507 defer toPath.Close() 508 509 _, err = io.Copy(toPath, fromPath) 510 if err != nil { 511 return errors.New("Error copying emojis " + err.Error()) 512 } 513 514 return nil 515 } 516 517 func (a *App) ExportAllDirectChannels(writer io.Writer) *model.AppError { 518 afterId := strings.Repeat("0", 26) 519 for { 520 result := <-a.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, afterId) 521 if result.Err != nil { 522 return result.Err 523 } 524 525 channels := result.Data.([]*model.DirectChannelForExport) 526 if len(channels) == 0 { 527 break 528 } 529 530 for _, channel := range channels { 531 afterId = channel.Id 532 533 // Skip deleted. 534 if channel.DeleteAt != 0 { 535 continue 536 } 537 538 // There's no import support for single member channels yet. 539 if len(*channel.Members) == 1 { 540 mlog.Debug("Bulk export for direct channels containing a single member is not supported.") 541 continue 542 } 543 544 channelLine := ImportLineFromDirectChannel(channel) 545 if err := a.ExportWriteLine(writer, channelLine); err != nil { 546 return err 547 } 548 } 549 } 550 551 return nil 552 } 553 554 func (a *App) ExportAllDirectPosts(writer io.Writer) *model.AppError { 555 afterId := strings.Repeat("0", 26) 556 for { 557 result := <-a.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, afterId) 558 if result.Err != nil { 559 return result.Err 560 } 561 562 posts := result.Data.([]*model.DirectPostForExport) 563 if len(posts) == 0 { 564 break 565 } 566 567 for _, post := range posts { 568 afterId = post.Id 569 570 // Skip deleted. 571 if post.DeleteAt != 0 { 572 continue 573 } 574 575 // There's no import support for single member channels yet. 576 if len(*post.ChannelMembers) == 1 { 577 mlog.Debug("Bulk export for posts containing a single member is not supported.") 578 continue 579 } 580 581 // Do the Replies. 582 replies, err := a.buildPostReplies(post.Id) 583 if err != nil { 584 return err 585 } 586 587 postLine := ImportLineForDirectPost(post) 588 postLine.DirectPost.Replies = replies 589 if err := a.ExportWriteLine(writer, postLine); err != nil { 590 return err 591 } 592 } 593 } 594 return nil 595 }