github.com/xzl8028/xenia-server@v0.0.0-20190809101854-18450a97da63/app/export.go (about) 1 // Copyright (c) 2015-present Xenia, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "encoding/json" 8 "io" 9 "net/http" 10 "os" 11 "path/filepath" 12 "strings" 13 14 "github.com/xzl8028/xenia-server/mlog" 15 "github.com/xzl8028/xenia-server/model" 16 "github.com/pkg/errors" 17 ) 18 19 // We use this map to identify the exportable preferences. 20 // Here we link the preference category and name, to the name of the relevant filed in the import struct. 21 var exportablePreferences = map[ComparablePreference]string{{ 22 Category: model.PREFERENCE_CATEGORY_THEME, 23 Name: "", 24 }: "Theme", { 25 Category: model.PREFERENCE_CATEGORY_ADVANCED_SETTINGS, 26 Name: "feature_enabled_markdown_preview", 27 }: "UseMarkdownPreview", { 28 Category: model.PREFERENCE_CATEGORY_ADVANCED_SETTINGS, 29 Name: "formatting", 30 }: "UseFormatting", { 31 Category: model.PREFERENCE_CATEGORY_SIDEBAR_SETTINGS, 32 Name: "show_unread_section", 33 }: "ShowUnreadSection", { 34 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 35 Name: model.PREFERENCE_NAME_USE_MILITARY_TIME, 36 }: "UseMilitaryTime", { 37 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 38 Name: model.PREFERENCE_NAME_COLLAPSE_SETTING, 39 }: "CollapsePreviews", { 40 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 41 Name: model.PREFERENCE_NAME_MESSAGE_DISPLAY, 42 }: "MessageDisplay", { 43 Category: model.PREFERENCE_CATEGORY_DISPLAY_SETTINGS, 44 Name: "channel_display_mode", 45 }: "ChannelDisplayMode", { 46 Category: model.PREFERENCE_CATEGORY_TUTORIAL_STEPS, 47 Name: "", 48 }: "TutorialStep", { 49 Category: model.PREFERENCE_CATEGORY_NOTIFICATIONS, 50 Name: model.PREFERENCE_NAME_EMAIL_INTERVAL, 51 }: "EmailInterval", 52 } 53 54 func (a *App) BulkExport(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { 55 if err := a.ExportVersion(writer); err != nil { 56 return err 57 } 58 59 if err := a.ExportAllTeams(writer); err != nil { 60 return err 61 } 62 63 if err := a.ExportAllChannels(writer); err != nil { 64 return err 65 } 66 67 if err := a.ExportAllUsers(writer); err != nil { 68 return err 69 } 70 71 if err := a.ExportAllPosts(writer); err != nil { 72 return err 73 } 74 75 if err := a.ExportCustomEmoji(writer, file, pathToEmojiDir, dirNameToExportEmoji); err != nil { 76 return err 77 } 78 79 if err := a.ExportAllDirectChannels(writer); err != nil { 80 return err 81 } 82 83 if err := a.ExportAllDirectPosts(writer); err != nil { 84 return err 85 } 86 87 return nil 88 } 89 90 func (a *App) ExportWriteLine(writer io.Writer, line *LineImportData) *model.AppError { 91 b, err := json.Marshal(line) 92 if err != nil { 93 return model.NewAppError("BulkExport", "app.export.export_write_line.json_marshall.error", nil, "err="+err.Error(), http.StatusBadRequest) 94 } 95 96 if _, err := writer.Write(append(b, '\n')); err != nil { 97 return model.NewAppError("BulkExport", "app.export.export_write_line.io_writer.error", nil, "err="+err.Error(), http.StatusBadRequest) 98 } 99 100 return nil 101 } 102 103 func (a *App) ExportVersion(writer io.Writer) *model.AppError { 104 version := 1 105 versionLine := &LineImportData{ 106 Type: "version", 107 Version: &version, 108 } 109 110 return a.ExportWriteLine(writer, versionLine) 111 } 112 113 func (a *App) ExportAllTeams(writer io.Writer) *model.AppError { 114 afterId := strings.Repeat("0", 26) 115 for { 116 result := <-a.Srv.Store.Team().GetAllForExportAfter(1000, afterId) 117 118 if result.Err != nil { 119 return result.Err 120 } 121 122 teams := result.Data.([]*model.TeamForExport) 123 124 if len(teams) == 0 { 125 break 126 } 127 128 for _, team := range teams { 129 afterId = team.Id 130 131 // Skip deleted. 132 if team.DeleteAt != 0 { 133 continue 134 } 135 136 teamLine := ImportLineFromTeam(team) 137 if err := a.ExportWriteLine(writer, teamLine); err != nil { 138 return err 139 } 140 } 141 } 142 143 return nil 144 } 145 146 func (a *App) ExportAllChannels(writer io.Writer) *model.AppError { 147 afterId := strings.Repeat("0", 26) 148 for { 149 channels, err := a.Srv.Store.Channel().GetAllChannelsForExportAfter(1000, afterId) 150 151 if err != nil { 152 return err 153 } 154 155 if len(channels) == 0 { 156 break 157 } 158 159 for _, channel := range channels { 160 afterId = channel.Id 161 162 // Skip deleted. 163 if channel.DeleteAt != 0 { 164 continue 165 } 166 167 channelLine := ImportLineFromChannel(channel) 168 if err := a.ExportWriteLine(writer, channelLine); err != nil { 169 return err 170 } 171 } 172 } 173 174 return nil 175 } 176 177 func (a *App) ExportAllUsers(writer io.Writer) *model.AppError { 178 afterId := strings.Repeat("0", 26) 179 for { 180 users, err := a.Srv.Store.User().GetAllAfter(1000, afterId) 181 182 if err != nil { 183 return err 184 } 185 186 if len(users) == 0 { 187 break 188 } 189 190 for _, user := range users { 191 afterId = user.Id 192 193 // Gathering here the exportable preferences to pass them on to ImportLineFromUser 194 exportedPrefs := make(map[string]*string) 195 allPrefs, err := a.GetPreferencesForUser(user.Id) 196 if err != nil { 197 return err 198 } 199 for _, pref := range allPrefs { 200 // We need to manage the special cases 201 // Here we manage Tutorial steps 202 if pref.Category == model.PREFERENCE_CATEGORY_TUTORIAL_STEPS { 203 pref.Name = "" 204 // Then the email interval 205 } else if pref.Category == model.PREFERENCE_CATEGORY_NOTIFICATIONS && pref.Name == model.PREFERENCE_NAME_EMAIL_INTERVAL { 206 switch pref.Value { 207 case model.PREFERENCE_EMAIL_INTERVAL_NO_BATCHING_SECONDS: 208 pref.Value = model.PREFERENCE_EMAIL_INTERVAL_IMMEDIATELY 209 case model.PREFERENCE_EMAIL_INTERVAL_FIFTEEN_AS_SECONDS: 210 pref.Value = model.PREFERENCE_EMAIL_INTERVAL_FIFTEEN 211 case model.PREFERENCE_EMAIL_INTERVAL_HOUR_AS_SECONDS: 212 pref.Value = model.PREFERENCE_EMAIL_INTERVAL_HOUR 213 case "0": 214 pref.Value = "" 215 } 216 } 217 id, ok := exportablePreferences[ComparablePreference{ 218 Category: pref.Category, 219 Name: pref.Name, 220 }] 221 if ok { 222 prefPtr := pref.Value 223 if prefPtr != "" { 224 exportedPrefs[id] = &prefPtr 225 } else { 226 exportedPrefs[id] = nil 227 } 228 } 229 } 230 231 userLine := ImportLineFromUser(user, exportedPrefs) 232 233 userLine.User.NotifyProps = a.buildUserNotifyProps(user.NotifyProps) 234 235 // Do the Team Memberships. 236 members, err := a.buildUserTeamAndChannelMemberships(user.Id) 237 if err != nil { 238 return err 239 } 240 241 userLine.User.Teams = members 242 243 if err := a.ExportWriteLine(writer, userLine); err != nil { 244 return err 245 } 246 } 247 } 248 249 return nil 250 } 251 252 func (a *App) buildUserTeamAndChannelMemberships(userId string) (*[]UserTeamImportData, *model.AppError) { 253 var memberships []UserTeamImportData 254 255 result := <-a.Srv.Store.Team().GetTeamMembersForExport(userId) 256 257 if result.Err != nil { 258 return nil, result.Err 259 } 260 261 members := result.Data.([]*model.TeamMemberForExport) 262 263 for _, member := range members { 264 // Skip deleted. 265 if member.DeleteAt != 0 { 266 continue 267 } 268 269 memberData := ImportUserTeamDataFromTeamMember(member) 270 271 // Do the Channel Memberships. 272 channelMembers, err := a.buildUserChannelMemberships(userId, member.TeamId) 273 if err != nil { 274 return nil, err 275 } 276 277 memberData.Channels = channelMembers 278 279 memberships = append(memberships, *memberData) 280 } 281 282 return &memberships, nil 283 } 284 285 func (a *App) buildUserChannelMemberships(userId string, teamId string) (*[]UserChannelImportData, *model.AppError) { 286 var memberships []UserChannelImportData 287 288 members, err := a.Srv.Store.Channel().GetChannelMembersForExport(userId, teamId) 289 if err != nil { 290 return nil, err 291 } 292 293 category := model.PREFERENCE_CATEGORY_FAVORITE_CHANNEL 294 preferences, err := a.GetPreferenceByCategoryForUser(userId, category) 295 if err != nil && err.StatusCode != http.StatusNotFound { 296 return nil, err 297 } 298 299 for _, member := range members { 300 memberships = append(memberships, *ImportUserChannelDataFromChannelMemberAndPreferences(member, &preferences)) 301 } 302 return &memberships, nil 303 } 304 305 func (a *App) buildUserNotifyProps(notifyProps model.StringMap) *UserNotifyPropsImportData { 306 307 getProp := func(key string) *string { 308 if v, ok := notifyProps[key]; ok { 309 return &v 310 } 311 return nil 312 } 313 314 return &UserNotifyPropsImportData{ 315 Desktop: getProp(model.DESKTOP_NOTIFY_PROP), 316 DesktopSound: getProp(model.DESKTOP_SOUND_NOTIFY_PROP), 317 Email: getProp(model.EMAIL_NOTIFY_PROP), 318 Mobile: getProp(model.PUSH_NOTIFY_PROP), 319 MobilePushStatus: getProp(model.PUSH_STATUS_NOTIFY_PROP), 320 ChannelTrigger: getProp(model.CHANNEL_MENTIONS_NOTIFY_PROP), 321 CommentsTrigger: getProp(model.COMMENTS_NOTIFY_PROP), 322 MentionKeys: getProp(model.MENTION_KEYS_NOTIFY_PROP), 323 } 324 } 325 326 func (a *App) ExportAllPosts(writer io.Writer) *model.AppError { 327 afterId := strings.Repeat("0", 26) 328 for { 329 posts, err := a.Srv.Store.Post().GetParentsForExportAfter(1000, afterId) 330 331 if err != nil { 332 return err 333 } 334 335 if len(posts) == 0 { 336 break 337 } 338 339 for _, post := range posts { 340 afterId = post.Id 341 342 // Skip deleted. 343 if post.DeleteAt != 0 { 344 continue 345 } 346 347 postLine := ImportLineForPost(post) 348 349 // Do the Replies. 350 replies, err := a.buildPostReplies(post.Id) 351 if err != nil { 352 return err 353 } 354 355 reactions, err := a.BuildPostReactions(post.Id) 356 if err != nil { 357 return err 358 } 359 360 postLine.Post.Replies = replies 361 362 postLine.Post.Reactions = reactions 363 364 if err := a.ExportWriteLine(writer, postLine); err != nil { 365 return err 366 } 367 } 368 } 369 370 return nil 371 } 372 373 func (a *App) buildPostReplies(postId string) (*[]ReplyImportData, *model.AppError) { 374 var replies []ReplyImportData 375 376 replyPosts, err := a.Srv.Store.Post().GetRepliesForExport(postId) 377 378 if err != nil { 379 return nil, err 380 } 381 382 for _, reply := range replyPosts { 383 replyImportObject := ImportReplyFromPost(reply) 384 if reply.HasReactions == true { 385 reactionsOfReply, err := a.BuildPostReactions(reply.Id) 386 if err != nil { 387 return nil, err 388 } 389 replyImportObject.Reactions = reactionsOfReply 390 } 391 replies = append(replies, *replyImportObject) 392 } 393 394 return &replies, nil 395 } 396 397 func (a *App) BuildPostReactions(postId string) (*[]ReactionImportData, *model.AppError) { 398 var reactionsOfPost []ReactionImportData 399 400 reactions, err := a.Srv.Store.Reaction().GetForPost(postId, true) 401 if err != nil { 402 return nil, err 403 } 404 405 for _, reaction := range reactions { 406 var user *model.User 407 user, err = a.Srv.Store.User().Get(reaction.UserId) 408 if err != nil { 409 return nil, err 410 } 411 reactionsOfPost = append(reactionsOfPost, *ImportReactionFromPost(user, reaction)) 412 } 413 414 return &reactionsOfPost, nil 415 416 } 417 418 func (a *App) ExportCustomEmoji(writer io.Writer, file string, pathToEmojiDir string, dirNameToExportEmoji string) *model.AppError { 419 pageNumber := 0 420 for { 421 customEmojiList, err := a.GetEmojiList(pageNumber, 100, model.EMOJI_SORT_BY_NAME) 422 423 if err != nil { 424 return err 425 } 426 427 if len(customEmojiList) == 0 { 428 break 429 } 430 431 pageNumber++ 432 433 pathToDir := a.createDirForEmoji(file, dirNameToExportEmoji) 434 435 for _, emoji := range customEmojiList { 436 emojiImagePath := pathToEmojiDir + emoji.Id + "/image" 437 err := a.copyEmojiImages(emoji.Id, emojiImagePath, pathToDir) 438 if err != nil { 439 return model.NewAppError("BulkExport", "app.export.export_custom_emoji.copy_emoji_images.error", nil, "err="+err.Error(), http.StatusBadRequest) 440 } 441 442 filePath := dirNameToExportEmoji + "/" + emoji.Id + "/image" 443 444 emojiImportObject := ImportLineFromEmoji(emoji, filePath) 445 446 if err := a.ExportWriteLine(writer, emojiImportObject); err != nil { 447 return err 448 } 449 } 450 } 451 452 return nil 453 } 454 455 // Creates directory named 'exported_emoji' to copy the emoji files 456 // Directory and the file specified by admin share the same path 457 func (a *App) createDirForEmoji(file string, dirName string) string { 458 pathToFile, _ := filepath.Abs(file) 459 pathSlice := strings.Split(pathToFile, "/") 460 if len(pathSlice) > 0 { 461 pathSlice = pathSlice[:len(pathSlice)-1] 462 } 463 pathToDir := strings.Join(pathSlice, "/") + "/" + dirName 464 465 if _, err := os.Stat(pathToDir); os.IsNotExist(err) { 466 os.Mkdir(pathToDir, os.ModePerm) 467 } 468 return pathToDir 469 } 470 471 // Copies emoji files from 'data/emoji' dir to 'exported_emoji' dir 472 func (a *App) copyEmojiImages(emojiId string, emojiImagePath string, pathToDir string) error { 473 fromPath, err := os.Open(emojiImagePath) 474 if fromPath == nil || err != nil { 475 return errors.New("Error reading " + emojiImagePath + "file") 476 } 477 defer fromPath.Close() 478 479 emojiDir := pathToDir + "/" + emojiId 480 481 if _, err = os.Stat(emojiDir); err != nil { 482 if !os.IsNotExist(err) { 483 return errors.Wrapf(err, "Error fetching file info of emoji directory %v", emojiDir) 484 } 485 486 if err = os.Mkdir(emojiDir, os.ModePerm); err != nil { 487 return errors.Wrapf(err, "Error creating emoji directory %v", emojiDir) 488 } 489 } 490 491 toPath, err := os.OpenFile(emojiDir+"/image", os.O_RDWR|os.O_CREATE, 0666) 492 if err != nil { 493 return errors.New("Error creating the image file " + err.Error()) 494 } 495 defer toPath.Close() 496 497 _, err = io.Copy(toPath, fromPath) 498 if err != nil { 499 return errors.New("Error copying emojis " + err.Error()) 500 } 501 502 return nil 503 } 504 505 func (a *App) ExportAllDirectChannels(writer io.Writer) *model.AppError { 506 afterId := strings.Repeat("0", 26) 507 for { 508 channels, err := a.Srv.Store.Channel().GetAllDirectChannelsForExportAfter(1000, afterId) 509 if err != nil { 510 return err 511 } 512 513 if len(channels) == 0 { 514 break 515 } 516 517 for _, channel := range channels { 518 afterId = channel.Id 519 520 // Skip deleted. 521 if channel.DeleteAt != 0 { 522 continue 523 } 524 525 // There's no import support for single member channels yet. 526 if len(*channel.Members) == 1 { 527 mlog.Debug("Bulk export for direct channels containing a single member is not supported.") 528 continue 529 } 530 531 channelLine := ImportLineFromDirectChannel(channel) 532 if err := a.ExportWriteLine(writer, channelLine); err != nil { 533 return err 534 } 535 } 536 } 537 538 return nil 539 } 540 541 func (a *App) ExportAllDirectPosts(writer io.Writer) *model.AppError { 542 afterId := strings.Repeat("0", 26) 543 for { 544 posts, err := a.Srv.Store.Post().GetDirectPostParentsForExportAfter(1000, afterId) 545 if err != nil { 546 return err 547 } 548 549 if len(posts) == 0 { 550 break 551 } 552 553 for _, post := range posts { 554 afterId = post.Id 555 556 // Skip deleted. 557 if post.DeleteAt != 0 { 558 continue 559 } 560 561 // There's no import support for single member channels yet. 562 if len(*post.ChannelMembers) == 1 { 563 mlog.Debug("Bulk export for posts containing a single member is not supported.") 564 continue 565 } 566 567 // Do the Replies. 568 replies, err := a.buildPostReplies(post.Id) 569 if err != nil { 570 return err 571 } 572 573 postLine := ImportLineForDirectPost(post) 574 postLine.DirectPost.Replies = replies 575 if err := a.ExportWriteLine(writer, postLine); err != nil { 576 return err 577 } 578 } 579 } 580 return nil 581 }