github.com/haalcala/mattermost-server-change-repo/v5@v5.33.2/services/slackimport/slackimport.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See LICENSE.txt for license information. 3 4 package slackimport 5 6 import ( 7 "archive/zip" 8 "bytes" 9 "image" 10 "io" 11 "mime/multipart" 12 "net/http" 13 "path/filepath" 14 "regexp" 15 "sort" 16 "strings" 17 "time" 18 "unicode/utf8" 19 20 "github.com/mattermost/mattermost-server/v5/mlog" 21 "github.com/mattermost/mattermost-server/v5/model" 22 "github.com/mattermost/mattermost-server/v5/store" 23 "github.com/mattermost/mattermost-server/v5/utils" 24 ) 25 26 type slackChannel struct { 27 Id string `json:"id"` 28 Name string `json:"name"` 29 Creator string `json:"creator"` 30 Members []string `json:"members"` 31 Purpose slackChannelSub `json:"purpose"` 32 Topic slackChannelSub `json:"topic"` 33 Type string 34 } 35 36 type slackChannelSub struct { 37 Value string `json:"value"` 38 } 39 40 type slackProfile struct { 41 FirstName string `json:"first_name"` 42 LastName string `json:"last_name"` 43 Email string `json:"email"` 44 } 45 46 type slackUser struct { 47 Id string `json:"id"` 48 Username string `json:"name"` 49 Profile slackProfile `json:"profile"` 50 } 51 52 type slackFile struct { 53 Id string `json:"id"` 54 Title string `json:"title"` 55 } 56 57 type slackPost struct { 58 User string `json:"user"` 59 BotId string `json:"bot_id"` 60 BotUsername string `json:"username"` 61 Text string `json:"text"` 62 TimeStamp string `json:"ts"` 63 ThreadTS string `json:"thread_ts"` 64 Type string `json:"type"` 65 SubType string `json:"subtype"` 66 Comment *slackComment `json:"comment"` 67 Upload bool `json:"upload"` 68 File *slackFile `json:"file"` 69 Files []*slackFile `json:"files"` 70 Attachments []*model.SlackAttachment `json:"attachments"` 71 } 72 73 var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString 74 75 const slackImportMaxFileSize = 1024 * 1024 * 70 76 77 type slackComment struct { 78 User string `json:"user"` 79 Comment string `json:"comment"` 80 } 81 82 // Actions provides the actions that needs to be used for import slack data 83 type Actions struct { 84 UpdateActive func(*model.User, bool) (*model.User, *model.AppError) 85 AddUserToChannel func(*model.User, *model.Channel) (*model.ChannelMember, *model.AppError) 86 JoinUserToTeam func(*model.Team, *model.User, string) *model.AppError 87 CreateDirectChannel func(string, string) (*model.Channel, *model.AppError) 88 CreateGroupChannel func([]string) (*model.Channel, *model.AppError) 89 CreateChannel func(*model.Channel, bool) (*model.Channel, *model.AppError) 90 DoUploadFile func(time.Time, string, string, string, string, []byte) (*model.FileInfo, *model.AppError) 91 GenerateThumbnailImage func(image.Image, string) 92 GeneratePreviewImage func(image.Image, string) 93 InvalidateAllCaches func() 94 MaxPostSize func() int 95 PrepareImage func(fileData []byte) (image.Image, int, int) 96 } 97 98 // SlackImporter is a service that allows to import slack dumps into mattermost 99 type SlackImporter struct { 100 store store.Store 101 actions Actions 102 config *model.Config 103 } 104 105 // New creates a new SlackImporter service instance. It receive a store, a set of actions and the current config. 106 // It is expected to be used right away and discarded after that 107 func New(store store.Store, actions Actions, config *model.Config) *SlackImporter { 108 return &SlackImporter{ 109 store: store, 110 actions: actions, 111 config: config, 112 } 113 } 114 115 func (si *SlackImporter) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { 116 // Create log file 117 log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log")) 118 119 zipreader, err := zip.NewReader(fileData, fileSize) 120 if err != nil || zipreader.File == nil { 121 log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error")) 122 return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error(), http.StatusBadRequest), log 123 } 124 125 var channels []slackChannel 126 var publicChannels []slackChannel 127 var privateChannels []slackChannel 128 var groupChannels []slackChannel 129 var directChannels []slackChannel 130 131 var users []slackUser 132 posts := make(map[string][]slackPost) 133 uploads := make(map[string]*zip.File) 134 for _, file := range zipreader.File { 135 if file.UncompressedSize64 > slackImportMaxFileSize { 136 log.WriteString(utils.T("api.slackimport.slack_import.zip.file_too_large", map[string]interface{}{"Filename": file.Name})) 137 continue 138 } 139 reader, err := file.Open() 140 if err != nil { 141 log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name})) 142 return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error(), http.StatusInternalServerError), log 143 } 144 if file.Name == "channels.json" { 145 publicChannels, _ = slackParseChannels(reader, model.CHANNEL_OPEN) 146 channels = append(channels, publicChannels...) 147 } else if file.Name == "dms.json" { 148 directChannels, _ = slackParseChannels(reader, model.CHANNEL_DIRECT) 149 channels = append(channels, directChannels...) 150 } else if file.Name == "groups.json" { 151 privateChannels, _ = slackParseChannels(reader, model.CHANNEL_PRIVATE) 152 channels = append(channels, privateChannels...) 153 } else if file.Name == "mpims.json" { 154 groupChannels, _ = slackParseChannels(reader, model.CHANNEL_GROUP) 155 channels = append(channels, groupChannels...) 156 } else if file.Name == "users.json" { 157 users, _ = slackParseUsers(reader) 158 } else { 159 spl := strings.Split(file.Name, "/") 160 if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { 161 newposts, _ := slackParsePosts(reader) 162 channel := spl[0] 163 if _, ok := posts[channel]; !ok { 164 posts[channel] = newposts 165 } else { 166 posts[channel] = append(posts[channel], newposts...) 167 } 168 } else if len(spl) == 3 && spl[0] == "__uploads" { 169 uploads[spl[1]] = file 170 } 171 } 172 } 173 174 posts = slackConvertUserMentions(users, posts) 175 posts = slackConvertChannelMentions(channels, posts) 176 posts = slackConvertPostsMarkup(posts) 177 178 addedUsers := si.slackAddUsers(teamID, users, log) 179 botUser := si.slackAddBotUser(teamID, log) 180 181 si.slackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log) 182 183 if botUser != nil { 184 si.deactivateSlackBotUser(botUser) 185 } 186 187 si.actions.InvalidateAllCaches() 188 189 log.WriteString(utils.T("api.slackimport.slack_import.notes")) 190 log.WriteString("=======\r\n\r\n") 191 192 log.WriteString(utils.T("api.slackimport.slack_import.note1")) 193 log.WriteString(utils.T("api.slackimport.slack_import.note2")) 194 log.WriteString(utils.T("api.slackimport.slack_import.note3")) 195 196 return nil, log 197 } 198 199 func truncateRunes(s string, i int) string { 200 runes := []rune(s) 201 if len(runes) > i { 202 return string(runes[:i]) 203 } 204 return s 205 } 206 207 func (si *SlackImporter) slackAddUsers(teamId string, slackusers []slackUser, importerLog *bytes.Buffer) map[string]*model.User { 208 // Log header 209 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.created")) 210 importerLog.WriteString("===============\r\n\r\n") 211 212 addedUsers := make(map[string]*model.User) 213 214 // Need the team 215 team, err := si.store.Team().Get(teamId) 216 if err != nil { 217 importerLog.WriteString(utils.T("api.slackimport.slack_import.team_fail")) 218 return addedUsers 219 } 220 221 for _, sUser := range slackusers { 222 firstName := sUser.Profile.FirstName 223 lastName := sUser.Profile.LastName 224 email := sUser.Profile.Email 225 if email == "" { 226 email = sUser.Username + "@example.com" 227 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.missing_email_address", map[string]interface{}{"Email": email, "Username": sUser.Username})) 228 mlog.Warn("Slack Import: User does not have an email address in the Slack export. Used username as a placeholder. The user should update their email address once logged in to the system.", mlog.String("user_email", email), mlog.String("user_name", sUser.Username)) 229 } 230 231 password := model.NewId() 232 233 // Check for email conflict and use existing user if found 234 if existingUser, err := si.store.User().GetByEmail(email); err == nil { 235 addedUsers[sUser.Id] = existingUser 236 if err := si.actions.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil { 237 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) 238 } else { 239 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) 240 } 241 continue 242 } 243 244 email = strings.ToLower(email) 245 newUser := model.User{ 246 Username: sUser.Username, 247 FirstName: firstName, 248 LastName: lastName, 249 Email: email, 250 Password: password, 251 } 252 253 mUser := si.oldImportUser(team, &newUser) 254 if mUser == nil { 255 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username})) 256 continue 257 } 258 addedUsers[sUser.Id] = mUser 259 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password})) 260 } 261 262 return addedUsers 263 } 264 265 func (si *SlackImporter) slackAddBotUser(teamId string, log *bytes.Buffer) *model.User { 266 team, err := si.store.Team().Get(teamId) 267 if err != nil { 268 log.WriteString(utils.T("api.slackimport.slack_import.team_fail")) 269 return nil 270 } 271 272 password := model.NewId() 273 username := "slackimportuser_" + model.NewId() 274 email := username + "@localhost" 275 276 botUser := model.User{ 277 Username: username, 278 FirstName: "", 279 LastName: "", 280 Email: email, 281 Password: password, 282 } 283 284 mUser := si.oldImportUser(team, &botUser) 285 if mUser == nil { 286 log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username})) 287 return nil 288 } 289 290 log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password})) 291 return mUser 292 } 293 294 func (si *SlackImporter) slackAddPosts(teamId string, channel *model.Channel, posts []slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { 295 sort.Slice(posts, func(i, j int) bool { 296 return slackConvertTimeStamp(posts[i].TimeStamp) < slackConvertTimeStamp(posts[j].TimeStamp) 297 }) 298 threads := make(map[string]string) 299 for _, sPost := range posts { 300 switch { 301 case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): 302 if sPost.User == "" { 303 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 304 continue 305 } 306 if users[sPost.User] == nil { 307 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 308 continue 309 } 310 newPost := model.Post{ 311 UserId: users[sPost.User].Id, 312 ChannelId: channel.Id, 313 Message: sPost.Text, 314 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 315 } 316 if sPost.Upload { 317 if sPost.File != nil { 318 if fileInfo, ok := si.slackUploadFile(sPost.File, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { 319 newPost.FileIds = append(newPost.FileIds, fileInfo.Id) 320 } 321 } else if sPost.Files != nil { 322 for _, file := range sPost.Files { 323 if fileInfo, ok := si.slackUploadFile(file, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { 324 newPost.FileIds = append(newPost.FileIds, fileInfo.Id) 325 } 326 } 327 } 328 } 329 // If post in thread 330 if sPost.ThreadTS != "" && sPost.ThreadTS != sPost.TimeStamp { 331 newPost.RootId = threads[sPost.ThreadTS] 332 newPost.ParentId = threads[sPost.ThreadTS] 333 } 334 postId := si.oldImportPost(&newPost) 335 // If post is thread starter 336 if sPost.ThreadTS == sPost.TimeStamp { 337 threads[sPost.ThreadTS] = postId 338 } 339 case sPost.Type == "message" && sPost.SubType == "file_comment": 340 if sPost.Comment == nil { 341 mlog.Debug("Slack Import: Unable to import the message as it has no comments.") 342 continue 343 } 344 if sPost.Comment.User == "" { 345 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 346 continue 347 } 348 if users[sPost.Comment.User] == nil { 349 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 350 continue 351 } 352 newPost := model.Post{ 353 UserId: users[sPost.Comment.User].Id, 354 ChannelId: channel.Id, 355 Message: sPost.Comment.Comment, 356 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 357 } 358 si.oldImportPost(&newPost) 359 case sPost.Type == "message" && sPost.SubType == "bot_message": 360 if botUser == nil { 361 mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.") 362 continue 363 } 364 if sPost.BotId == "" { 365 mlog.Warn("Slack Import: Unable to import bot message as the BotId field is missing.") 366 continue 367 } 368 369 props := make(model.StringInterface) 370 props["override_username"] = sPost.BotUsername 371 if len(sPost.Attachments) > 0 { 372 props["attachments"] = sPost.Attachments 373 } 374 375 post := &model.Post{ 376 UserId: botUser.Id, 377 ChannelId: channel.Id, 378 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 379 Message: sPost.Text, 380 Type: model.POST_SLACK_ATTACHMENT, 381 } 382 383 postId := si.oldImportIncomingWebhookPost(post, props) 384 // If post is thread starter 385 if sPost.ThreadTS == sPost.TimeStamp { 386 threads[sPost.ThreadTS] = postId 387 } 388 case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): 389 if sPost.User == "" { 390 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 391 continue 392 } 393 if users[sPost.User] == nil { 394 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 395 continue 396 } 397 398 var postType string 399 if sPost.SubType == "channel_join" { 400 postType = model.POST_JOIN_CHANNEL 401 } else { 402 postType = model.POST_LEAVE_CHANNEL 403 } 404 405 newPost := model.Post{ 406 UserId: users[sPost.User].Id, 407 ChannelId: channel.Id, 408 Message: sPost.Text, 409 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 410 Type: postType, 411 Props: model.StringInterface{ 412 "username": users[sPost.User].Username, 413 }, 414 } 415 si.oldImportPost(&newPost) 416 case sPost.Type == "message" && sPost.SubType == "me_message": 417 if sPost.User == "" { 418 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 419 continue 420 } 421 if users[sPost.User] == nil { 422 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 423 continue 424 } 425 newPost := model.Post{ 426 UserId: users[sPost.User].Id, 427 ChannelId: channel.Id, 428 Message: "*" + sPost.Text + "*", 429 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 430 } 431 postId := si.oldImportPost(&newPost) 432 // If post is thread starter 433 if sPost.ThreadTS == sPost.TimeStamp { 434 threads[sPost.ThreadTS] = postId 435 } 436 case sPost.Type == "message" && sPost.SubType == "channel_topic": 437 if sPost.User == "" { 438 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 439 continue 440 } 441 if users[sPost.User] == nil { 442 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 443 continue 444 } 445 newPost := model.Post{ 446 UserId: users[sPost.User].Id, 447 ChannelId: channel.Id, 448 Message: sPost.Text, 449 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 450 Type: model.POST_HEADER_CHANGE, 451 } 452 si.oldImportPost(&newPost) 453 case sPost.Type == "message" && sPost.SubType == "channel_purpose": 454 if sPost.User == "" { 455 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 456 continue 457 } 458 if users[sPost.User] == nil { 459 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 460 continue 461 } 462 newPost := model.Post{ 463 UserId: users[sPost.User].Id, 464 ChannelId: channel.Id, 465 Message: sPost.Text, 466 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 467 Type: model.POST_PURPOSE_CHANGE, 468 } 469 si.oldImportPost(&newPost) 470 case sPost.Type == "message" && sPost.SubType == "channel_name": 471 if sPost.User == "" { 472 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 473 continue 474 } 475 if users[sPost.User] == nil { 476 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 477 continue 478 } 479 newPost := model.Post{ 480 UserId: users[sPost.User].Id, 481 ChannelId: channel.Id, 482 Message: sPost.Text, 483 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 484 Type: model.POST_DISPLAYNAME_CHANGE, 485 } 486 si.oldImportPost(&newPost) 487 default: 488 mlog.Warn( 489 "Slack Import: Unable to import the message as its type is not supported", 490 mlog.String("post_type", sPost.Type), 491 mlog.String("post_subtype", sPost.SubType), 492 ) 493 } 494 } 495 } 496 497 func (si *SlackImporter) slackUploadFile(slackPostFile *slackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) { 498 if slackPostFile == nil { 499 mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.") 500 return nil, false 501 } 502 file, ok := uploads[slackPostFile.Id] 503 if !ok { 504 mlog.Warn("Slack Import: Unable to import file as the file is missing from the Slack export zip file.", mlog.String("file_id", slackPostFile.Id)) 505 return nil, false 506 } 507 openFile, err := file.Open() 508 if err != nil { 509 mlog.Warn("Slack Import: Unable to open the file from the Slack export.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err)) 510 return nil, false 511 } 512 defer openFile.Close() 513 514 timestamp := utils.TimeFromMillis(slackConvertTimeStamp(slackTimestamp)) 515 uploadedFile, err := si.oldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) 516 if err != nil { 517 mlog.Warn("Slack Import: An error occurred when uploading file.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err)) 518 return nil, false 519 } 520 521 return uploadedFile, true 522 } 523 524 func (si *SlackImporter) deactivateSlackBotUser(user *model.User) { 525 if _, err := si.actions.UpdateActive(user, false); err != nil { 526 mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.") 527 } 528 } 529 530 func (si *SlackImporter) addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { 531 for _, member := range members { 532 user, ok := users[member] 533 if !ok { 534 log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"})) 535 continue 536 } 537 if _, err := si.actions.AddUserToChannel(user, channel); err != nil { 538 log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username})) 539 } 540 } 541 } 542 543 func slackSanitiseChannelProperties(channel model.Channel) model.Channel { 544 if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { 545 mlog.Warn("Slack Import: Channel display name exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 546 channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES) 547 } 548 549 if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH { 550 mlog.Warn("Slack Import: Channel handle exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 551 channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH] 552 } 553 554 if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES { 555 mlog.Warn("Slack Import: Channel purpose exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 556 channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES) 557 } 558 559 if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES { 560 mlog.Warn("Slack Import: Channel header exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 561 channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES) 562 } 563 564 return channel 565 } 566 567 func (si *SlackImporter) slackAddChannels(teamId string, slackchannels []slackChannel, posts map[string][]slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel { 568 // Write Header 569 importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.added")) 570 importerLog.WriteString("=================\r\n\r\n") 571 572 addedChannels := make(map[string]*model.Channel) 573 for _, sChannel := range slackchannels { 574 newChannel := model.Channel{ 575 TeamId: teamId, 576 Type: sChannel.Type, 577 DisplayName: sChannel.Name, 578 Name: slackConvertChannelName(sChannel.Name, sChannel.Id), 579 Purpose: sChannel.Purpose.Value, 580 Header: sChannel.Topic.Value, 581 } 582 583 // Direct message channels in Slack don't have a name so we set the id as name or else the messages won't get imported. 584 if newChannel.Type == model.CHANNEL_DIRECT { 585 sChannel.Name = sChannel.Id 586 } 587 588 newChannel = slackSanitiseChannelProperties(newChannel) 589 590 var mChannel *model.Channel 591 var err error 592 if mChannel, err = si.store.Channel().GetByName(teamId, sChannel.Name, true); err == nil { 593 // The channel already exists as an active channel. Merge with the existing one. 594 importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName})) 595 } else if _, nErr := si.store.Channel().GetDeletedByName(teamId, sChannel.Name); nErr == nil { 596 // The channel already exists but has been deleted. Generate a random string for the handle instead. 597 newChannel.Name = model.NewId() 598 newChannel = slackSanitiseChannelProperties(newChannel) 599 } 600 601 if mChannel == nil { 602 // Haven't found an existing channel to merge with. Try importing it as a new one. 603 mChannel = si.oldImportChannel(&newChannel, sChannel, users) 604 if mChannel == nil { 605 mlog.Warn("Slack Import: Unable to import Slack channel.", mlog.String("channel_display_name", newChannel.DisplayName)) 606 importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName})) 607 continue 608 } 609 } 610 611 // Members for direct and group channels are added during the creation of the channel in the oldImportChannel function 612 if sChannel.Type == model.CHANNEL_OPEN || sChannel.Type == model.CHANNEL_PRIVATE { 613 si.addSlackUsersToChannel(sChannel.Members, users, mChannel, importerLog) 614 } 615 importerLog.WriteString(newChannel.DisplayName + "\r\n") 616 addedChannels[sChannel.Id] = mChannel 617 si.slackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) 618 } 619 620 return addedChannels 621 } 622 623 // 624 // -- Old SlackImport Functions -- 625 // Import functions are suitable for entering posts and users into the database without 626 // some of the usual checks. (IsValid is still run) 627 // 628 629 func (si *SlackImporter) oldImportPost(post *model.Post) string { 630 // Workaround for empty messages, which may be the case if they are webhook posts. 631 firstIteration := true 632 firstPostId := "" 633 if post.ParentId != "" { 634 firstPostId = post.ParentId 635 } 636 maxPostSize := si.actions.MaxPostSize() 637 for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { 638 var remainder string 639 if messageRuneCount > maxPostSize { 640 remainder = string(([]rune(post.Message))[maxPostSize:]) 641 post.Message = truncateRunes(post.Message, maxPostSize) 642 } else { 643 remainder = "" 644 } 645 646 post.Hashtags, _ = model.ParseHashtags(post.Message) 647 648 post.RootId = firstPostId 649 post.ParentId = firstPostId 650 651 _, err := si.store.Post().Save(post) 652 if err != nil { 653 mlog.Debug("Error saving post.", mlog.String("user_id", post.UserId), mlog.String("message", post.Message)) 654 } 655 656 if firstIteration { 657 if firstPostId == "" { 658 firstPostId = post.Id 659 } 660 for _, fileId := range post.FileIds { 661 if err := si.store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); err != nil { 662 mlog.Error( 663 "Error attaching files to post.", 664 mlog.String("post_id", post.Id), 665 mlog.String("file_ids", strings.Join(post.FileIds, ",")), 666 mlog.String("user_id", post.UserId), 667 mlog.Err(err), 668 ) 669 } 670 } 671 post.FileIds = nil 672 } 673 674 post.Id = "" 675 post.CreateAt++ 676 post.Message = remainder 677 firstIteration = false 678 } 679 return firstPostId 680 } 681 682 func (si *SlackImporter) oldImportUser(team *model.Team, user *model.User) *model.User { 683 user.MakeNonNil() 684 685 user.Roles = model.SYSTEM_USER_ROLE_ID 686 687 ruser, nErr := si.store.User().Save(user) 688 if nErr != nil { 689 mlog.Debug("Error saving user.", mlog.Err(nErr)) 690 return nil 691 } 692 693 if _, err := si.store.User().VerifyEmail(ruser.Id, ruser.Email); err != nil { 694 mlog.Warn("Failed to set email verified.", mlog.Err(err)) 695 } 696 697 if err := si.actions.JoinUserToTeam(team, user, ""); err != nil { 698 mlog.Warn("Failed to join team when importing.", mlog.Err(err)) 699 } 700 701 return ruser 702 } 703 704 func (si *SlackImporter) oldImportChannel(channel *model.Channel, sChannel slackChannel, users map[string]*model.User) *model.Channel { 705 switch { 706 case channel.Type == model.CHANNEL_DIRECT: 707 if len(sChannel.Members) < 2 { 708 return nil 709 } 710 u1 := users[sChannel.Members[0]] 711 u2 := users[sChannel.Members[1]] 712 if u1 == nil || u2 == nil { 713 mlog.Warn("Either or both of user ids not found in users.json. Ignoring.", mlog.String("id1", sChannel.Members[0]), mlog.String("id2", sChannel.Members[1])) 714 return nil 715 } 716 sc, err := si.actions.CreateDirectChannel(u1.Id, u2.Id) 717 if err != nil { 718 return nil 719 } 720 721 return sc 722 // check if direct channel has less than 8 members and if not import as private channel instead 723 case channel.Type == model.CHANNEL_GROUP && len(sChannel.Members) < 8: 724 members := make([]string, len(sChannel.Members)) 725 726 for i := range sChannel.Members { 727 u := users[sChannel.Members[i]] 728 if u == nil { 729 mlog.Warn("User not found in users.json. Ignoring.", mlog.String("id", sChannel.Members[i])) 730 continue 731 } 732 members[i] = u.Id 733 } 734 735 creator := users[sChannel.Creator] 736 if creator == nil { 737 return nil 738 } 739 sc, err := si.actions.CreateGroupChannel(members) 740 if err != nil { 741 return nil 742 } 743 744 return sc 745 case channel.Type == model.CHANNEL_GROUP: 746 channel.Type = model.CHANNEL_PRIVATE 747 sc, err := si.actions.CreateChannel(channel, false) 748 if err != nil { 749 return nil 750 } 751 752 return sc 753 } 754 755 sc, err := si.store.Channel().Save(channel, *si.config.TeamSettings.MaxChannelsPerTeam) 756 if err != nil { 757 return nil 758 } 759 760 return sc 761 } 762 763 func (si *SlackImporter) oldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { 764 buf := bytes.NewBuffer(nil) 765 io.Copy(buf, file) 766 data := buf.Bytes() 767 768 fileInfo, err := si.actions.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) 769 if err != nil { 770 return nil, err 771 } 772 773 if fileInfo.IsImage() && fileInfo.MimeType != "image/svg+xml" { 774 img, _, _ := si.actions.PrepareImage(data) 775 if img != nil { 776 si.actions.GenerateThumbnailImage(img, fileInfo.ThumbnailPath) 777 si.actions.GeneratePreviewImage(img, fileInfo.PreviewPath) 778 } 779 } 780 781 return fileInfo, nil 782 } 783 784 func (si *SlackImporter) oldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string { 785 linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) 786 post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") 787 788 post.AddProp("from_webhook", "true") 789 790 if _, ok := props["override_username"]; !ok { 791 post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME) 792 } 793 794 if len(props) > 0 { 795 for key, val := range props { 796 if key == "attachments" { 797 if attachments, success := val.([]*model.SlackAttachment); success { 798 model.ParseSlackAttachment(post, attachments) 799 } 800 } else if key != "from_webhook" { 801 post.AddProp(key, val) 802 } 803 } 804 } 805 806 return si.oldImportPost(post) 807 }