github.com/mattermost/mattermost-server/server/v8@v8.0.0-20230610055354-a6d1d38b273d/platform/services/slackimport/slackimport.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See LICENSE.txt for license information. 3 4 package slackimport 5 6 import ( 7 "archive/zip" 8 "bytes" 9 "errors" 10 "image" 11 "io" 12 "mime/multipart" 13 "net/http" 14 "path/filepath" 15 "regexp" 16 "sort" 17 "strings" 18 "time" 19 "unicode/utf8" 20 21 "github.com/mattermost/mattermost-server/server/public/model" 22 "github.com/mattermost/mattermost-server/server/public/shared/i18n" 23 "github.com/mattermost/mattermost-server/server/public/shared/mlog" 24 "github.com/mattermost/mattermost-server/server/v8/channels/app/request" 25 "github.com/mattermost/mattermost-server/server/v8/channels/store" 26 "github.com/mattermost/mattermost-server/server/v8/channels/utils" 27 ) 28 29 type slackChannel struct { 30 Id string `json:"id"` 31 Name string `json:"name"` 32 Creator string `json:"creator"` 33 Members []string `json:"members"` 34 Purpose slackChannelSub `json:"purpose"` 35 Topic slackChannelSub `json:"topic"` 36 Type model.ChannelType 37 } 38 39 type slackChannelSub struct { 40 Value string `json:"value"` 41 } 42 43 type slackProfile struct { 44 FirstName string `json:"first_name"` 45 LastName string `json:"last_name"` 46 Email string `json:"email"` 47 } 48 49 type slackUser struct { 50 Id string `json:"id"` 51 Username string `json:"name"` 52 Profile slackProfile `json:"profile"` 53 } 54 55 type slackFile struct { 56 Id string `json:"id"` 57 Title string `json:"title"` 58 } 59 60 type slackPost struct { 61 User string `json:"user"` 62 BotId string `json:"bot_id"` 63 BotUsername string `json:"username"` 64 Text string `json:"text"` 65 TimeStamp string `json:"ts"` 66 ThreadTS string `json:"thread_ts"` 67 Type string `json:"type"` 68 SubType string `json:"subtype"` 69 Comment *slackComment `json:"comment"` 70 Upload bool `json:"upload"` 71 File *slackFile `json:"file"` 72 Files []*slackFile `json:"files"` 73 Attachments []*model.SlackAttachment `json:"attachments"` 74 } 75 76 var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString 77 78 const slackImportMaxFileSize = 1024 * 1024 * 70 79 80 type slackComment struct { 81 User string `json:"user"` 82 Comment string `json:"comment"` 83 } 84 85 // Actions provides the actions that needs to be used for import slack data 86 type Actions struct { 87 UpdateActive func(*model.User, bool) (*model.User, *model.AppError) 88 AddUserToChannel func(request.CTX, *model.User, *model.Channel, bool) (*model.ChannelMember, *model.AppError) 89 JoinUserToTeam func(*model.Team, *model.User, string) (*model.TeamMember, *model.AppError) 90 CreateDirectChannel func(request.CTX, string, string, ...model.ChannelOption) (*model.Channel, *model.AppError) 91 CreateGroupChannel func(request.CTX, []string) (*model.Channel, *model.AppError) 92 CreateChannel func(*model.Channel, bool) (*model.Channel, *model.AppError) 93 DoUploadFile func(time.Time, string, string, string, string, []byte) (*model.FileInfo, *model.AppError) 94 GenerateThumbnailImage func(image.Image, string, string) 95 GeneratePreviewImage func(image.Image, string, string) 96 InvalidateAllCaches func() 97 MaxPostSize func() int 98 PrepareImage func(fileData []byte) (image.Image, string, func(), error) 99 } 100 101 // SlackImporter is a service that allows to import slack dumps into mattermost 102 type SlackImporter struct { 103 store store.Store 104 actions Actions 105 config *model.Config 106 } 107 108 // New creates a new SlackImporter service instance. It receive a store, a set of actions and the current config. 109 // It is expected to be used right away and discarded after that 110 func New(store store.Store, actions Actions, config *model.Config) *SlackImporter { 111 return &SlackImporter{ 112 store: store, 113 actions: actions, 114 config: config, 115 } 116 } 117 118 func (si *SlackImporter) SlackImport(c request.CTX, fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { 119 // Create log file 120 log := bytes.NewBufferString(i18n.T("api.slackimport.slack_import.log")) 121 122 zipreader, err := zip.NewReader(fileData, fileSize) 123 if err != nil || zipreader.File == nil { 124 log.WriteString(i18n.T("api.slackimport.slack_import.zip.app_error")) 125 return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, "", http.StatusBadRequest).Wrap(err), log 126 } 127 128 var channels []slackChannel 129 var publicChannels []slackChannel 130 var privateChannels []slackChannel 131 var groupChannels []slackChannel 132 var directChannels []slackChannel 133 134 var users []slackUser 135 posts := make(map[string][]slackPost) 136 uploads := make(map[string]*zip.File) 137 for _, file := range zipreader.File { 138 fileReader, err := file.Open() 139 if err != nil { 140 log.WriteString(i18n.T("api.slackimport.slack_import.open.app_error", map[string]any{"Filename": file.Name})) 141 return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]any{"Filename": file.Name}, "", http.StatusInternalServerError).Wrap(err), log 142 } 143 reader := utils.NewLimitedReaderWithError(fileReader, slackImportMaxFileSize) 144 if file.Name == "channels.json" { 145 publicChannels, err = slackParseChannels(reader, model.ChannelTypeOpen) 146 if errors.Is(err, utils.SizeLimitExceeded) { 147 log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name})) 148 continue 149 } 150 channels = append(channels, publicChannels...) 151 } else if file.Name == "dms.json" { 152 directChannels, err = slackParseChannels(reader, model.ChannelTypeDirect) 153 if errors.Is(err, utils.SizeLimitExceeded) { 154 log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name})) 155 continue 156 } 157 channels = append(channels, directChannels...) 158 } else if file.Name == "groups.json" { 159 privateChannels, err = slackParseChannels(reader, model.ChannelTypePrivate) 160 if errors.Is(err, utils.SizeLimitExceeded) { 161 log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name})) 162 continue 163 } 164 channels = append(channels, privateChannels...) 165 } else if file.Name == "mpims.json" { 166 groupChannels, err = slackParseChannels(reader, model.ChannelTypeGroup) 167 if errors.Is(err, utils.SizeLimitExceeded) { 168 log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name})) 169 continue 170 } 171 channels = append(channels, groupChannels...) 172 } else if file.Name == "users.json" { 173 users, err = slackParseUsers(reader) 174 if errors.Is(err, utils.SizeLimitExceeded) { 175 log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name})) 176 continue 177 } 178 } else { 179 spl := strings.Split(file.Name, "/") 180 if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { 181 newposts, err := slackParsePosts(reader) 182 if errors.Is(err, utils.SizeLimitExceeded) { 183 log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]any{"Filename": file.Name})) 184 continue 185 } 186 channel := spl[0] 187 if _, ok := posts[channel]; !ok { 188 posts[channel] = newposts 189 } else { 190 posts[channel] = append(posts[channel], newposts...) 191 } 192 } else if len(spl) == 3 && spl[0] == "__uploads" { 193 uploads[spl[1]] = file 194 } 195 } 196 } 197 198 posts = slackConvertUserMentions(users, posts) 199 posts = slackConvertChannelMentions(channels, posts) 200 posts = slackConvertPostsMarkup(posts) 201 202 addedUsers := si.slackAddUsers(teamID, users, log) 203 botUser := si.slackAddBotUser(teamID, log) 204 205 si.slackAddChannels(c, teamID, channels, posts, addedUsers, uploads, botUser, log) 206 207 if botUser != nil { 208 si.deactivateSlackBotUser(botUser) 209 } 210 211 si.actions.InvalidateAllCaches() 212 213 log.WriteString(i18n.T("api.slackimport.slack_import.notes")) 214 log.WriteString("=======\r\n\r\n") 215 216 log.WriteString(i18n.T("api.slackimport.slack_import.note1")) 217 log.WriteString(i18n.T("api.slackimport.slack_import.note2")) 218 log.WriteString(i18n.T("api.slackimport.slack_import.note3")) 219 220 return nil, log 221 } 222 223 func truncateRunes(s string, i int) string { 224 runes := []rune(s) 225 if len(runes) > i { 226 return string(runes[:i]) 227 } 228 return s 229 } 230 231 func (si *SlackImporter) slackAddUsers(teamId string, slackusers []slackUser, importerLog *bytes.Buffer) map[string]*model.User { 232 // Log header 233 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.created")) 234 importerLog.WriteString("===============\r\n\r\n") 235 236 addedUsers := make(map[string]*model.User) 237 238 // Need the team 239 team, err := si.store.Team().Get(teamId) 240 if err != nil { 241 importerLog.WriteString(i18n.T("api.slackimport.slack_import.team_fail")) 242 return addedUsers 243 } 244 245 for _, sUser := range slackusers { 246 firstName := sUser.Profile.FirstName 247 lastName := sUser.Profile.LastName 248 email := sUser.Profile.Email 249 if email == "" { 250 email = sUser.Username + "@example.com" 251 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.missing_email_address", map[string]any{"Email": email, "Username": sUser.Username})) 252 mlog.Warn("Slack Import: User does not have an email address in the Slack export. Used username as a placeholder. The user should update their email address once logged in to the system.", mlog.String("user_email", email), mlog.String("user_name", sUser.Username)) 253 } 254 255 password := model.NewId() 256 257 // Check for email conflict and use existing user if found 258 if existingUser, err := si.store.User().GetByEmail(email); err == nil { 259 addedUsers[sUser.Id] = existingUser 260 if _, err := si.actions.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil { 261 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]any{"Email": existingUser.Email, "Username": existingUser.Username})) 262 } else { 263 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing", map[string]any{"Email": existingUser.Email, "Username": existingUser.Username})) 264 } 265 continue 266 } 267 268 email = strings.ToLower(email) 269 newUser := model.User{ 270 Username: sUser.Username, 271 FirstName: firstName, 272 LastName: lastName, 273 Email: email, 274 Password: password, 275 } 276 277 mUser := si.oldImportUser(team, &newUser) 278 if mUser == nil { 279 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.unable_import", map[string]any{"Username": sUser.Username})) 280 continue 281 } 282 addedUsers[sUser.Id] = mUser 283 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.email_pwd", map[string]any{"Email": newUser.Email, "Password": password})) 284 } 285 286 return addedUsers 287 } 288 289 func (si *SlackImporter) slackAddBotUser(teamId string, log *bytes.Buffer) *model.User { 290 team, err := si.store.Team().Get(teamId) 291 if err != nil { 292 log.WriteString(i18n.T("api.slackimport.slack_import.team_fail")) 293 return nil 294 } 295 296 password := model.NewId() 297 username := "slackimportuser_" + model.NewId() 298 email := username + "@localhost" 299 300 botUser := model.User{ 301 Username: username, 302 FirstName: "", 303 LastName: "", 304 Email: email, 305 Password: password, 306 } 307 308 mUser := si.oldImportUser(team, &botUser) 309 if mUser == nil { 310 log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.unable_import", map[string]any{"Username": username})) 311 return nil 312 } 313 314 log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]any{"Email": botUser.Email, "Password": password})) 315 return mUser 316 } 317 318 func (si *SlackImporter) slackAddPosts(teamId string, channel *model.Channel, posts []slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { 319 sort.Slice(posts, func(i, j int) bool { 320 return slackConvertTimeStamp(posts[i].TimeStamp) < slackConvertTimeStamp(posts[j].TimeStamp) 321 }) 322 threads := make(map[string]string) 323 for _, sPost := range posts { 324 switch { 325 case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): 326 if sPost.User == "" { 327 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 328 continue 329 } 330 if users[sPost.User] == nil { 331 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 332 continue 333 } 334 newPost := model.Post{ 335 UserId: users[sPost.User].Id, 336 ChannelId: channel.Id, 337 Message: sPost.Text, 338 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 339 } 340 if sPost.Upload { 341 if sPost.File != nil { 342 if fileInfo, ok := si.slackUploadFile(sPost.File, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { 343 newPost.FileIds = append(newPost.FileIds, fileInfo.Id) 344 } 345 } else if sPost.Files != nil { 346 for _, file := range sPost.Files { 347 if fileInfo, ok := si.slackUploadFile(file, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { 348 newPost.FileIds = append(newPost.FileIds, fileInfo.Id) 349 } 350 } 351 } 352 } 353 // If post in thread 354 if sPost.ThreadTS != "" && sPost.ThreadTS != sPost.TimeStamp { 355 newPost.RootId = threads[sPost.ThreadTS] 356 } 357 postId := si.oldImportPost(&newPost) 358 // If post is thread starter 359 if sPost.ThreadTS == sPost.TimeStamp { 360 threads[sPost.ThreadTS] = postId 361 } 362 case sPost.Type == "message" && sPost.SubType == "file_comment": 363 if sPost.Comment == nil { 364 mlog.Debug("Slack Import: Unable to import the message as it has no comments.") 365 continue 366 } 367 if sPost.Comment.User == "" { 368 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 369 continue 370 } 371 if users[sPost.Comment.User] == nil { 372 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 373 continue 374 } 375 newPost := model.Post{ 376 UserId: users[sPost.Comment.User].Id, 377 ChannelId: channel.Id, 378 Message: sPost.Comment.Comment, 379 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 380 } 381 si.oldImportPost(&newPost) 382 case sPost.Type == "message" && sPost.SubType == "bot_message": 383 if botUser == nil { 384 mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.") 385 continue 386 } 387 if sPost.BotId == "" { 388 mlog.Warn("Slack Import: Unable to import bot message as the BotId field is missing.") 389 continue 390 } 391 392 props := make(model.StringInterface) 393 props["override_username"] = sPost.BotUsername 394 if len(sPost.Attachments) > 0 { 395 props["attachments"] = sPost.Attachments 396 } 397 398 post := &model.Post{ 399 UserId: botUser.Id, 400 ChannelId: channel.Id, 401 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 402 Message: sPost.Text, 403 Type: model.PostTypeSlackAttachment, 404 } 405 406 postId := si.oldImportIncomingWebhookPost(post, props) 407 // If post is thread starter 408 if sPost.ThreadTS == sPost.TimeStamp { 409 threads[sPost.ThreadTS] = postId 410 } 411 case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): 412 if sPost.User == "" { 413 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 414 continue 415 } 416 if users[sPost.User] == nil { 417 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 418 continue 419 } 420 421 var postType string 422 if sPost.SubType == "channel_join" { 423 postType = model.PostTypeJoinChannel 424 } else { 425 postType = model.PostTypeLeaveChannel 426 } 427 428 newPost := model.Post{ 429 UserId: users[sPost.User].Id, 430 ChannelId: channel.Id, 431 Message: sPost.Text, 432 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 433 Type: postType, 434 Props: model.StringInterface{ 435 "username": users[sPost.User].Username, 436 }, 437 } 438 si.oldImportPost(&newPost) 439 case sPost.Type == "message" && sPost.SubType == "me_message": 440 if sPost.User == "" { 441 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 442 continue 443 } 444 if users[sPost.User] == nil { 445 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 446 continue 447 } 448 newPost := model.Post{ 449 UserId: users[sPost.User].Id, 450 ChannelId: channel.Id, 451 Message: "*" + sPost.Text + "*", 452 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 453 } 454 postId := si.oldImportPost(&newPost) 455 // If post is thread starter 456 if sPost.ThreadTS == sPost.TimeStamp { 457 threads[sPost.ThreadTS] = postId 458 } 459 case sPost.Type == "message" && sPost.SubType == "channel_topic": 460 if sPost.User == "" { 461 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 462 continue 463 } 464 if users[sPost.User] == nil { 465 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 466 continue 467 } 468 newPost := model.Post{ 469 UserId: users[sPost.User].Id, 470 ChannelId: channel.Id, 471 Message: sPost.Text, 472 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 473 Type: model.PostTypeHeaderChange, 474 } 475 si.oldImportPost(&newPost) 476 case sPost.Type == "message" && sPost.SubType == "channel_purpose": 477 if sPost.User == "" { 478 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 479 continue 480 } 481 if users[sPost.User] == nil { 482 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 483 continue 484 } 485 newPost := model.Post{ 486 UserId: users[sPost.User].Id, 487 ChannelId: channel.Id, 488 Message: sPost.Text, 489 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 490 Type: model.PostTypePurposeChange, 491 } 492 si.oldImportPost(&newPost) 493 case sPost.Type == "message" && sPost.SubType == "channel_name": 494 if sPost.User == "" { 495 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 496 continue 497 } 498 if users[sPost.User] == nil { 499 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 500 continue 501 } 502 newPost := model.Post{ 503 UserId: users[sPost.User].Id, 504 ChannelId: channel.Id, 505 Message: sPost.Text, 506 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 507 Type: model.PostTypeDisplaynameChange, 508 } 509 si.oldImportPost(&newPost) 510 default: 511 mlog.Warn( 512 "Slack Import: Unable to import the message as its type is not supported", 513 mlog.String("post_type", sPost.Type), 514 mlog.String("post_subtype", sPost.SubType), 515 ) 516 } 517 } 518 } 519 520 func (si *SlackImporter) slackUploadFile(slackPostFile *slackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) { 521 if slackPostFile == nil { 522 mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.") 523 return nil, false 524 } 525 file, ok := uploads[slackPostFile.Id] 526 if !ok { 527 mlog.Warn("Slack Import: Unable to import file as the file is missing from the Slack export zip file.", mlog.String("file_id", slackPostFile.Id)) 528 return nil, false 529 } 530 openFile, err := file.Open() 531 if err != nil { 532 mlog.Warn("Slack Import: Unable to open the file from the Slack export.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err)) 533 return nil, false 534 } 535 defer openFile.Close() 536 537 timestamp := utils.TimeFromMillis(slackConvertTimeStamp(slackTimestamp)) 538 uploadedFile, err := si.oldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) 539 if err != nil { 540 mlog.Warn("Slack Import: An error occurred when uploading file.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err)) 541 return nil, false 542 } 543 544 return uploadedFile, true 545 } 546 547 func (si *SlackImporter) deactivateSlackBotUser(user *model.User) { 548 if _, err := si.actions.UpdateActive(user, false); err != nil { 549 mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.") 550 } 551 } 552 553 func (si *SlackImporter) addSlackUsersToChannel(c request.CTX, members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { 554 for _, member := range members { 555 user, ok := users[member] 556 if !ok { 557 log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]any{"Username": "?"})) 558 continue 559 } 560 if _, err := si.actions.AddUserToChannel(c, user, channel, false); err != nil { 561 log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]any{"Username": user.Username})) 562 } 563 } 564 } 565 566 func slackSanitiseChannelProperties(channel model.Channel) model.Channel { 567 if utf8.RuneCountInString(channel.DisplayName) > model.ChannelDisplayNameMaxRunes { 568 mlog.Warn("Slack Import: Channel display name exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 569 channel.DisplayName = truncateRunes(channel.DisplayName, model.ChannelDisplayNameMaxRunes) 570 } 571 572 if len(channel.Name) > model.ChannelNameMaxLength { 573 mlog.Warn("Slack Import: Channel handle exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 574 channel.Name = channel.Name[0:model.ChannelNameMaxLength] 575 } 576 577 if utf8.RuneCountInString(channel.Purpose) > model.ChannelPurposeMaxRunes { 578 mlog.Warn("Slack Import: Channel purpose exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 579 channel.Purpose = truncateRunes(channel.Purpose, model.ChannelPurposeMaxRunes) 580 } 581 582 if utf8.RuneCountInString(channel.Header) > model.ChannelHeaderMaxRunes { 583 mlog.Warn("Slack Import: Channel header exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 584 channel.Header = truncateRunes(channel.Header, model.ChannelHeaderMaxRunes) 585 } 586 587 return channel 588 } 589 590 func (si *SlackImporter) slackAddChannels(c request.CTX, teamId string, slackchannels []slackChannel, posts map[string][]slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel { 591 // Write Header 592 importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.added")) 593 importerLog.WriteString("=================\r\n\r\n") 594 595 addedChannels := make(map[string]*model.Channel) 596 for _, sChannel := range slackchannels { 597 newChannel := model.Channel{ 598 TeamId: teamId, 599 Type: sChannel.Type, 600 DisplayName: sChannel.Name, 601 Name: slackConvertChannelName(sChannel.Name, sChannel.Id), 602 Purpose: sChannel.Purpose.Value, 603 Header: sChannel.Topic.Value, 604 } 605 606 // Direct message channels in Slack don't have a name so we set the id as name or else the messages won't get imported. 607 if newChannel.Type == model.ChannelTypeDirect { 608 sChannel.Name = sChannel.Id 609 } 610 611 newChannel = slackSanitiseChannelProperties(newChannel) 612 613 var mChannel *model.Channel 614 var err error 615 if mChannel, err = si.store.Channel().GetByName(teamId, sChannel.Name, true); err == nil { 616 // The channel already exists as an active channel. Merge with the existing one. 617 importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.merge", map[string]any{"DisplayName": newChannel.DisplayName})) 618 } else if _, nErr := si.store.Channel().GetDeletedByName(teamId, sChannel.Name); nErr == nil { 619 // The channel already exists but has been deleted. Generate a random string for the handle instead. 620 newChannel.Name = model.NewId() 621 newChannel = slackSanitiseChannelProperties(newChannel) 622 } 623 624 if mChannel == nil { 625 // Haven't found an existing channel to merge with. Try importing it as a new one. 626 mChannel = si.oldImportChannel(c, &newChannel, sChannel, users) 627 if mChannel == nil { 628 mlog.Warn("Slack Import: Unable to import Slack channel.", mlog.String("channel_display_name", newChannel.DisplayName)) 629 importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.import_failed", map[string]any{"DisplayName": newChannel.DisplayName})) 630 continue 631 } 632 } 633 634 // Members for direct and group channels are added during the creation of the channel in the oldImportChannel function 635 if sChannel.Type == model.ChannelTypeOpen || sChannel.Type == model.ChannelTypePrivate { 636 si.addSlackUsersToChannel(c, sChannel.Members, users, mChannel, importerLog) 637 } 638 importerLog.WriteString(newChannel.DisplayName + "\r\n") 639 addedChannels[sChannel.Id] = mChannel 640 si.slackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) 641 } 642 643 return addedChannels 644 } 645 646 // 647 // -- Old SlackImport Functions -- 648 // Import functions are suitable for entering posts and users into the database without 649 // some of the usual checks. (IsValid is still run) 650 // 651 652 func (si *SlackImporter) oldImportPost(post *model.Post) string { 653 // Workaround for empty messages, which may be the case if they are webhook posts. 654 firstIteration := true 655 firstPostId := "" 656 if post.RootId != "" { 657 firstPostId = post.RootId 658 } 659 maxPostSize := si.actions.MaxPostSize() 660 for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { 661 var remainder string 662 if messageRuneCount > maxPostSize { 663 remainder = string(([]rune(post.Message))[maxPostSize:]) 664 post.Message = truncateRunes(post.Message, maxPostSize) 665 } else { 666 remainder = "" 667 } 668 669 post.Hashtags, _ = model.ParseHashtags(post.Message) 670 671 post.RootId = firstPostId 672 673 _, err := si.store.Post().Save(post) 674 if err != nil { 675 mlog.Debug("Error saving post.", mlog.String("user_id", post.UserId), mlog.String("message", post.Message)) 676 } 677 678 if firstIteration { 679 if firstPostId == "" { 680 firstPostId = post.Id 681 } 682 for _, fileId := range post.FileIds { 683 if err := si.store.FileInfo().AttachToPost(fileId, post.Id, post.ChannelId, post.UserId); err != nil { 684 mlog.Error( 685 "Error attaching files to post.", 686 mlog.String("post_id", post.Id), 687 mlog.String("file_ids", strings.Join(post.FileIds, ",")), 688 mlog.String("user_id", post.UserId), 689 mlog.Err(err), 690 ) 691 } 692 } 693 post.FileIds = nil 694 } 695 696 post.Id = "" 697 post.CreateAt++ 698 post.Message = remainder 699 firstIteration = false 700 } 701 return firstPostId 702 } 703 704 func (si *SlackImporter) oldImportUser(team *model.Team, user *model.User) *model.User { 705 user.MakeNonNil() 706 707 user.Roles = model.SystemUserRoleId 708 709 ruser, nErr := si.store.User().Save(user) 710 if nErr != nil { 711 mlog.Debug("Error saving user.", mlog.Err(nErr)) 712 return nil 713 } 714 715 if _, err := si.store.User().VerifyEmail(ruser.Id, ruser.Email); err != nil { 716 mlog.Warn("Failed to set email verified.", mlog.Err(err)) 717 } 718 719 if _, err := si.actions.JoinUserToTeam(team, user, ""); err != nil { 720 mlog.Warn("Failed to join team when importing.", mlog.Err(err)) 721 } 722 723 return ruser 724 } 725 726 func (si *SlackImporter) oldImportChannel(c request.CTX, channel *model.Channel, sChannel slackChannel, users map[string]*model.User) *model.Channel { 727 switch { 728 case channel.Type == model.ChannelTypeDirect: 729 if len(sChannel.Members) < 2 { 730 return nil 731 } 732 u1 := users[sChannel.Members[0]] 733 u2 := users[sChannel.Members[1]] 734 if u1 == nil || u2 == nil { 735 mlog.Warn("Either or both of user ids not found in users.json. Ignoring.", mlog.String("id1", sChannel.Members[0]), mlog.String("id2", sChannel.Members[1])) 736 return nil 737 } 738 sc, err := si.actions.CreateDirectChannel(c, u1.Id, u2.Id) 739 if err != nil { 740 return nil 741 } 742 743 return sc 744 // check if direct channel has less than 8 members and if not import as private channel instead 745 case channel.Type == model.ChannelTypeGroup && len(sChannel.Members) < 8: 746 members := make([]string, len(sChannel.Members)) 747 748 for i := range sChannel.Members { 749 u := users[sChannel.Members[i]] 750 if u == nil { 751 mlog.Warn("User not found in users.json. Ignoring.", mlog.String("id", sChannel.Members[i])) 752 continue 753 } 754 members[i] = u.Id 755 } 756 757 creator := users[sChannel.Creator] 758 if creator == nil { 759 return nil 760 } 761 sc, err := si.actions.CreateGroupChannel(c, members) 762 if err != nil { 763 return nil 764 } 765 766 return sc 767 case channel.Type == model.ChannelTypeGroup: 768 channel.Type = model.ChannelTypePrivate 769 sc, err := si.actions.CreateChannel(channel, false) 770 if err != nil { 771 return nil 772 } 773 774 return sc 775 } 776 777 sc, err := si.store.Channel().Save(channel, *si.config.TeamSettings.MaxChannelsPerTeam) 778 if err != nil { 779 return nil 780 } 781 782 return sc 783 } 784 785 func (si *SlackImporter) oldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { 786 buf := bytes.NewBuffer(nil) 787 io.Copy(buf, file) 788 data := buf.Bytes() 789 790 fileInfo, err := si.actions.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) 791 if err != nil { 792 return nil, err 793 } 794 795 if fileInfo.IsImage() && !fileInfo.IsSvg() { 796 img, imgType, release, err := si.actions.PrepareImage(data) 797 if err != nil { 798 return nil, err 799 } 800 defer release() 801 si.actions.GenerateThumbnailImage(img, imgType, fileInfo.ThumbnailPath) 802 si.actions.GeneratePreviewImage(img, imgType, fileInfo.PreviewPath) 803 } 804 805 return fileInfo, nil 806 } 807 808 func (si *SlackImporter) oldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string { 809 linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) 810 post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") 811 812 post.AddProp("from_webhook", "true") 813 814 if _, ok := props["override_username"]; !ok { 815 post.AddProp("override_username", model.DefaultWebhookUsername) 816 } 817 818 if len(props) > 0 { 819 for key, val := range props { 820 if key == "attachments" { 821 if attachments, success := val.([]*model.SlackAttachment); success { 822 model.ParseSlackAttachment(post, attachments) 823 } 824 } else if key != "from_webhook" { 825 post.AddProp(key, val) 826 } 827 } 828 } 829 830 return si.oldImportPost(post) 831 }