github.com/masterhung0112/hk_server/v5@v5.0.0-20220302090640-ec71aef15e1c/services/slackimport/slackimport.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See LICENSE.txt for license information. 3 4 package slackimport 5 6 import ( 7 "archive/zip" 8 "bytes" 9 "image" 10 "io" 11 "mime/multipart" 12 "net/http" 13 "path/filepath" 14 "regexp" 15 "sort" 16 "strings" 17 "time" 18 "unicode/utf8" 19 20 "github.com/masterhung0112/hk_server/v5/model" 21 "github.com/masterhung0112/hk_server/v5/shared/i18n" 22 "github.com/masterhung0112/hk_server/v5/shared/mlog" 23 "github.com/masterhung0112/hk_server/v5/store" 24 "github.com/masterhung0112/hk_server/v5/utils" 25 ) 26 27 type slackChannel struct { 28 Id string `json:"id"` 29 Name string `json:"name"` 30 Creator string `json:"creator"` 31 Members []string `json:"members"` 32 Purpose slackChannelSub `json:"purpose"` 33 Topic slackChannelSub `json:"topic"` 34 Type string 35 } 36 37 type slackChannelSub struct { 38 Value string `json:"value"` 39 } 40 41 type slackProfile struct { 42 FirstName string `json:"first_name"` 43 LastName string `json:"last_name"` 44 Email string `json:"email"` 45 } 46 47 type slackUser struct { 48 Id string `json:"id"` 49 Username string `json:"name"` 50 Profile slackProfile `json:"profile"` 51 } 52 53 type slackFile struct { 54 Id string `json:"id"` 55 Title string `json:"title"` 56 } 57 58 type slackPost struct { 59 User string `json:"user"` 60 BotId string `json:"bot_id"` 61 BotUsername string `json:"username"` 62 Text string `json:"text"` 63 TimeStamp string `json:"ts"` 64 ThreadTS string `json:"thread_ts"` 65 Type string `json:"type"` 66 SubType string `json:"subtype"` 67 Comment *slackComment `json:"comment"` 68 Upload bool `json:"upload"` 69 File *slackFile `json:"file"` 70 Files []*slackFile `json:"files"` 71 Attachments []*model.SlackAttachment `json:"attachments"` 72 } 73 74 var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString 75 76 const slackImportMaxFileSize = 1024 * 1024 * 70 77 78 type slackComment struct { 79 User string `json:"user"` 80 Comment string `json:"comment"` 81 } 82 83 // Actions provides the actions that needs to be used for import slack data 84 type Actions struct { 85 UpdateActive func(*model.User, bool) (*model.User, *model.AppError) 86 AddUserToChannel func(*model.User, *model.Channel, bool) (*model.ChannelMember, *model.AppError) 87 JoinUserToTeam func(*model.Team, *model.User, string) (*model.TeamMember, *model.AppError) 88 CreateDirectChannel func(string, string, ...model.ChannelOption) (*model.Channel, *model.AppError) 89 CreateGroupChannel func([]string) (*model.Channel, *model.AppError) 90 CreateChannel func(*model.Channel, bool) (*model.Channel, *model.AppError) 91 DoUploadFile func(time.Time, string, string, string, string, []byte) (*model.FileInfo, *model.AppError) 92 GenerateThumbnailImage func(image.Image, string) 93 GeneratePreviewImage func(image.Image, string) 94 InvalidateAllCaches func() 95 MaxPostSize func() int 96 PrepareImage func(fileData []byte) (image.Image, func(), error) 97 } 98 99 // SlackImporter is a service that allows to import slack dumps into mattermost 100 type SlackImporter struct { 101 store store.Store 102 actions Actions 103 config *model.Config 104 } 105 106 // New creates a new SlackImporter service instance. It receive a store, a set of actions and the current config. 107 // It is expected to be used right away and discarded after that 108 func New(store store.Store, actions Actions, config *model.Config) *SlackImporter { 109 return &SlackImporter{ 110 store: store, 111 actions: actions, 112 config: config, 113 } 114 } 115 116 func (si *SlackImporter) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { 117 // Create log file 118 log := bytes.NewBufferString(i18n.T("api.slackimport.slack_import.log")) 119 120 zipreader, err := zip.NewReader(fileData, fileSize) 121 if err != nil || zipreader.File == nil { 122 log.WriteString(i18n.T("api.slackimport.slack_import.zip.app_error")) 123 return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error(), http.StatusBadRequest), log 124 } 125 126 var channels []slackChannel 127 var publicChannels []slackChannel 128 var privateChannels []slackChannel 129 var groupChannels []slackChannel 130 var directChannels []slackChannel 131 132 var users []slackUser 133 posts := make(map[string][]slackPost) 134 uploads := make(map[string]*zip.File) 135 for _, file := range zipreader.File { 136 if file.UncompressedSize64 > slackImportMaxFileSize { 137 log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]interface{}{"Filename": file.Name})) 138 continue 139 } 140 reader, err := file.Open() 141 if err != nil { 142 log.WriteString(i18n.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name})) 143 return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error(), http.StatusInternalServerError), log 144 } 145 if file.Name == "channels.json" { 146 publicChannels, _ = slackParseChannels(reader, model.CHANNEL_OPEN) 147 channels = append(channels, publicChannels...) 148 } else if file.Name == "dms.json" { 149 directChannels, _ = slackParseChannels(reader, model.CHANNEL_DIRECT) 150 channels = append(channels, directChannels...) 151 } else if file.Name == "groups.json" { 152 privateChannels, _ = slackParseChannels(reader, model.CHANNEL_PRIVATE) 153 channels = append(channels, privateChannels...) 154 } else if file.Name == "mpims.json" { 155 groupChannels, _ = slackParseChannels(reader, model.CHANNEL_GROUP) 156 channels = append(channels, groupChannels...) 157 } else if file.Name == "users.json" { 158 users, _ = slackParseUsers(reader) 159 } else { 160 spl := strings.Split(file.Name, "/") 161 if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { 162 newposts, _ := slackParsePosts(reader) 163 channel := spl[0] 164 if _, ok := posts[channel]; !ok { 165 posts[channel] = newposts 166 } else { 167 posts[channel] = append(posts[channel], newposts...) 168 } 169 } else if len(spl) == 3 && spl[0] == "__uploads" { 170 uploads[spl[1]] = file 171 } 172 } 173 } 174 175 posts = slackConvertUserMentions(users, posts) 176 posts = slackConvertChannelMentions(channels, posts) 177 posts = slackConvertPostsMarkup(posts) 178 179 addedUsers := si.slackAddUsers(teamID, users, log) 180 botUser := si.slackAddBotUser(teamID, log) 181 182 si.slackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log) 183 184 if botUser != nil { 185 si.deactivateSlackBotUser(botUser) 186 } 187 188 si.actions.InvalidateAllCaches() 189 190 log.WriteString(i18n.T("api.slackimport.slack_import.notes")) 191 log.WriteString("=======\r\n\r\n") 192 193 log.WriteString(i18n.T("api.slackimport.slack_import.note1")) 194 log.WriteString(i18n.T("api.slackimport.slack_import.note2")) 195 log.WriteString(i18n.T("api.slackimport.slack_import.note3")) 196 197 return nil, log 198 } 199 200 func truncateRunes(s string, i int) string { 201 runes := []rune(s) 202 if len(runes) > i { 203 return string(runes[:i]) 204 } 205 return s 206 } 207 208 func (si *SlackImporter) slackAddUsers(teamId string, slackusers []slackUser, importerLog *bytes.Buffer) map[string]*model.User { 209 // Log header 210 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.created")) 211 importerLog.WriteString("===============\r\n\r\n") 212 213 addedUsers := make(map[string]*model.User) 214 215 // Need the team 216 team, err := si.store.Team().Get(teamId) 217 if err != nil { 218 importerLog.WriteString(i18n.T("api.slackimport.slack_import.team_fail")) 219 return addedUsers 220 } 221 222 for _, sUser := range slackusers { 223 firstName := sUser.Profile.FirstName 224 lastName := sUser.Profile.LastName 225 email := sUser.Profile.Email 226 if email == "" { 227 email = sUser.Username + "@example.com" 228 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.missing_email_address", map[string]interface{}{"Email": email, "Username": sUser.Username})) 229 mlog.Warn("Slack Import: User does not have an email address in the Slack export. Used username as a placeholder. The user should update their email address once logged in to the system.", mlog.String("user_email", email), mlog.String("user_name", sUser.Username)) 230 } 231 232 password := model.NewId() 233 234 // Check for email conflict and use existing user if found 235 if existingUser, err := si.store.User().GetByEmail(email); err == nil { 236 addedUsers[sUser.Id] = existingUser 237 if _, err := si.actions.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil { 238 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) 239 } else { 240 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) 241 } 242 continue 243 } 244 245 email = strings.ToLower(email) 246 newUser := model.User{ 247 Username: sUser.Username, 248 FirstName: firstName, 249 LastName: lastName, 250 Email: email, 251 Password: password, 252 } 253 254 mUser := si.oldImportUser(team, &newUser) 255 if mUser == nil { 256 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username})) 257 continue 258 } 259 addedUsers[sUser.Id] = mUser 260 importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password})) 261 } 262 263 return addedUsers 264 } 265 266 func (si *SlackImporter) slackAddBotUser(teamId string, log *bytes.Buffer) *model.User { 267 team, err := si.store.Team().Get(teamId) 268 if err != nil { 269 log.WriteString(i18n.T("api.slackimport.slack_import.team_fail")) 270 return nil 271 } 272 273 password := model.NewId() 274 username := "slackimportuser_" + model.NewId() 275 email := username + "@localhost" 276 277 botUser := model.User{ 278 Username: username, 279 FirstName: "", 280 LastName: "", 281 Email: email, 282 Password: password, 283 } 284 285 mUser := si.oldImportUser(team, &botUser) 286 if mUser == nil { 287 log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username})) 288 return nil 289 } 290 291 log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password})) 292 return mUser 293 } 294 295 func (si *SlackImporter) slackAddPosts(teamId string, channel *model.Channel, posts []slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { 296 sort.Slice(posts, func(i, j int) bool { 297 return slackConvertTimeStamp(posts[i].TimeStamp) < slackConvertTimeStamp(posts[j].TimeStamp) 298 }) 299 threads := make(map[string]string) 300 for _, sPost := range posts { 301 switch { 302 case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): 303 if sPost.User == "" { 304 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 305 continue 306 } 307 if users[sPost.User] == nil { 308 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 309 continue 310 } 311 newPost := model.Post{ 312 UserId: users[sPost.User].Id, 313 ChannelId: channel.Id, 314 Message: sPost.Text, 315 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 316 } 317 if sPost.Upload { 318 if sPost.File != nil { 319 if fileInfo, ok := si.slackUploadFile(sPost.File, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { 320 newPost.FileIds = append(newPost.FileIds, fileInfo.Id) 321 } 322 } else if sPost.Files != nil { 323 for _, file := range sPost.Files { 324 if fileInfo, ok := si.slackUploadFile(file, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok { 325 newPost.FileIds = append(newPost.FileIds, fileInfo.Id) 326 } 327 } 328 } 329 } 330 // If post in thread 331 if sPost.ThreadTS != "" && sPost.ThreadTS != sPost.TimeStamp { 332 newPost.RootId = threads[sPost.ThreadTS] 333 newPost.ParentId = threads[sPost.ThreadTS] 334 } 335 postId := si.oldImportPost(&newPost) 336 // If post is thread starter 337 if sPost.ThreadTS == sPost.TimeStamp { 338 threads[sPost.ThreadTS] = postId 339 } 340 case sPost.Type == "message" && sPost.SubType == "file_comment": 341 if sPost.Comment == nil { 342 mlog.Debug("Slack Import: Unable to import the message as it has no comments.") 343 continue 344 } 345 if sPost.Comment.User == "" { 346 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 347 continue 348 } 349 if users[sPost.Comment.User] == nil { 350 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 351 continue 352 } 353 newPost := model.Post{ 354 UserId: users[sPost.Comment.User].Id, 355 ChannelId: channel.Id, 356 Message: sPost.Comment.Comment, 357 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 358 } 359 si.oldImportPost(&newPost) 360 case sPost.Type == "message" && sPost.SubType == "bot_message": 361 if botUser == nil { 362 mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.") 363 continue 364 } 365 if sPost.BotId == "" { 366 mlog.Warn("Slack Import: Unable to import bot message as the BotId field is missing.") 367 continue 368 } 369 370 props := make(model.StringInterface) 371 props["override_username"] = sPost.BotUsername 372 if len(sPost.Attachments) > 0 { 373 props["attachments"] = sPost.Attachments 374 } 375 376 post := &model.Post{ 377 UserId: botUser.Id, 378 ChannelId: channel.Id, 379 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 380 Message: sPost.Text, 381 Type: model.POST_SLACK_ATTACHMENT, 382 } 383 384 postId := si.oldImportIncomingWebhookPost(post, props) 385 // If post is thread starter 386 if sPost.ThreadTS == sPost.TimeStamp { 387 threads[sPost.ThreadTS] = postId 388 } 389 case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): 390 if sPost.User == "" { 391 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 392 continue 393 } 394 if users[sPost.User] == nil { 395 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 396 continue 397 } 398 399 var postType string 400 if sPost.SubType == "channel_join" { 401 postType = model.POST_JOIN_CHANNEL 402 } else { 403 postType = model.POST_LEAVE_CHANNEL 404 } 405 406 newPost := model.Post{ 407 UserId: users[sPost.User].Id, 408 ChannelId: channel.Id, 409 Message: sPost.Text, 410 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 411 Type: postType, 412 Props: model.StringInterface{ 413 "username": users[sPost.User].Username, 414 }, 415 } 416 si.oldImportPost(&newPost) 417 case sPost.Type == "message" && sPost.SubType == "me_message": 418 if sPost.User == "" { 419 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 420 continue 421 } 422 if users[sPost.User] == nil { 423 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 424 continue 425 } 426 newPost := model.Post{ 427 UserId: users[sPost.User].Id, 428 ChannelId: channel.Id, 429 Message: "*" + sPost.Text + "*", 430 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 431 } 432 postId := si.oldImportPost(&newPost) 433 // If post is thread starter 434 if sPost.ThreadTS == sPost.TimeStamp { 435 threads[sPost.ThreadTS] = postId 436 } 437 case sPost.Type == "message" && sPost.SubType == "channel_topic": 438 if sPost.User == "" { 439 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 440 continue 441 } 442 if users[sPost.User] == nil { 443 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 444 continue 445 } 446 newPost := model.Post{ 447 UserId: users[sPost.User].Id, 448 ChannelId: channel.Id, 449 Message: sPost.Text, 450 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 451 Type: model.POST_HEADER_CHANGE, 452 } 453 si.oldImportPost(&newPost) 454 case sPost.Type == "message" && sPost.SubType == "channel_purpose": 455 if sPost.User == "" { 456 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 457 continue 458 } 459 if users[sPost.User] == nil { 460 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 461 continue 462 } 463 newPost := model.Post{ 464 UserId: users[sPost.User].Id, 465 ChannelId: channel.Id, 466 Message: sPost.Text, 467 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 468 Type: model.POST_PURPOSE_CHANGE, 469 } 470 si.oldImportPost(&newPost) 471 case sPost.Type == "message" && sPost.SubType == "channel_name": 472 if sPost.User == "" { 473 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 474 continue 475 } 476 if users[sPost.User] == nil { 477 mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User)) 478 continue 479 } 480 newPost := model.Post{ 481 UserId: users[sPost.User].Id, 482 ChannelId: channel.Id, 483 Message: sPost.Text, 484 CreateAt: slackConvertTimeStamp(sPost.TimeStamp), 485 Type: model.POST_DISPLAYNAME_CHANGE, 486 } 487 si.oldImportPost(&newPost) 488 default: 489 mlog.Warn( 490 "Slack Import: Unable to import the message as its type is not supported", 491 mlog.String("post_type", sPost.Type), 492 mlog.String("post_subtype", sPost.SubType), 493 ) 494 } 495 } 496 } 497 498 func (si *SlackImporter) slackUploadFile(slackPostFile *slackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) { 499 if slackPostFile == nil { 500 mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.") 501 return nil, false 502 } 503 file, ok := uploads[slackPostFile.Id] 504 if !ok { 505 mlog.Warn("Slack Import: Unable to import file as the file is missing from the Slack export zip file.", mlog.String("file_id", slackPostFile.Id)) 506 return nil, false 507 } 508 openFile, err := file.Open() 509 if err != nil { 510 mlog.Warn("Slack Import: Unable to open the file from the Slack export.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err)) 511 return nil, false 512 } 513 defer openFile.Close() 514 515 timestamp := utils.TimeFromMillis(slackConvertTimeStamp(slackTimestamp)) 516 uploadedFile, err := si.oldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) 517 if err != nil { 518 mlog.Warn("Slack Import: An error occurred when uploading file.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err)) 519 return nil, false 520 } 521 522 return uploadedFile, true 523 } 524 525 func (si *SlackImporter) deactivateSlackBotUser(user *model.User) { 526 if _, err := si.actions.UpdateActive(user, false); err != nil { 527 mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.") 528 } 529 } 530 531 func (si *SlackImporter) addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { 532 for _, member := range members { 533 user, ok := users[member] 534 if !ok { 535 log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"})) 536 continue 537 } 538 if _, err := si.actions.AddUserToChannel(user, channel, false); err != nil { 539 log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username})) 540 } 541 } 542 } 543 544 func slackSanitiseChannelProperties(channel model.Channel) model.Channel { 545 if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { 546 mlog.Warn("Slack Import: Channel display name exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 547 channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES) 548 } 549 550 if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH { 551 mlog.Warn("Slack Import: Channel handle exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 552 channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH] 553 } 554 555 if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES { 556 mlog.Warn("Slack Import: Channel purpose exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 557 channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES) 558 } 559 560 if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES { 561 mlog.Warn("Slack Import: Channel header exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName)) 562 channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES) 563 } 564 565 return channel 566 } 567 568 func (si *SlackImporter) slackAddChannels(teamId string, slackchannels []slackChannel, posts map[string][]slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel { 569 // Write Header 570 importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.added")) 571 importerLog.WriteString("=================\r\n\r\n") 572 573 addedChannels := make(map[string]*model.Channel) 574 for _, sChannel := range slackchannels { 575 newChannel := model.Channel{ 576 TeamId: teamId, 577 Type: sChannel.Type, 578 DisplayName: sChannel.Name, 579 Name: slackConvertChannelName(sChannel.Name, sChannel.Id), 580 Purpose: sChannel.Purpose.Value, 581 Header: sChannel.Topic.Value, 582 } 583 584 // Direct message channels in Slack don't have a name so we set the id as name or else the messages won't get imported. 585 if newChannel.Type == model.CHANNEL_DIRECT { 586 sChannel.Name = sChannel.Id 587 } 588 589 newChannel = slackSanitiseChannelProperties(newChannel) 590 591 var mChannel *model.Channel 592 var err error 593 if mChannel, err = si.store.Channel().GetByName(teamId, sChannel.Name, true); err == nil { 594 // The channel already exists as an active channel. Merge with the existing one. 595 importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName})) 596 } else if _, nErr := si.store.Channel().GetDeletedByName(teamId, sChannel.Name); nErr == nil { 597 // The channel already exists but has been deleted. Generate a random string for the handle instead. 598 newChannel.Name = model.NewId() 599 newChannel = slackSanitiseChannelProperties(newChannel) 600 } 601 602 if mChannel == nil { 603 // Haven't found an existing channel to merge with. Try importing it as a new one. 604 mChannel = si.oldImportChannel(&newChannel, sChannel, users) 605 if mChannel == nil { 606 mlog.Warn("Slack Import: Unable to import Slack channel.", mlog.String("channel_display_name", newChannel.DisplayName)) 607 importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName})) 608 continue 609 } 610 } 611 612 // Members for direct and group channels are added during the creation of the channel in the oldImportChannel function 613 if sChannel.Type == model.CHANNEL_OPEN || sChannel.Type == model.CHANNEL_PRIVATE { 614 si.addSlackUsersToChannel(sChannel.Members, users, mChannel, importerLog) 615 } 616 importerLog.WriteString(newChannel.DisplayName + "\r\n") 617 addedChannels[sChannel.Id] = mChannel 618 si.slackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) 619 } 620 621 return addedChannels 622 } 623 624 // 625 // -- Old SlackImport Functions -- 626 // Import functions are suitable for entering posts and users into the database without 627 // some of the usual checks. (IsValid is still run) 628 // 629 630 func (si *SlackImporter) oldImportPost(post *model.Post) string { 631 // Workaround for empty messages, which may be the case if they are webhook posts. 632 firstIteration := true 633 firstPostId := "" 634 if post.ParentId != "" { 635 firstPostId = post.ParentId 636 } 637 maxPostSize := si.actions.MaxPostSize() 638 for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { 639 var remainder string 640 if messageRuneCount > maxPostSize { 641 remainder = string(([]rune(post.Message))[maxPostSize:]) 642 post.Message = truncateRunes(post.Message, maxPostSize) 643 } else { 644 remainder = "" 645 } 646 647 post.Hashtags, _ = model.ParseHashtags(post.Message) 648 649 post.RootId = firstPostId 650 post.ParentId = firstPostId 651 652 _, err := si.store.Post().Save(post) 653 if err != nil { 654 mlog.Debug("Error saving post.", mlog.String("user_id", post.UserId), mlog.String("message", post.Message)) 655 } 656 657 if firstIteration { 658 if firstPostId == "" { 659 firstPostId = post.Id 660 } 661 for _, fileId := range post.FileIds { 662 if err := si.store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); err != nil { 663 mlog.Error( 664 "Error attaching files to post.", 665 mlog.String("post_id", post.Id), 666 mlog.String("file_ids", strings.Join(post.FileIds, ",")), 667 mlog.String("user_id", post.UserId), 668 mlog.Err(err), 669 ) 670 } 671 } 672 post.FileIds = nil 673 } 674 675 post.Id = "" 676 post.CreateAt++ 677 post.Message = remainder 678 firstIteration = false 679 } 680 return firstPostId 681 } 682 683 func (si *SlackImporter) oldImportUser(team *model.Team, user *model.User) *model.User { 684 user.MakeNonNil() 685 686 user.Roles = model.SYSTEM_USER_ROLE_ID 687 688 ruser, nErr := si.store.User().Save(user) 689 if nErr != nil { 690 mlog.Debug("Error saving user.", mlog.Err(nErr)) 691 return nil 692 } 693 694 if _, err := si.store.User().VerifyEmail(ruser.Id, ruser.Email); err != nil { 695 mlog.Warn("Failed to set email verified.", mlog.Err(err)) 696 } 697 698 if _, err := si.actions.JoinUserToTeam(team, user, ""); err != nil { 699 mlog.Warn("Failed to join team when importing.", mlog.Err(err)) 700 } 701 702 return ruser 703 } 704 705 func (si *SlackImporter) oldImportChannel(channel *model.Channel, sChannel slackChannel, users map[string]*model.User) *model.Channel { 706 switch { 707 case channel.Type == model.CHANNEL_DIRECT: 708 if len(sChannel.Members) < 2 { 709 return nil 710 } 711 u1 := users[sChannel.Members[0]] 712 u2 := users[sChannel.Members[1]] 713 if u1 == nil || u2 == nil { 714 mlog.Warn("Either or both of user ids not found in users.json. Ignoring.", mlog.String("id1", sChannel.Members[0]), mlog.String("id2", sChannel.Members[1])) 715 return nil 716 } 717 sc, err := si.actions.CreateDirectChannel(u1.Id, u2.Id) 718 if err != nil { 719 return nil 720 } 721 722 return sc 723 // check if direct channel has less than 8 members and if not import as private channel instead 724 case channel.Type == model.CHANNEL_GROUP && len(sChannel.Members) < 8: 725 members := make([]string, len(sChannel.Members)) 726 727 for i := range sChannel.Members { 728 u := users[sChannel.Members[i]] 729 if u == nil { 730 mlog.Warn("User not found in users.json. Ignoring.", mlog.String("id", sChannel.Members[i])) 731 continue 732 } 733 members[i] = u.Id 734 } 735 736 creator := users[sChannel.Creator] 737 if creator == nil { 738 return nil 739 } 740 sc, err := si.actions.CreateGroupChannel(members) 741 if err != nil { 742 return nil 743 } 744 745 return sc 746 case channel.Type == model.CHANNEL_GROUP: 747 channel.Type = model.CHANNEL_PRIVATE 748 sc, err := si.actions.CreateChannel(channel, false) 749 if err != nil { 750 return nil 751 } 752 753 return sc 754 } 755 756 sc, err := si.store.Channel().Save(channel, *si.config.TeamSettings.MaxChannelsPerTeam) 757 if err != nil { 758 return nil 759 } 760 761 return sc 762 } 763 764 func (si *SlackImporter) oldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { 765 buf := bytes.NewBuffer(nil) 766 io.Copy(buf, file) 767 data := buf.Bytes() 768 769 fileInfo, err := si.actions.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) 770 if err != nil { 771 return nil, err 772 } 773 774 if fileInfo.IsImage() && fileInfo.MimeType != "image/svg+xml" { 775 img, release, err := si.actions.PrepareImage(data) 776 if err != nil { 777 return nil, err 778 } 779 defer release() 780 si.actions.GenerateThumbnailImage(img, fileInfo.ThumbnailPath) 781 si.actions.GeneratePreviewImage(img, fileInfo.PreviewPath) 782 } 783 784 return fileInfo, nil 785 } 786 787 func (si *SlackImporter) oldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string { 788 linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) 789 post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") 790 791 post.AddProp("from_webhook", "true") 792 793 if _, ok := props["override_username"]; !ok { 794 post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME) 795 } 796 797 if len(props) > 0 { 798 for key, val := range props { 799 if key == "attachments" { 800 if attachments, success := val.([]*model.SlackAttachment); success { 801 model.ParseSlackAttachment(post, attachments) 802 } 803 } else if key != "from_webhook" { 804 post.AddProp(key, val) 805 } 806 } 807 } 808 809 return si.oldImportPost(post) 810 }