github.com/vnforks/kid@v5.11.1+incompatible/app/slackimport.go (about) 1 // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package app 5 6 import ( 7 "archive/zip" 8 "bytes" 9 "encoding/json" 10 "fmt" 11 "io" 12 "mime/multipart" 13 "net/http" 14 "path/filepath" 15 "regexp" 16 "strconv" 17 "strings" 18 "time" 19 "unicode/utf8" 20 21 "github.com/mattermost/mattermost-server/mlog" 22 "github.com/mattermost/mattermost-server/model" 23 "github.com/mattermost/mattermost-server/utils" 24 ) 25 26 type SlackChannel struct { 27 Id string `json:"id"` 28 Name string `json:"name"` 29 Members []string `json:"members"` 30 Topic map[string]string `json:"topic"` 31 Purpose map[string]string `json:"purpose"` 32 } 33 34 type SlackProfile struct { 35 FirstName string `json:"first_name"` 36 LastName string `json:"last_name"` 37 Email string `json:"email"` 38 } 39 40 type SlackUser struct { 41 Id string `json:"id"` 42 Username string `json:"name"` 43 Profile SlackProfile `json:"profile"` 44 } 45 46 type SlackFile struct { 47 Id string `json:"id"` 48 Title string `json:"title"` 49 } 50 51 type SlackPost struct { 52 User string `json:"user"` 53 BotId string `json:"bot_id"` 54 BotUsername string `json:"username"` 55 Text string `json:"text"` 56 TimeStamp string `json:"ts"` 57 Type string `json:"type"` 58 SubType string `json:"subtype"` 59 Comment *SlackComment `json:"comment"` 60 Upload bool `json:"upload"` 61 File *SlackFile `json:"file"` 62 Attachments []*model.SlackAttachment `json:"attachments"` 63 } 64 65 var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString 66 67 type SlackComment struct { 68 User string `json:"user"` 69 Comment string `json:"comment"` 70 } 71 72 func truncateRunes(s string, i int) string { 73 runes := []rune(s) 74 if len(runes) > i { 75 return string(runes[:i]) 76 } 77 return s 78 } 79 80 func SlackConvertTimeStamp(ts string) int64 { 81 timeString := strings.SplitN(ts, ".", 2)[0] 82 83 timeStamp, err := strconv.ParseInt(timeString, 10, 64) 84 if err != nil { 85 mlog.Warn("Slack Import: Bad timestamp detected.") 86 return 1 87 } 88 return timeStamp * 1000 // Convert to milliseconds 89 } 90 91 func SlackConvertChannelName(channelName string, channelId string) string { 92 newName := strings.Trim(channelName, "_-") 93 if len(newName) == 1 { 94 return "slack-channel-" + newName 95 } 96 97 if isValidChannelNameCharacters(newName) { 98 return newName 99 } 100 return strings.ToLower(channelId) 101 } 102 103 func SlackParseChannels(data io.Reader) ([]SlackChannel, error) { 104 decoder := json.NewDecoder(data) 105 106 var channels []SlackChannel 107 if err := decoder.Decode(&channels); err != nil { 108 mlog.Warn("Slack Import: Error occurred when parsing some Slack channels. Import may work anyway.") 109 return channels, err 110 } 111 return channels, nil 112 } 113 114 func SlackParseUsers(data io.Reader) ([]SlackUser, error) { 115 decoder := json.NewDecoder(data) 116 117 var users []SlackUser 118 err := decoder.Decode(&users) 119 // This actually returns errors that are ignored. 120 // In this case it is erroring because of a null that Slack 121 // introduced. So we just return the users here. 122 return users, err 123 } 124 125 func SlackParsePosts(data io.Reader) ([]SlackPost, error) { 126 decoder := json.NewDecoder(data) 127 128 var posts []SlackPost 129 if err := decoder.Decode(&posts); err != nil { 130 mlog.Warn("Slack Import: Error occurred when parsing some Slack posts. Import may work anyway.") 131 return posts, err 132 } 133 return posts, nil 134 } 135 136 func (a *App) SlackAddUsers(teamId string, slackusers []SlackUser, importerLog *bytes.Buffer) map[string]*model.User { 137 // Log header 138 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.created")) 139 importerLog.WriteString("===============\r\n\r\n") 140 141 addedUsers := make(map[string]*model.User) 142 143 // Need the team 144 result := <-a.Srv.Store.Team().Get(teamId) 145 if result.Err != nil { 146 importerLog.WriteString(utils.T("api.slackimport.slack_import.team_fail")) 147 return addedUsers 148 } 149 team := result.Data.(*model.Team) 150 151 for _, sUser := range slackusers { 152 firstName := sUser.Profile.FirstName 153 lastName := sUser.Profile.LastName 154 email := sUser.Profile.Email 155 if email == "" { 156 email = sUser.Username + "@example.com" 157 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.missing_email_address", map[string]interface{}{"Email": email, "Username": sUser.Username})) 158 mlog.Warn(fmt.Sprintf("Slack Import: User %v does not have an email address in the Slack export. Used %v as a placeholder. The user should update their email address once logged in to the system.", email, sUser.Username)) 159 } 160 161 password := model.NewId() 162 163 // Check for email conflict and use existing user if found 164 if result := <-a.Srv.Store.User().GetByEmail(email); result.Err == nil { 165 existingUser := result.Data.(*model.User) 166 addedUsers[sUser.Id] = existingUser 167 if err := a.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil { 168 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) 169 } else { 170 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username})) 171 } 172 continue 173 } 174 175 newUser := model.User{ 176 Username: sUser.Username, 177 FirstName: firstName, 178 LastName: lastName, 179 Email: email, 180 Password: password, 181 } 182 183 mUser := a.OldImportUser(team, &newUser) 184 if mUser == nil { 185 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username})) 186 continue 187 } 188 addedUsers[sUser.Id] = mUser 189 importerLog.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password})) 190 } 191 192 return addedUsers 193 } 194 195 func (a *App) SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User { 196 result := <-a.Srv.Store.Team().Get(teamId) 197 if result.Err != nil { 198 log.WriteString(utils.T("api.slackimport.slack_import.team_fail")) 199 return nil 200 } 201 team := result.Data.(*model.Team) 202 203 password := model.NewId() 204 username := "slackimportuser_" + model.NewId() 205 email := username + "@localhost" 206 207 botUser := model.User{ 208 Username: username, 209 FirstName: "", 210 LastName: "", 211 Email: email, 212 Password: password, 213 } 214 215 mUser := a.OldImportUser(team, &botUser) 216 if mUser == nil { 217 log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username})) 218 return nil 219 } 220 221 log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password})) 222 return mUser 223 } 224 225 func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) { 226 for _, sPost := range posts { 227 switch { 228 case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"): 229 if sPost.User == "" { 230 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 231 continue 232 } 233 if users[sPost.User] == nil { 234 mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User)) 235 continue 236 } 237 newPost := model.Post{ 238 UserId: users[sPost.User].Id, 239 ChannelId: channel.Id, 240 Message: sPost.Text, 241 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 242 } 243 if sPost.Upload { 244 if fileInfo, ok := a.SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok { 245 newPost.FileIds = append(newPost.FileIds, fileInfo.Id) 246 newPost.Message = sPost.File.Title 247 } 248 } 249 a.OldImportPost(&newPost) 250 for _, fileId := range newPost.FileIds { 251 if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id, newPost.UserId); result.Err != nil { 252 mlog.Error(fmt.Sprintf("Slack Import: An error occurred when attaching files to a message, post_id=%s, file_ids=%v, err=%v.", newPost.Id, newPost.FileIds, result.Err)) 253 } 254 } 255 256 case sPost.Type == "message" && sPost.SubType == "file_comment": 257 if sPost.Comment == nil { 258 mlog.Debug("Slack Import: Unable to import the message as it has no comments.") 259 continue 260 } 261 if sPost.Comment.User == "" { 262 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 263 continue 264 } 265 if users[sPost.Comment.User] == nil { 266 mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User)) 267 continue 268 } 269 newPost := model.Post{ 270 UserId: users[sPost.Comment.User].Id, 271 ChannelId: channel.Id, 272 Message: sPost.Comment.Comment, 273 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 274 } 275 a.OldImportPost(&newPost) 276 case sPost.Type == "message" && sPost.SubType == "bot_message": 277 if botUser == nil { 278 mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.") 279 continue 280 } 281 if sPost.BotId == "" { 282 mlog.Warn("Slack Import: Unable to import bot message as the BotId field is missing.") 283 continue 284 } 285 286 props := make(model.StringInterface) 287 props["override_username"] = sPost.BotUsername 288 if len(sPost.Attachments) > 0 { 289 props["attachments"] = sPost.Attachments 290 } 291 292 post := &model.Post{ 293 UserId: botUser.Id, 294 ChannelId: channel.Id, 295 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 296 Message: sPost.Text, 297 Type: model.POST_SLACK_ATTACHMENT, 298 } 299 300 a.OldImportIncomingWebhookPost(post, props) 301 case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"): 302 if sPost.User == "" { 303 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 304 continue 305 } 306 if users[sPost.User] == nil { 307 mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User)) 308 continue 309 } 310 311 var postType string 312 if sPost.SubType == "channel_join" { 313 postType = model.POST_JOIN_CHANNEL 314 } else { 315 postType = model.POST_LEAVE_CHANNEL 316 } 317 318 newPost := model.Post{ 319 UserId: users[sPost.User].Id, 320 ChannelId: channel.Id, 321 Message: sPost.Text, 322 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 323 Type: postType, 324 Props: model.StringInterface{ 325 "username": users[sPost.User].Username, 326 }, 327 } 328 a.OldImportPost(&newPost) 329 case sPost.Type == "message" && sPost.SubType == "me_message": 330 if sPost.User == "" { 331 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 332 continue 333 } 334 if users[sPost.User] == nil { 335 mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User)) 336 continue 337 } 338 newPost := model.Post{ 339 UserId: users[sPost.User].Id, 340 ChannelId: channel.Id, 341 Message: "*" + sPost.Text + "*", 342 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 343 } 344 a.OldImportPost(&newPost) 345 case sPost.Type == "message" && sPost.SubType == "channel_topic": 346 if sPost.User == "" { 347 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 348 continue 349 } 350 if users[sPost.User] == nil { 351 mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User)) 352 continue 353 } 354 newPost := model.Post{ 355 UserId: users[sPost.User].Id, 356 ChannelId: channel.Id, 357 Message: sPost.Text, 358 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 359 Type: model.POST_HEADER_CHANGE, 360 } 361 a.OldImportPost(&newPost) 362 case sPost.Type == "message" && sPost.SubType == "channel_purpose": 363 if sPost.User == "" { 364 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 365 continue 366 } 367 if users[sPost.User] == nil { 368 mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User)) 369 continue 370 } 371 newPost := model.Post{ 372 UserId: users[sPost.User].Id, 373 ChannelId: channel.Id, 374 Message: sPost.Text, 375 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 376 Type: model.POST_PURPOSE_CHANGE, 377 } 378 a.OldImportPost(&newPost) 379 case sPost.Type == "message" && sPost.SubType == "channel_name": 380 if sPost.User == "" { 381 mlog.Debug("Slack Import: Unable to import the message as the user field is missing.") 382 continue 383 } 384 if users[sPost.User] == nil { 385 mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User)) 386 continue 387 } 388 newPost := model.Post{ 389 UserId: users[sPost.User].Id, 390 ChannelId: channel.Id, 391 Message: sPost.Text, 392 CreateAt: SlackConvertTimeStamp(sPost.TimeStamp), 393 Type: model.POST_DISPLAYNAME_CHANGE, 394 } 395 a.OldImportPost(&newPost) 396 default: 397 mlog.Warn(fmt.Sprintf("Slack Import: Unable to import the message as its type is not supported: post_type=%v, post_subtype=%v.", sPost.Type, sPost.SubType)) 398 } 399 } 400 } 401 402 func (a *App) SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) { 403 if sPost.File == nil { 404 mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.") 405 return nil, false 406 } 407 file, ok := uploads[sPost.File.Id] 408 if !ok { 409 mlog.Warn(fmt.Sprintf("Slack Import: Unable to import file %v as the file is missing from the Slack export zip file.", sPost.File.Id)) 410 return nil, false 411 } 412 openFile, err := file.Open() 413 if err != nil { 414 mlog.Warn(fmt.Sprintf("Slack Import: Unable to open the file %v from the Slack export: %v.", sPost.File.Id, err.Error())) 415 return nil, false 416 } 417 defer openFile.Close() 418 419 timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(sPost.TimeStamp)) 420 uploadedFile, err := a.OldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name)) 421 if err != nil { 422 mlog.Warn(fmt.Sprintf("Slack Import: An error occurred when uploading file %v: %v.", sPost.File.Id, err.Error())) 423 return nil, false 424 } 425 426 return uploadedFile, true 427 } 428 429 func (a *App) deactivateSlackBotUser(user *model.User) { 430 if _, err := a.UpdateActive(user, false); err != nil { 431 mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.") 432 } 433 } 434 435 func (a *App) addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) { 436 for _, member := range members { 437 user, ok := users[member] 438 if !ok { 439 log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"})) 440 continue 441 } 442 if _, err := a.AddUserToChannel(user, channel); err != nil { 443 log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username})) 444 } 445 } 446 } 447 448 func SlackSanitiseChannelProperties(channel model.Channel) model.Channel { 449 if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES { 450 mlog.Warn(fmt.Sprintf("Slack Import: Channel %v display name exceeds the maximum length. It will be truncated when imported.", channel.DisplayName)) 451 channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES) 452 } 453 454 if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH { 455 mlog.Warn(fmt.Sprintf("Slack Import: Channel %v handle exceeds the maximum length. It will be truncated when imported.", channel.DisplayName)) 456 channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH] 457 } 458 459 if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES { 460 mlog.Warn(fmt.Sprintf("Slack Import: Channel %v purpose exceeds the maximum length. It will be truncated when imported.", channel.DisplayName)) 461 channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES) 462 } 463 464 if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES { 465 mlog.Warn(fmt.Sprintf("Slack Import: Channel %v header exceeds the maximum length. It will be truncated when imported.", channel.DisplayName)) 466 channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES) 467 } 468 469 return channel 470 } 471 472 func (a *App) SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel { 473 // Write Header 474 importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.added")) 475 importerLog.WriteString("=================\r\n\r\n") 476 477 addedChannels := make(map[string]*model.Channel) 478 for _, sChannel := range slackchannels { 479 newChannel := model.Channel{ 480 TeamId: teamId, 481 Type: model.CHANNEL_OPEN, 482 DisplayName: sChannel.Name, 483 Name: SlackConvertChannelName(sChannel.Name, sChannel.Id), 484 Purpose: sChannel.Purpose["value"], 485 Header: sChannel.Topic["value"], 486 } 487 newChannel = SlackSanitiseChannelProperties(newChannel) 488 489 var mChannel *model.Channel 490 if result := <-a.Srv.Store.Channel().GetByName(teamId, sChannel.Name, true); result.Err == nil { 491 // The channel already exists as an active channel. Merge with the existing one. 492 mChannel = result.Data.(*model.Channel) 493 importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName})) 494 } else if result := <-a.Srv.Store.Channel().GetDeletedByName(teamId, sChannel.Name); result.Err == nil { 495 // The channel already exists but has been deleted. Generate a random string for the handle instead. 496 newChannel.Name = model.NewId() 497 newChannel = SlackSanitiseChannelProperties(newChannel) 498 } 499 500 if mChannel == nil { 501 // Haven't found an existing channel to merge with. Try importing it as a new one. 502 mChannel = a.OldImportChannel(&newChannel) 503 if mChannel == nil { 504 mlog.Warn(fmt.Sprintf("Slack Import: Unable to import Slack channel: %s.", newChannel.DisplayName)) 505 importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName})) 506 continue 507 } 508 } 509 510 a.addSlackUsersToChannel(sChannel.Members, users, mChannel, importerLog) 511 importerLog.WriteString(newChannel.DisplayName + "\r\n") 512 addedChannels[sChannel.Id] = mChannel 513 a.SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser) 514 } 515 516 return addedChannels 517 } 518 519 func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost { 520 var regexes = make(map[string]*regexp.Regexp, len(users)) 521 for _, user := range users { 522 r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>") 523 if err != nil { 524 mlog.Warn(fmt.Sprintf("Slack Import: Unable to compile the @mention, matching regular expression for the Slack user %v (id=%v).", user.Id, user.Username), mlog.String("user_id", user.Id)) 525 continue 526 } 527 regexes["@"+user.Username] = r 528 } 529 530 // Special cases. 531 regexes["@here"], _ = regexp.Compile(`<!here\|@here>`) 532 regexes["@channel"], _ = regexp.Compile("<!channel>") 533 regexes["@all"], _ = regexp.Compile("<!everyone>") 534 535 for channelName, channelPosts := range posts { 536 for postIdx, post := range channelPosts { 537 for mention, r := range regexes { 538 post.Text = r.ReplaceAllString(post.Text, mention) 539 posts[channelName][postIdx] = post 540 } 541 } 542 } 543 544 return posts 545 } 546 547 func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost { 548 var regexes = make(map[string]*regexp.Regexp, len(channels)) 549 for _, channel := range channels { 550 r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>") 551 if err != nil { 552 mlog.Warn(fmt.Sprintf("Slack Import: Unable to compile the !channel, matching regular expression for the Slack channel %v (id=%v).", channel.Id, channel.Name)) 553 continue 554 } 555 regexes["~"+channel.Name] = r 556 } 557 558 for channelName, channelPosts := range posts { 559 for postIdx, post := range channelPosts { 560 for channelReplace, r := range regexes { 561 post.Text = r.ReplaceAllString(post.Text, channelReplace) 562 posts[channelName][postIdx] = post 563 } 564 } 565 } 566 567 return posts 568 } 569 570 func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost { 571 regexReplaceAllString := []struct { 572 regex *regexp.Regexp 573 rpl string 574 }{ 575 // URL 576 { 577 regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`), 578 "[$2]($1)", 579 }, 580 // bold 581 { 582 regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`), 583 "$1**$2**", 584 }, 585 // strikethrough 586 { 587 regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`), 588 "$1~~$2~~", 589 }, 590 // single paragraph blockquote 591 // Slack converts > character to > 592 { 593 regexp.MustCompile(`(?sm)^>`), 594 ">", 595 }, 596 } 597 598 regexReplaceAllStringFunc := []struct { 599 regex *regexp.Regexp 600 fn func(string) string 601 }{ 602 // multiple paragraphs blockquotes 603 { 604 regexp.MustCompile(`(?sm)^>>>(.+)$`), 605 func(src string) string { 606 // remove >>> prefix, might have leading \n 607 prefixRegexp := regexp.MustCompile(`^([\n])?>>>(.*)`) 608 src = prefixRegexp.ReplaceAllString(src, "$1$2") 609 // append > to start of line 610 appendRegexp := regexp.MustCompile(`(?m)^`) 611 return appendRegexp.ReplaceAllString(src, ">$0") 612 }, 613 }, 614 } 615 616 for channelName, channelPosts := range posts { 617 for postIdx, post := range channelPosts { 618 result := post.Text 619 620 for _, rule := range regexReplaceAllString { 621 result = rule.regex.ReplaceAllString(result, rule.rpl) 622 } 623 624 for _, rule := range regexReplaceAllStringFunc { 625 result = rule.regex.ReplaceAllStringFunc(result, rule.fn) 626 } 627 posts[channelName][postIdx].Text = result 628 } 629 } 630 631 return posts 632 } 633 634 func (a *App) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) { 635 // Create log file 636 log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log")) 637 638 zipreader, err := zip.NewReader(fileData, fileSize) 639 if err != nil || zipreader.File == nil { 640 log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error")) 641 return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error(), http.StatusBadRequest), log 642 } 643 644 var channels []SlackChannel 645 var users []SlackUser 646 posts := make(map[string][]SlackPost) 647 uploads := make(map[string]*zip.File) 648 for _, file := range zipreader.File { 649 reader, err := file.Open() 650 if err != nil { 651 log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name})) 652 return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error(), http.StatusInternalServerError), log 653 } 654 if file.Name == "channels.json" { 655 channels, _ = SlackParseChannels(reader) 656 } else if file.Name == "users.json" { 657 users, _ = SlackParseUsers(reader) 658 } else { 659 spl := strings.Split(file.Name, "/") 660 if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") { 661 newposts, _ := SlackParsePosts(reader) 662 channel := spl[0] 663 if _, ok := posts[channel]; !ok { 664 posts[channel] = newposts 665 } else { 666 posts[channel] = append(posts[channel], newposts...) 667 } 668 } else if len(spl) == 3 && spl[0] == "__uploads" { 669 uploads[spl[1]] = file 670 } 671 } 672 } 673 674 posts = SlackConvertUserMentions(users, posts) 675 posts = SlackConvertChannelMentions(channels, posts) 676 posts = SlackConvertPostsMarkup(posts) 677 678 addedUsers := a.SlackAddUsers(teamID, users, log) 679 botUser := a.SlackAddBotUser(teamID, log) 680 681 a.SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log) 682 683 if botUser != nil { 684 a.deactivateSlackBotUser(botUser) 685 } 686 687 a.InvalidateAllCaches() 688 689 log.WriteString(utils.T("api.slackimport.slack_import.notes")) 690 log.WriteString("=======\r\n\r\n") 691 692 log.WriteString(utils.T("api.slackimport.slack_import.note1")) 693 log.WriteString(utils.T("api.slackimport.slack_import.note2")) 694 log.WriteString(utils.T("api.slackimport.slack_import.note3")) 695 696 return nil, log 697 } 698 699 // 700 // -- Old SlackImport Functions -- 701 // Import functions are sutible for entering posts and users into the database without 702 // some of the usual checks. (IsValid is still run) 703 // 704 705 func (a *App) OldImportPost(post *model.Post) { 706 // Workaround for empty messages, which may be the case if they are webhook posts. 707 firstIteration := true 708 maxPostSize := a.MaxPostSize() 709 for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) { 710 firstIteration = false 711 var remainder string 712 if messageRuneCount > maxPostSize { 713 remainder = string(([]rune(post.Message))[maxPostSize:]) 714 post.Message = truncateRunes(post.Message, maxPostSize) 715 } else { 716 remainder = "" 717 } 718 719 post.Hashtags, _ = model.ParseHashtags(post.Message) 720 721 if result := <-a.Srv.Store.Post().Save(post); result.Err != nil { 722 mlog.Debug(fmt.Sprintf("Error saving post. user=%v, message=%v", post.UserId, post.Message)) 723 } 724 725 for _, fileId := range post.FileIds { 726 if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); result.Err != nil { 727 mlog.Error(fmt.Sprintf("Error attaching files to post. postId=%v, fileIds=%v, message=%v", post.Id, post.FileIds, result.Err), mlog.String("post_id", post.Id)) 728 } 729 } 730 731 post.Id = "" 732 post.CreateAt++ 733 post.Message = remainder 734 } 735 } 736 737 func (a *App) OldImportUser(team *model.Team, user *model.User) *model.User { 738 user.MakeNonNil() 739 740 user.Roles = model.SYSTEM_USER_ROLE_ID 741 742 result := <-a.Srv.Store.User().Save(user) 743 if result.Err != nil { 744 mlog.Error(fmt.Sprintf("Error saving user. err=%v", result.Err)) 745 return nil 746 } 747 ruser := result.Data.(*model.User) 748 749 if cresult := <-a.Srv.Store.User().VerifyEmail(ruser.Id, ruser.Email); cresult.Err != nil { 750 mlog.Error(fmt.Sprintf("Failed to set email verified err=%v", cresult.Err)) 751 } 752 753 if err := a.JoinUserToTeam(team, user, ""); err != nil { 754 mlog.Error(fmt.Sprintf("Failed to join team when importing err=%v", err)) 755 } 756 757 return ruser 758 } 759 760 func (a *App) OldImportChannel(channel *model.Channel) *model.Channel { 761 result := <-a.Srv.Store.Channel().Save(channel, *a.Config().TeamSettings.MaxChannelsPerTeam) 762 if result.Err != nil { 763 return nil 764 } 765 sc := result.Data.(*model.Channel) 766 767 return sc 768 } 769 770 func (a *App) OldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) { 771 buf := bytes.NewBuffer(nil) 772 io.Copy(buf, file) 773 data := buf.Bytes() 774 775 fileInfo, err := a.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data) 776 if err != nil { 777 return nil, err 778 } 779 780 if fileInfo.IsImage() && fileInfo.MimeType != "image/svg+xml" { 781 img, width, height := prepareImage(data) 782 if img != nil { 783 a.generateThumbnailImage(img, fileInfo.ThumbnailPath, width, height) 784 a.generatePreviewImage(img, fileInfo.PreviewPath, width) 785 } 786 } 787 788 return fileInfo, nil 789 } 790 791 func (a *App) OldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) { 792 linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`) 793 post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})") 794 795 post.AddProp("from_webhook", "true") 796 797 if _, ok := props["override_username"]; !ok { 798 post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME) 799 } 800 801 if len(props) > 0 { 802 for key, val := range props { 803 if key == "attachments" { 804 if attachments, success := val.([]*model.SlackAttachment); success { 805 model.ParseSlackAttachment(post, attachments) 806 } 807 } else if key != "from_webhook" { 808 post.AddProp(key, val) 809 } 810 } 811 } 812 813 a.OldImportPost(post) 814 }