github.com/spline-fu/mattermost-server@v4.10.10+incompatible/app/slackimport.go (about)

     1  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
     2  // See License.txt for license information.
     3  
     4  package app
     5  
     6  import (
     7  	"archive/zip"
     8  	"bytes"
     9  	"encoding/json"
    10  	"fmt"
    11  	"io"
    12  	"mime/multipart"
    13  	"path/filepath"
    14  	"regexp"
    15  	"strconv"
    16  	"strings"
    17  	"unicode/utf8"
    18  
    19  	"net/http"
    20  
    21  	"github.com/mattermost/mattermost-server/mlog"
    22  	"github.com/mattermost/mattermost-server/model"
    23  	"github.com/mattermost/mattermost-server/utils"
    24  )
    25  
    26  type SlackChannel struct {
    27  	Id      string            `json:"id"`
    28  	Name    string            `json:"name"`
    29  	Members []string          `json:"members"`
    30  	Topic   map[string]string `json:"topic"`
    31  	Purpose map[string]string `json:"purpose"`
    32  }
    33  
    34  type SlackProfile struct {
    35  	FirstName string `json:"first_name"`
    36  	LastName  string `json:"last_name"`
    37  	Email     string `json:"email"`
    38  }
    39  
    40  type SlackUser struct {
    41  	Id       string       `json:"id"`
    42  	Username string       `json:"name"`
    43  	Profile  SlackProfile `json:"profile"`
    44  }
    45  
    46  type SlackFile struct {
    47  	Id    string `json:"id"`
    48  	Title string `json:"title"`
    49  }
    50  
    51  type SlackPost struct {
    52  	User        string                   `json:"user"`
    53  	BotId       string                   `json:"bot_id"`
    54  	BotUsername string                   `json:"username"`
    55  	Text        string                   `json:"text"`
    56  	TimeStamp   string                   `json:"ts"`
    57  	Type        string                   `json:"type"`
    58  	SubType     string                   `json:"subtype"`
    59  	Comment     *SlackComment            `json:"comment"`
    60  	Upload      bool                     `json:"upload"`
    61  	File        *SlackFile               `json:"file"`
    62  	Attachments []*model.SlackAttachment `json:"attachments"`
    63  }
    64  
    65  var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString
    66  
    67  type SlackComment struct {
    68  	User    string `json:"user"`
    69  	Comment string `json:"comment"`
    70  }
    71  
    72  func truncateRunes(s string, i int) string {
    73  	runes := []rune(s)
    74  	if len(runes) > i {
    75  		return string(runes[:i])
    76  	}
    77  	return s
    78  }
    79  
    80  func SlackConvertTimeStamp(ts string) int64 {
    81  	timeString := strings.SplitN(ts, ".", 2)[0]
    82  
    83  	timeStamp, err := strconv.ParseInt(timeString, 10, 64)
    84  	if err != nil {
    85  		mlog.Warn("Slack Import: Bad timestamp detected.")
    86  		return 1
    87  	}
    88  	return timeStamp * 1000 // Convert to milliseconds
    89  }
    90  
    91  func SlackConvertChannelName(channelName string, channelId string) string {
    92  	newName := strings.Trim(channelName, "_-")
    93  	if len(newName) == 1 {
    94  		return "slack-channel-" + newName
    95  	}
    96  
    97  	if isValidChannelNameCharacters(newName) {
    98  		return newName
    99  	} else {
   100  		return strings.ToLower(channelId)
   101  	}
   102  }
   103  
   104  func SlackParseChannels(data io.Reader) ([]SlackChannel, error) {
   105  	decoder := json.NewDecoder(data)
   106  
   107  	var channels []SlackChannel
   108  	if err := decoder.Decode(&channels); err != nil {
   109  		mlog.Warn("Slack Import: Error occurred when parsing some Slack channels. Import may work anyway.")
   110  		return channels, err
   111  	}
   112  	return channels, nil
   113  }
   114  
   115  func SlackParseUsers(data io.Reader) ([]SlackUser, error) {
   116  	decoder := json.NewDecoder(data)
   117  
   118  	var users []SlackUser
   119  	err := decoder.Decode(&users)
   120  	// This actually returns errors that are ignored.
   121  	// In this case it is erroring because of a null that Slack
   122  	// introduced. So we just return the users here.
   123  	return users, err
   124  }
   125  
   126  func SlackParsePosts(data io.Reader) ([]SlackPost, error) {
   127  	decoder := json.NewDecoder(data)
   128  
   129  	var posts []SlackPost
   130  	if err := decoder.Decode(&posts); err != nil {
   131  		mlog.Warn("Slack Import: Error occurred when parsing some Slack posts. Import may work anyway.")
   132  		return posts, err
   133  	}
   134  	return posts, nil
   135  }
   136  
   137  func (a *App) SlackAddUsers(teamId string, slackusers []SlackUser, importerLog *bytes.Buffer) map[string]*model.User {
   138  	// Log header
   139  	importerLog.WriteString(utils.T("api.slackimport.slack_add_users.created"))
   140  	importerLog.WriteString("===============\r\n\r\n")
   141  
   142  	addedUsers := make(map[string]*model.User)
   143  
   144  	// Need the team
   145  	var team *model.Team
   146  	if result := <-a.Srv.Store.Team().Get(teamId); result.Err != nil {
   147  		importerLog.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
   148  		return addedUsers
   149  	} else {
   150  		team = result.Data.(*model.Team)
   151  	}
   152  
   153  	for _, sUser := range slackusers {
   154  		firstName := sUser.Profile.FirstName
   155  		lastName := sUser.Profile.LastName
   156  		email := sUser.Profile.Email
   157  		if email == "" {
   158  			email = sUser.Username + "@example.com"
   159  			importerLog.WriteString(utils.T("api.slackimport.slack_add_users.missing_email_address", map[string]interface{}{"Email": email, "Username": sUser.Username}))
   160  			mlog.Warn("Slack Import: User {{.Username}} does not have an email address in the Slack export. Used {{.Email}} as a placeholder. The user should update their email address once logged in to the system.")
   161  		}
   162  
   163  		password := model.NewId()
   164  
   165  		// Check for email conflict and use existing user if found
   166  		if result := <-a.Srv.Store.User().GetByEmail(email); result.Err == nil {
   167  			existingUser := result.Data.(*model.User)
   168  			addedUsers[sUser.Id] = existingUser
   169  			if err := a.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil {
   170  				importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
   171  			} else {
   172  				importerLog.WriteString(utils.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
   173  			}
   174  			continue
   175  		}
   176  
   177  		newUser := model.User{
   178  			Username:  sUser.Username,
   179  			FirstName: firstName,
   180  			LastName:  lastName,
   181  			Email:     email,
   182  			Password:  password,
   183  		}
   184  
   185  		if mUser := a.OldImportUser(team, &newUser); mUser != nil {
   186  			addedUsers[sUser.Id] = mUser
   187  			importerLog.WriteString(utils.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password}))
   188  		} else {
   189  			importerLog.WriteString(utils.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username}))
   190  		}
   191  	}
   192  
   193  	return addedUsers
   194  }
   195  
   196  func (a *App) SlackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
   197  	var team *model.Team
   198  	if result := <-a.Srv.Store.Team().Get(teamId); result.Err != nil {
   199  		log.WriteString(utils.T("api.slackimport.slack_import.team_fail"))
   200  		return nil
   201  	} else {
   202  		team = result.Data.(*model.Team)
   203  	}
   204  
   205  	password := model.NewId()
   206  	username := "slackimportuser_" + model.NewId()
   207  	email := username + "@localhost"
   208  
   209  	botUser := model.User{
   210  		Username:  username,
   211  		FirstName: "",
   212  		LastName:  "",
   213  		Email:     email,
   214  		Password:  password,
   215  	}
   216  
   217  	if mUser := a.OldImportUser(team, &botUser); mUser != nil {
   218  		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password}))
   219  		return mUser
   220  	} else {
   221  		log.WriteString(utils.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username}))
   222  		return nil
   223  	}
   224  }
   225  
   226  func (a *App) SlackAddPosts(teamId string, channel *model.Channel, posts []SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
   227  	for _, sPost := range posts {
   228  		switch {
   229  		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
   230  			if sPost.User == "" {
   231  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   232  				continue
   233  			} else if users[sPost.User] == nil {
   234  				mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User))
   235  				continue
   236  			}
   237  			newPost := model.Post{
   238  				UserId:    users[sPost.User].Id,
   239  				ChannelId: channel.Id,
   240  				Message:   sPost.Text,
   241  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   242  			}
   243  			if sPost.Upload {
   244  				if fileInfo, ok := a.SlackUploadFile(sPost, uploads, teamId, newPost.ChannelId, newPost.UserId); ok {
   245  					newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
   246  					newPost.Message = sPost.File.Title
   247  				}
   248  			}
   249  			a.OldImportPost(&newPost)
   250  			for _, fileId := range newPost.FileIds {
   251  				if result := <-a.Srv.Store.FileInfo().AttachToPost(fileId, newPost.Id); result.Err != nil {
   252  					mlog.Error(fmt.Sprintf("Slack Import: An error occurred when attaching files to a message, post_id=%s, file_ids=%v, err=%v.", newPost.Id, newPost.FileIds, result.Err))
   253  				}
   254  			}
   255  
   256  		case sPost.Type == "message" && sPost.SubType == "file_comment":
   257  			if sPost.Comment == nil {
   258  				mlog.Debug("Slack Import: Unable to import the message as it has no comments.")
   259  				continue
   260  			} else if sPost.Comment.User == "" {
   261  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   262  				continue
   263  			} else if users[sPost.Comment.User] == nil {
   264  				mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User))
   265  				continue
   266  			}
   267  			newPost := model.Post{
   268  				UserId:    users[sPost.Comment.User].Id,
   269  				ChannelId: channel.Id,
   270  				Message:   sPost.Comment.Comment,
   271  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   272  			}
   273  			a.OldImportPost(&newPost)
   274  		case sPost.Type == "message" && sPost.SubType == "bot_message":
   275  			if botUser == nil {
   276  				mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.")
   277  				continue
   278  			} else if sPost.BotId == "" {
   279  				mlog.Warn("Slack Import: Unable to import bot message as the BotId field is missing.")
   280  				continue
   281  			}
   282  
   283  			props := make(model.StringInterface)
   284  			props["override_username"] = sPost.BotUsername
   285  			if len(sPost.Attachments) > 0 {
   286  				props["attachments"] = sPost.Attachments
   287  			}
   288  
   289  			post := &model.Post{
   290  				UserId:    botUser.Id,
   291  				ChannelId: channel.Id,
   292  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   293  				Message:   sPost.Text,
   294  				Type:      model.POST_SLACK_ATTACHMENT,
   295  			}
   296  
   297  			a.OldImportIncomingWebhookPost(post, props)
   298  		case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
   299  			if sPost.User == "" {
   300  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   301  				continue
   302  			} else if users[sPost.User] == nil {
   303  				mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User))
   304  				continue
   305  			}
   306  
   307  			var postType string
   308  			if sPost.SubType == "channel_join" {
   309  				postType = model.POST_JOIN_CHANNEL
   310  			} else {
   311  				postType = model.POST_LEAVE_CHANNEL
   312  			}
   313  
   314  			newPost := model.Post{
   315  				UserId:    users[sPost.User].Id,
   316  				ChannelId: channel.Id,
   317  				Message:   sPost.Text,
   318  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   319  				Type:      postType,
   320  				Props: model.StringInterface{
   321  					"username": users[sPost.User].Username,
   322  				},
   323  			}
   324  			a.OldImportPost(&newPost)
   325  		case sPost.Type == "message" && sPost.SubType == "me_message":
   326  			if sPost.User == "" {
   327  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   328  				continue
   329  			} else if users[sPost.User] == nil {
   330  				mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User))
   331  				continue
   332  			}
   333  			newPost := model.Post{
   334  				UserId:    users[sPost.User].Id,
   335  				ChannelId: channel.Id,
   336  				Message:   "*" + sPost.Text + "*",
   337  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   338  			}
   339  			a.OldImportPost(&newPost)
   340  		case sPost.Type == "message" && sPost.SubType == "channel_topic":
   341  			if sPost.User == "" {
   342  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   343  				continue
   344  			} else if users[sPost.User] == nil {
   345  				mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User))
   346  				continue
   347  			}
   348  			newPost := model.Post{
   349  				UserId:    users[sPost.User].Id,
   350  				ChannelId: channel.Id,
   351  				Message:   sPost.Text,
   352  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   353  				Type:      model.POST_HEADER_CHANGE,
   354  			}
   355  			a.OldImportPost(&newPost)
   356  		case sPost.Type == "message" && sPost.SubType == "channel_purpose":
   357  			if sPost.User == "" {
   358  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   359  				continue
   360  			} else if users[sPost.User] == nil {
   361  				mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User))
   362  				continue
   363  			}
   364  			newPost := model.Post{
   365  				UserId:    users[sPost.User].Id,
   366  				ChannelId: channel.Id,
   367  				Message:   sPost.Text,
   368  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   369  				Type:      model.POST_PURPOSE_CHANGE,
   370  			}
   371  			a.OldImportPost(&newPost)
   372  		case sPost.Type == "message" && sPost.SubType == "channel_name":
   373  			if sPost.User == "" {
   374  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   375  				continue
   376  			} else if users[sPost.User] == nil {
   377  				mlog.Debug(fmt.Sprintf("Slack Import: Unable to add the message as the Slack user %v does not exist in Mattermost.", sPost.User))
   378  				continue
   379  			}
   380  			newPost := model.Post{
   381  				UserId:    users[sPost.User].Id,
   382  				ChannelId: channel.Id,
   383  				Message:   sPost.Text,
   384  				CreateAt:  SlackConvertTimeStamp(sPost.TimeStamp),
   385  				Type:      model.POST_DISPLAYNAME_CHANGE,
   386  			}
   387  			a.OldImportPost(&newPost)
   388  		default:
   389  			mlog.Warn(fmt.Sprintf("Slack Import: Unable to import the message as its type is not supported: post_type=%v, post_subtype=%v.", sPost.Type, sPost.SubType))
   390  		}
   391  	}
   392  }
   393  
   394  func (a *App) SlackUploadFile(sPost SlackPost, uploads map[string]*zip.File, teamId string, channelId string, userId string) (*model.FileInfo, bool) {
   395  	if sPost.File != nil {
   396  		if file, ok := uploads[sPost.File.Id]; ok {
   397  			openFile, err := file.Open()
   398  			if err != nil {
   399  				mlog.Warn("Slack Import: Unable to open the file {{.FileId}} from the Slack export: {{.Error}}.")
   400  				return nil, false
   401  			}
   402  			defer openFile.Close()
   403  
   404  			timestamp := utils.TimeFromMillis(SlackConvertTimeStamp(sPost.TimeStamp))
   405  			uploadedFile, err := a.OldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name))
   406  			if err != nil {
   407  				mlog.Warn("Slack Import: An error occurred when uploading file {{.FileId}}: {{.Error}}.")
   408  				return nil, false
   409  			}
   410  
   411  			return uploadedFile, true
   412  		} else {
   413  			mlog.Warn("Slack Import: Unable to import file {{.FileId}} as the file is missing from the Slack export zip file.")
   414  			return nil, false
   415  		}
   416  	} else {
   417  		mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.")
   418  		return nil, false
   419  	}
   420  }
   421  
   422  func (a *App) deactivateSlackBotUser(user *model.User) {
   423  	_, err := a.UpdateActive(user, false)
   424  	if err != nil {
   425  		mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.")
   426  	}
   427  }
   428  
   429  func (a *App) addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
   430  	for _, member := range members {
   431  		if user, ok := users[member]; !ok {
   432  			log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
   433  		} else {
   434  			if _, err := a.AddUserToChannel(user, channel); err != nil {
   435  				log.WriteString(utils.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
   436  			}
   437  		}
   438  	}
   439  }
   440  
   441  func SlackSanitiseChannelProperties(channel model.Channel) model.Channel {
   442  	if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES {
   443  		mlog.Warn(fmt.Sprint("api.slackimport.slack_sanitise_channel_properties.display_name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}))
   444  		channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES)
   445  	}
   446  
   447  	if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH {
   448  		mlog.Warn(fmt.Sprint("api.slackimport.slack_sanitise_channel_properties.name_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}))
   449  		channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH]
   450  	}
   451  
   452  	if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES {
   453  		mlog.Warn(fmt.Sprint("api.slackimport.slack_sanitise_channel_properties.purpose_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}))
   454  		channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES)
   455  	}
   456  
   457  	if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES {
   458  		mlog.Warn(fmt.Sprint("api.slackimport.slack_sanitise_channel_properties.header_too_long.warn", map[string]interface{}{"ChannelName": channel.DisplayName}))
   459  		channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES)
   460  	}
   461  
   462  	return channel
   463  }
   464  
   465  func (a *App) SlackAddChannels(teamId string, slackchannels []SlackChannel, posts map[string][]SlackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel {
   466  	// Write Header
   467  	importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.added"))
   468  	importerLog.WriteString("=================\r\n\r\n")
   469  
   470  	addedChannels := make(map[string]*model.Channel)
   471  	for _, sChannel := range slackchannels {
   472  		newChannel := model.Channel{
   473  			TeamId:      teamId,
   474  			Type:        model.CHANNEL_OPEN,
   475  			DisplayName: sChannel.Name,
   476  			Name:        SlackConvertChannelName(sChannel.Name, sChannel.Id),
   477  			Purpose:     sChannel.Purpose["value"],
   478  			Header:      sChannel.Topic["value"],
   479  		}
   480  		newChannel = SlackSanitiseChannelProperties(newChannel)
   481  
   482  		var mChannel *model.Channel
   483  		if result := <-a.Srv.Store.Channel().GetByName(teamId, sChannel.Name, true); result.Err == nil {
   484  			// The channel already exists as an active channel. Merge with the existing one.
   485  			mChannel = result.Data.(*model.Channel)
   486  			importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
   487  		} else if result := <-a.Srv.Store.Channel().GetDeletedByName(teamId, sChannel.Name); result.Err == nil {
   488  			// The channel already exists but has been deleted. Generate a random string for the handle instead.
   489  			newChannel.Name = model.NewId()
   490  			newChannel = SlackSanitiseChannelProperties(newChannel)
   491  		}
   492  
   493  		if mChannel == nil {
   494  			// Haven't found an existing channel to merge with. Try importing it as a new one.
   495  			mChannel = a.OldImportChannel(&newChannel)
   496  			if mChannel == nil {
   497  				mlog.Warn(fmt.Sprintf("Slack Import: Unable to import Slack channel: %s.", newChannel.DisplayName))
   498  				importerLog.WriteString(utils.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
   499  				continue
   500  			}
   501  		}
   502  
   503  		a.addSlackUsersToChannel(sChannel.Members, users, mChannel, importerLog)
   504  		importerLog.WriteString(newChannel.DisplayName + "\r\n")
   505  		addedChannels[sChannel.Id] = mChannel
   506  		a.SlackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
   507  	}
   508  
   509  	return addedChannels
   510  }
   511  
   512  func SlackConvertUserMentions(users []SlackUser, posts map[string][]SlackPost) map[string][]SlackPost {
   513  	var regexes = make(map[string]*regexp.Regexp, len(users))
   514  	for _, user := range users {
   515  		r, err := regexp.Compile("<@" + user.Id + `(\|` + user.Username + ")?>")
   516  		if err != nil {
   517  			mlog.Warn(fmt.Sprint("Slack Import: Unable to compile the @mention, matching regular expression for the Slack user {{.Username}} (id={{.UserID}}).", user.Id, user.Username), mlog.String("user_id", user.Id))
   518  			continue
   519  		}
   520  		regexes["@"+user.Username] = r
   521  	}
   522  
   523  	// Special cases.
   524  	regexes["@here"], _ = regexp.Compile(`<!here\|@here>`)
   525  	regexes["@channel"], _ = regexp.Compile("<!channel>")
   526  	regexes["@all"], _ = regexp.Compile("<!everyone>")
   527  
   528  	for channelName, channelPosts := range posts {
   529  		for postIdx, post := range channelPosts {
   530  			for mention, r := range regexes {
   531  				post.Text = r.ReplaceAllString(post.Text, mention)
   532  				posts[channelName][postIdx] = post
   533  			}
   534  		}
   535  	}
   536  
   537  	return posts
   538  }
   539  
   540  func SlackConvertChannelMentions(channels []SlackChannel, posts map[string][]SlackPost) map[string][]SlackPost {
   541  	var regexes = make(map[string]*regexp.Regexp, len(channels))
   542  	for _, channel := range channels {
   543  		r, err := regexp.Compile("<#" + channel.Id + `(\|` + channel.Name + ")?>")
   544  		if err != nil {
   545  			mlog.Warn(fmt.Sprint("Slack Import: Unable to compile the !channel, matching regular expression for the Slack channel {{.ChannelName}} (id={{.ChannelID}}).", channel.Id, channel.Name))
   546  			continue
   547  		}
   548  		regexes["~"+channel.Name] = r
   549  	}
   550  
   551  	for channelName, channelPosts := range posts {
   552  		for postIdx, post := range channelPosts {
   553  			for channelReplace, r := range regexes {
   554  				post.Text = r.ReplaceAllString(post.Text, channelReplace)
   555  				posts[channelName][postIdx] = post
   556  			}
   557  		}
   558  	}
   559  
   560  	return posts
   561  }
   562  
   563  func SlackConvertPostsMarkup(posts map[string][]SlackPost) map[string][]SlackPost {
   564  	regexReplaceAllString := []struct {
   565  		regex *regexp.Regexp
   566  		rpl   string
   567  	}{
   568  		// URL
   569  		{
   570  			regexp.MustCompile(`<([^|<>]+)\|([^|<>]+)>`),
   571  			"[$2]($1)",
   572  		},
   573  		// bold
   574  		{
   575  			regexp.MustCompile(`(^|[\s.;,])\*(\S[^*\n]+)\*`),
   576  			"$1**$2**",
   577  		},
   578  		// strikethrough
   579  		{
   580  			regexp.MustCompile(`(^|[\s.;,])\~(\S[^~\n]+)\~`),
   581  			"$1~~$2~~",
   582  		},
   583  		// single paragraph blockquote
   584  		// Slack converts > character to &gt;
   585  		{
   586  			regexp.MustCompile(`(?sm)^&gt;`),
   587  			">",
   588  		},
   589  	}
   590  
   591  	regexReplaceAllStringFunc := []struct {
   592  		regex *regexp.Regexp
   593  		fn    func(string) string
   594  	}{
   595  		// multiple paragraphs blockquotes
   596  		{
   597  			regexp.MustCompile(`(?sm)^>&gt;&gt;(.+)$`),
   598  			func(src string) string {
   599  				// remove >>> prefix, might have leading \n
   600  				prefixRegexp := regexp.MustCompile(`^([\n])?>&gt;&gt;(.*)`)
   601  				src = prefixRegexp.ReplaceAllString(src, "$1$2")
   602  				// append > to start of line
   603  				appendRegexp := regexp.MustCompile(`(?m)^`)
   604  				return appendRegexp.ReplaceAllString(src, ">$0")
   605  			},
   606  		},
   607  	}
   608  
   609  	for channelName, channelPosts := range posts {
   610  		for postIdx, post := range channelPosts {
   611  			result := post.Text
   612  
   613  			for _, rule := range regexReplaceAllString {
   614  				result = rule.regex.ReplaceAllString(result, rule.rpl)
   615  			}
   616  
   617  			for _, rule := range regexReplaceAllStringFunc {
   618  				result = rule.regex.ReplaceAllStringFunc(result, rule.fn)
   619  			}
   620  			posts[channelName][postIdx].Text = result
   621  		}
   622  	}
   623  
   624  	return posts
   625  }
   626  
   627  func (a *App) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
   628  	// Create log file
   629  	log := bytes.NewBufferString(utils.T("api.slackimport.slack_import.log"))
   630  
   631  	zipreader, err := zip.NewReader(fileData, fileSize)
   632  	if err != nil || zipreader.File == nil {
   633  		log.WriteString(utils.T("api.slackimport.slack_import.zip.app_error"))
   634  		return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error(), http.StatusBadRequest), log
   635  	}
   636  
   637  	var channels []SlackChannel
   638  	var users []SlackUser
   639  	posts := make(map[string][]SlackPost)
   640  	uploads := make(map[string]*zip.File)
   641  	for _, file := range zipreader.File {
   642  		reader, err := file.Open()
   643  		if err != nil {
   644  			log.WriteString(utils.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}))
   645  			return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error(), http.StatusInternalServerError), log
   646  		}
   647  		if file.Name == "channels.json" {
   648  			channels, _ = SlackParseChannels(reader)
   649  		} else if file.Name == "users.json" {
   650  			users, _ = SlackParseUsers(reader)
   651  		} else {
   652  			spl := strings.Split(file.Name, "/")
   653  			if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
   654  				newposts, _ := SlackParsePosts(reader)
   655  				channel := spl[0]
   656  				if _, ok := posts[channel]; !ok {
   657  					posts[channel] = newposts
   658  				} else {
   659  					posts[channel] = append(posts[channel], newposts...)
   660  				}
   661  			} else if len(spl) == 3 && spl[0] == "__uploads" {
   662  				uploads[spl[1]] = file
   663  			}
   664  		}
   665  	}
   666  
   667  	posts = SlackConvertUserMentions(users, posts)
   668  	posts = SlackConvertChannelMentions(channels, posts)
   669  	posts = SlackConvertPostsMarkup(posts)
   670  
   671  	addedUsers := a.SlackAddUsers(teamID, users, log)
   672  	botUser := a.SlackAddBotUser(teamID, log)
   673  
   674  	a.SlackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log)
   675  
   676  	if botUser != nil {
   677  		a.deactivateSlackBotUser(botUser)
   678  	}
   679  
   680  	a.InvalidateAllCaches()
   681  
   682  	log.WriteString(utils.T("api.slackimport.slack_import.notes"))
   683  	log.WriteString("=======\r\n\r\n")
   684  
   685  	log.WriteString(utils.T("api.slackimport.slack_import.note1"))
   686  	log.WriteString(utils.T("api.slackimport.slack_import.note2"))
   687  	log.WriteString(utils.T("api.slackimport.slack_import.note3"))
   688  
   689  	return nil, log
   690  }