github.com/masterhung0112/hk_server/v5@v5.0.0-20220302090640-ec71aef15e1c/services/slackimport/slackimport.go (about)

     1  // Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
     2  // See LICENSE.txt for license information.
     3  
     4  package slackimport
     5  
     6  import (
     7  	"archive/zip"
     8  	"bytes"
     9  	"image"
    10  	"io"
    11  	"mime/multipart"
    12  	"net/http"
    13  	"path/filepath"
    14  	"regexp"
    15  	"sort"
    16  	"strings"
    17  	"time"
    18  	"unicode/utf8"
    19  
    20  	"github.com/masterhung0112/hk_server/v5/model"
    21  	"github.com/masterhung0112/hk_server/v5/shared/i18n"
    22  	"github.com/masterhung0112/hk_server/v5/shared/mlog"
    23  	"github.com/masterhung0112/hk_server/v5/store"
    24  	"github.com/masterhung0112/hk_server/v5/utils"
    25  )
    26  
    27  type slackChannel struct {
    28  	Id      string          `json:"id"`
    29  	Name    string          `json:"name"`
    30  	Creator string          `json:"creator"`
    31  	Members []string        `json:"members"`
    32  	Purpose slackChannelSub `json:"purpose"`
    33  	Topic   slackChannelSub `json:"topic"`
    34  	Type    string
    35  }
    36  
    37  type slackChannelSub struct {
    38  	Value string `json:"value"`
    39  }
    40  
    41  type slackProfile struct {
    42  	FirstName string `json:"first_name"`
    43  	LastName  string `json:"last_name"`
    44  	Email     string `json:"email"`
    45  }
    46  
    47  type slackUser struct {
    48  	Id       string       `json:"id"`
    49  	Username string       `json:"name"`
    50  	Profile  slackProfile `json:"profile"`
    51  }
    52  
    53  type slackFile struct {
    54  	Id    string `json:"id"`
    55  	Title string `json:"title"`
    56  }
    57  
    58  type slackPost struct {
    59  	User        string                   `json:"user"`
    60  	BotId       string                   `json:"bot_id"`
    61  	BotUsername string                   `json:"username"`
    62  	Text        string                   `json:"text"`
    63  	TimeStamp   string                   `json:"ts"`
    64  	ThreadTS    string                   `json:"thread_ts"`
    65  	Type        string                   `json:"type"`
    66  	SubType     string                   `json:"subtype"`
    67  	Comment     *slackComment            `json:"comment"`
    68  	Upload      bool                     `json:"upload"`
    69  	File        *slackFile               `json:"file"`
    70  	Files       []*slackFile             `json:"files"`
    71  	Attachments []*model.SlackAttachment `json:"attachments"`
    72  }
    73  
    74  var isValidChannelNameCharacters = regexp.MustCompile(`^[a-zA-Z0-9\-_]+$`).MatchString
    75  
    76  const slackImportMaxFileSize = 1024 * 1024 * 70
    77  
    78  type slackComment struct {
    79  	User    string `json:"user"`
    80  	Comment string `json:"comment"`
    81  }
    82  
    83  // Actions provides the actions that needs to be used for import slack data
    84  type Actions struct {
    85  	UpdateActive           func(*model.User, bool) (*model.User, *model.AppError)
    86  	AddUserToChannel       func(*model.User, *model.Channel, bool) (*model.ChannelMember, *model.AppError)
    87  	JoinUserToTeam         func(*model.Team, *model.User, string) (*model.TeamMember, *model.AppError)
    88  	CreateDirectChannel    func(string, string, ...model.ChannelOption) (*model.Channel, *model.AppError)
    89  	CreateGroupChannel     func([]string) (*model.Channel, *model.AppError)
    90  	CreateChannel          func(*model.Channel, bool) (*model.Channel, *model.AppError)
    91  	DoUploadFile           func(time.Time, string, string, string, string, []byte) (*model.FileInfo, *model.AppError)
    92  	GenerateThumbnailImage func(image.Image, string)
    93  	GeneratePreviewImage   func(image.Image, string)
    94  	InvalidateAllCaches    func()
    95  	MaxPostSize            func() int
    96  	PrepareImage           func(fileData []byte) (image.Image, func(), error)
    97  }
    98  
    99  // SlackImporter is a service that allows to import slack dumps into mattermost
   100  type SlackImporter struct {
   101  	store   store.Store
   102  	actions Actions
   103  	config  *model.Config
   104  }
   105  
   106  // New creates a new SlackImporter service instance. It receive a store, a set of actions and the current config.
   107  // It is expected to be used right away and discarded after that
   108  func New(store store.Store, actions Actions, config *model.Config) *SlackImporter {
   109  	return &SlackImporter{
   110  		store:   store,
   111  		actions: actions,
   112  		config:  config,
   113  	}
   114  }
   115  
   116  func (si *SlackImporter) SlackImport(fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer) {
   117  	// Create log file
   118  	log := bytes.NewBufferString(i18n.T("api.slackimport.slack_import.log"))
   119  
   120  	zipreader, err := zip.NewReader(fileData, fileSize)
   121  	if err != nil || zipreader.File == nil {
   122  		log.WriteString(i18n.T("api.slackimport.slack_import.zip.app_error"))
   123  		return model.NewAppError("SlackImport", "api.slackimport.slack_import.zip.app_error", nil, err.Error(), http.StatusBadRequest), log
   124  	}
   125  
   126  	var channels []slackChannel
   127  	var publicChannels []slackChannel
   128  	var privateChannels []slackChannel
   129  	var groupChannels []slackChannel
   130  	var directChannels []slackChannel
   131  
   132  	var users []slackUser
   133  	posts := make(map[string][]slackPost)
   134  	uploads := make(map[string]*zip.File)
   135  	for _, file := range zipreader.File {
   136  		if file.UncompressedSize64 > slackImportMaxFileSize {
   137  			log.WriteString(i18n.T("api.slackimport.slack_import.zip.file_too_large", map[string]interface{}{"Filename": file.Name}))
   138  			continue
   139  		}
   140  		reader, err := file.Open()
   141  		if err != nil {
   142  			log.WriteString(i18n.T("api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}))
   143  			return model.NewAppError("SlackImport", "api.slackimport.slack_import.open.app_error", map[string]interface{}{"Filename": file.Name}, err.Error(), http.StatusInternalServerError), log
   144  		}
   145  		if file.Name == "channels.json" {
   146  			publicChannels, _ = slackParseChannels(reader, model.CHANNEL_OPEN)
   147  			channels = append(channels, publicChannels...)
   148  		} else if file.Name == "dms.json" {
   149  			directChannels, _ = slackParseChannels(reader, model.CHANNEL_DIRECT)
   150  			channels = append(channels, directChannels...)
   151  		} else if file.Name == "groups.json" {
   152  			privateChannels, _ = slackParseChannels(reader, model.CHANNEL_PRIVATE)
   153  			channels = append(channels, privateChannels...)
   154  		} else if file.Name == "mpims.json" {
   155  			groupChannels, _ = slackParseChannels(reader, model.CHANNEL_GROUP)
   156  			channels = append(channels, groupChannels...)
   157  		} else if file.Name == "users.json" {
   158  			users, _ = slackParseUsers(reader)
   159  		} else {
   160  			spl := strings.Split(file.Name, "/")
   161  			if len(spl) == 2 && strings.HasSuffix(spl[1], ".json") {
   162  				newposts, _ := slackParsePosts(reader)
   163  				channel := spl[0]
   164  				if _, ok := posts[channel]; !ok {
   165  					posts[channel] = newposts
   166  				} else {
   167  					posts[channel] = append(posts[channel], newposts...)
   168  				}
   169  			} else if len(spl) == 3 && spl[0] == "__uploads" {
   170  				uploads[spl[1]] = file
   171  			}
   172  		}
   173  	}
   174  
   175  	posts = slackConvertUserMentions(users, posts)
   176  	posts = slackConvertChannelMentions(channels, posts)
   177  	posts = slackConvertPostsMarkup(posts)
   178  
   179  	addedUsers := si.slackAddUsers(teamID, users, log)
   180  	botUser := si.slackAddBotUser(teamID, log)
   181  
   182  	si.slackAddChannels(teamID, channels, posts, addedUsers, uploads, botUser, log)
   183  
   184  	if botUser != nil {
   185  		si.deactivateSlackBotUser(botUser)
   186  	}
   187  
   188  	si.actions.InvalidateAllCaches()
   189  
   190  	log.WriteString(i18n.T("api.slackimport.slack_import.notes"))
   191  	log.WriteString("=======\r\n\r\n")
   192  
   193  	log.WriteString(i18n.T("api.slackimport.slack_import.note1"))
   194  	log.WriteString(i18n.T("api.slackimport.slack_import.note2"))
   195  	log.WriteString(i18n.T("api.slackimport.slack_import.note3"))
   196  
   197  	return nil, log
   198  }
   199  
   200  func truncateRunes(s string, i int) string {
   201  	runes := []rune(s)
   202  	if len(runes) > i {
   203  		return string(runes[:i])
   204  	}
   205  	return s
   206  }
   207  
   208  func (si *SlackImporter) slackAddUsers(teamId string, slackusers []slackUser, importerLog *bytes.Buffer) map[string]*model.User {
   209  	// Log header
   210  	importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.created"))
   211  	importerLog.WriteString("===============\r\n\r\n")
   212  
   213  	addedUsers := make(map[string]*model.User)
   214  
   215  	// Need the team
   216  	team, err := si.store.Team().Get(teamId)
   217  	if err != nil {
   218  		importerLog.WriteString(i18n.T("api.slackimport.slack_import.team_fail"))
   219  		return addedUsers
   220  	}
   221  
   222  	for _, sUser := range slackusers {
   223  		firstName := sUser.Profile.FirstName
   224  		lastName := sUser.Profile.LastName
   225  		email := sUser.Profile.Email
   226  		if email == "" {
   227  			email = sUser.Username + "@example.com"
   228  			importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.missing_email_address", map[string]interface{}{"Email": email, "Username": sUser.Username}))
   229  			mlog.Warn("Slack Import: User does not have an email address in the Slack export. Used username as a placeholder. The user should update their email address once logged in to the system.", mlog.String("user_email", email), mlog.String("user_name", sUser.Username))
   230  		}
   231  
   232  		password := model.NewId()
   233  
   234  		// Check for email conflict and use existing user if found
   235  		if existingUser, err := si.store.User().GetByEmail(email); err == nil {
   236  			addedUsers[sUser.Id] = existingUser
   237  			if _, err := si.actions.JoinUserToTeam(team, addedUsers[sUser.Id], ""); err != nil {
   238  				importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing_failed", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
   239  			} else {
   240  				importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.merge_existing", map[string]interface{}{"Email": existingUser.Email, "Username": existingUser.Username}))
   241  			}
   242  			continue
   243  		}
   244  
   245  		email = strings.ToLower(email)
   246  		newUser := model.User{
   247  			Username:  sUser.Username,
   248  			FirstName: firstName,
   249  			LastName:  lastName,
   250  			Email:     email,
   251  			Password:  password,
   252  		}
   253  
   254  		mUser := si.oldImportUser(team, &newUser)
   255  		if mUser == nil {
   256  			importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.unable_import", map[string]interface{}{"Username": sUser.Username}))
   257  			continue
   258  		}
   259  		addedUsers[sUser.Id] = mUser
   260  		importerLog.WriteString(i18n.T("api.slackimport.slack_add_users.email_pwd", map[string]interface{}{"Email": newUser.Email, "Password": password}))
   261  	}
   262  
   263  	return addedUsers
   264  }
   265  
   266  func (si *SlackImporter) slackAddBotUser(teamId string, log *bytes.Buffer) *model.User {
   267  	team, err := si.store.Team().Get(teamId)
   268  	if err != nil {
   269  		log.WriteString(i18n.T("api.slackimport.slack_import.team_fail"))
   270  		return nil
   271  	}
   272  
   273  	password := model.NewId()
   274  	username := "slackimportuser_" + model.NewId()
   275  	email := username + "@localhost"
   276  
   277  	botUser := model.User{
   278  		Username:  username,
   279  		FirstName: "",
   280  		LastName:  "",
   281  		Email:     email,
   282  		Password:  password,
   283  	}
   284  
   285  	mUser := si.oldImportUser(team, &botUser)
   286  	if mUser == nil {
   287  		log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.unable_import", map[string]interface{}{"Username": username}))
   288  		return nil
   289  	}
   290  
   291  	log.WriteString(i18n.T("api.slackimport.slack_add_bot_user.email_pwd", map[string]interface{}{"Email": botUser.Email, "Password": password}))
   292  	return mUser
   293  }
   294  
   295  func (si *SlackImporter) slackAddPosts(teamId string, channel *model.Channel, posts []slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User) {
   296  	sort.Slice(posts, func(i, j int) bool {
   297  		return slackConvertTimeStamp(posts[i].TimeStamp) < slackConvertTimeStamp(posts[j].TimeStamp)
   298  	})
   299  	threads := make(map[string]string)
   300  	for _, sPost := range posts {
   301  		switch {
   302  		case sPost.Type == "message" && (sPost.SubType == "" || sPost.SubType == "file_share"):
   303  			if sPost.User == "" {
   304  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   305  				continue
   306  			}
   307  			if users[sPost.User] == nil {
   308  				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
   309  				continue
   310  			}
   311  			newPost := model.Post{
   312  				UserId:    users[sPost.User].Id,
   313  				ChannelId: channel.Id,
   314  				Message:   sPost.Text,
   315  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   316  			}
   317  			if sPost.Upload {
   318  				if sPost.File != nil {
   319  					if fileInfo, ok := si.slackUploadFile(sPost.File, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok {
   320  						newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
   321  					}
   322  				} else if sPost.Files != nil {
   323  					for _, file := range sPost.Files {
   324  						if fileInfo, ok := si.slackUploadFile(file, uploads, teamId, newPost.ChannelId, newPost.UserId, sPost.TimeStamp); ok {
   325  							newPost.FileIds = append(newPost.FileIds, fileInfo.Id)
   326  						}
   327  					}
   328  				}
   329  			}
   330  			// If post in thread
   331  			if sPost.ThreadTS != "" && sPost.ThreadTS != sPost.TimeStamp {
   332  				newPost.RootId = threads[sPost.ThreadTS]
   333  				newPost.ParentId = threads[sPost.ThreadTS]
   334  			}
   335  			postId := si.oldImportPost(&newPost)
   336  			// If post is thread starter
   337  			if sPost.ThreadTS == sPost.TimeStamp {
   338  				threads[sPost.ThreadTS] = postId
   339  			}
   340  		case sPost.Type == "message" && sPost.SubType == "file_comment":
   341  			if sPost.Comment == nil {
   342  				mlog.Debug("Slack Import: Unable to import the message as it has no comments.")
   343  				continue
   344  			}
   345  			if sPost.Comment.User == "" {
   346  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   347  				continue
   348  			}
   349  			if users[sPost.Comment.User] == nil {
   350  				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
   351  				continue
   352  			}
   353  			newPost := model.Post{
   354  				UserId:    users[sPost.Comment.User].Id,
   355  				ChannelId: channel.Id,
   356  				Message:   sPost.Comment.Comment,
   357  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   358  			}
   359  			si.oldImportPost(&newPost)
   360  		case sPost.Type == "message" && sPost.SubType == "bot_message":
   361  			if botUser == nil {
   362  				mlog.Warn("Slack Import: Unable to import the bot message as the bot user does not exist.")
   363  				continue
   364  			}
   365  			if sPost.BotId == "" {
   366  				mlog.Warn("Slack Import: Unable to import bot message as the BotId field is missing.")
   367  				continue
   368  			}
   369  
   370  			props := make(model.StringInterface)
   371  			props["override_username"] = sPost.BotUsername
   372  			if len(sPost.Attachments) > 0 {
   373  				props["attachments"] = sPost.Attachments
   374  			}
   375  
   376  			post := &model.Post{
   377  				UserId:    botUser.Id,
   378  				ChannelId: channel.Id,
   379  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   380  				Message:   sPost.Text,
   381  				Type:      model.POST_SLACK_ATTACHMENT,
   382  			}
   383  
   384  			postId := si.oldImportIncomingWebhookPost(post, props)
   385  			// If post is thread starter
   386  			if sPost.ThreadTS == sPost.TimeStamp {
   387  				threads[sPost.ThreadTS] = postId
   388  			}
   389  		case sPost.Type == "message" && (sPost.SubType == "channel_join" || sPost.SubType == "channel_leave"):
   390  			if sPost.User == "" {
   391  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   392  				continue
   393  			}
   394  			if users[sPost.User] == nil {
   395  				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
   396  				continue
   397  			}
   398  
   399  			var postType string
   400  			if sPost.SubType == "channel_join" {
   401  				postType = model.POST_JOIN_CHANNEL
   402  			} else {
   403  				postType = model.POST_LEAVE_CHANNEL
   404  			}
   405  
   406  			newPost := model.Post{
   407  				UserId:    users[sPost.User].Id,
   408  				ChannelId: channel.Id,
   409  				Message:   sPost.Text,
   410  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   411  				Type:      postType,
   412  				Props: model.StringInterface{
   413  					"username": users[sPost.User].Username,
   414  				},
   415  			}
   416  			si.oldImportPost(&newPost)
   417  		case sPost.Type == "message" && sPost.SubType == "me_message":
   418  			if sPost.User == "" {
   419  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   420  				continue
   421  			}
   422  			if users[sPost.User] == nil {
   423  				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
   424  				continue
   425  			}
   426  			newPost := model.Post{
   427  				UserId:    users[sPost.User].Id,
   428  				ChannelId: channel.Id,
   429  				Message:   "*" + sPost.Text + "*",
   430  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   431  			}
   432  			postId := si.oldImportPost(&newPost)
   433  			// If post is thread starter
   434  			if sPost.ThreadTS == sPost.TimeStamp {
   435  				threads[sPost.ThreadTS] = postId
   436  			}
   437  		case sPost.Type == "message" && sPost.SubType == "channel_topic":
   438  			if sPost.User == "" {
   439  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   440  				continue
   441  			}
   442  			if users[sPost.User] == nil {
   443  				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
   444  				continue
   445  			}
   446  			newPost := model.Post{
   447  				UserId:    users[sPost.User].Id,
   448  				ChannelId: channel.Id,
   449  				Message:   sPost.Text,
   450  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   451  				Type:      model.POST_HEADER_CHANGE,
   452  			}
   453  			si.oldImportPost(&newPost)
   454  		case sPost.Type == "message" && sPost.SubType == "channel_purpose":
   455  			if sPost.User == "" {
   456  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   457  				continue
   458  			}
   459  			if users[sPost.User] == nil {
   460  				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
   461  				continue
   462  			}
   463  			newPost := model.Post{
   464  				UserId:    users[sPost.User].Id,
   465  				ChannelId: channel.Id,
   466  				Message:   sPost.Text,
   467  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   468  				Type:      model.POST_PURPOSE_CHANGE,
   469  			}
   470  			si.oldImportPost(&newPost)
   471  		case sPost.Type == "message" && sPost.SubType == "channel_name":
   472  			if sPost.User == "" {
   473  				mlog.Debug("Slack Import: Unable to import the message as the user field is missing.")
   474  				continue
   475  			}
   476  			if users[sPost.User] == nil {
   477  				mlog.Debug("Slack Import: Unable to add the message as the Slack user does not exist in Mattermost.", mlog.String("user", sPost.User))
   478  				continue
   479  			}
   480  			newPost := model.Post{
   481  				UserId:    users[sPost.User].Id,
   482  				ChannelId: channel.Id,
   483  				Message:   sPost.Text,
   484  				CreateAt:  slackConvertTimeStamp(sPost.TimeStamp),
   485  				Type:      model.POST_DISPLAYNAME_CHANGE,
   486  			}
   487  			si.oldImportPost(&newPost)
   488  		default:
   489  			mlog.Warn(
   490  				"Slack Import: Unable to import the message as its type is not supported",
   491  				mlog.String("post_type", sPost.Type),
   492  				mlog.String("post_subtype", sPost.SubType),
   493  			)
   494  		}
   495  	}
   496  }
   497  
   498  func (si *SlackImporter) slackUploadFile(slackPostFile *slackFile, uploads map[string]*zip.File, teamId string, channelId string, userId string, slackTimestamp string) (*model.FileInfo, bool) {
   499  	if slackPostFile == nil {
   500  		mlog.Warn("Slack Import: Unable to attach the file to the post as the latter has no file section present in Slack export.")
   501  		return nil, false
   502  	}
   503  	file, ok := uploads[slackPostFile.Id]
   504  	if !ok {
   505  		mlog.Warn("Slack Import: Unable to import file as the file is missing from the Slack export zip file.", mlog.String("file_id", slackPostFile.Id))
   506  		return nil, false
   507  	}
   508  	openFile, err := file.Open()
   509  	if err != nil {
   510  		mlog.Warn("Slack Import: Unable to open the file from the Slack export.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err))
   511  		return nil, false
   512  	}
   513  	defer openFile.Close()
   514  
   515  	timestamp := utils.TimeFromMillis(slackConvertTimeStamp(slackTimestamp))
   516  	uploadedFile, err := si.oldImportFile(timestamp, openFile, teamId, channelId, userId, filepath.Base(file.Name))
   517  	if err != nil {
   518  		mlog.Warn("Slack Import: An error occurred when uploading file.", mlog.String("file_id", slackPostFile.Id), mlog.Err(err))
   519  		return nil, false
   520  	}
   521  
   522  	return uploadedFile, true
   523  }
   524  
   525  func (si *SlackImporter) deactivateSlackBotUser(user *model.User) {
   526  	if _, err := si.actions.UpdateActive(user, false); err != nil {
   527  		mlog.Warn("Slack Import: Unable to deactivate the user account used for the bot.")
   528  	}
   529  }
   530  
   531  func (si *SlackImporter) addSlackUsersToChannel(members []string, users map[string]*model.User, channel *model.Channel, log *bytes.Buffer) {
   532  	for _, member := range members {
   533  		user, ok := users[member]
   534  		if !ok {
   535  			log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": "?"}))
   536  			continue
   537  		}
   538  		if _, err := si.actions.AddUserToChannel(user, channel, false); err != nil {
   539  			log.WriteString(i18n.T("api.slackimport.slack_add_channels.failed_to_add_user", map[string]interface{}{"Username": user.Username}))
   540  		}
   541  	}
   542  }
   543  
   544  func slackSanitiseChannelProperties(channel model.Channel) model.Channel {
   545  	if utf8.RuneCountInString(channel.DisplayName) > model.CHANNEL_DISPLAY_NAME_MAX_RUNES {
   546  		mlog.Warn("Slack Import: Channel display name exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
   547  		channel.DisplayName = truncateRunes(channel.DisplayName, model.CHANNEL_DISPLAY_NAME_MAX_RUNES)
   548  	}
   549  
   550  	if len(channel.Name) > model.CHANNEL_NAME_MAX_LENGTH {
   551  		mlog.Warn("Slack Import: Channel handle exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
   552  		channel.Name = channel.Name[0:model.CHANNEL_NAME_MAX_LENGTH]
   553  	}
   554  
   555  	if utf8.RuneCountInString(channel.Purpose) > model.CHANNEL_PURPOSE_MAX_RUNES {
   556  		mlog.Warn("Slack Import: Channel purpose exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
   557  		channel.Purpose = truncateRunes(channel.Purpose, model.CHANNEL_PURPOSE_MAX_RUNES)
   558  	}
   559  
   560  	if utf8.RuneCountInString(channel.Header) > model.CHANNEL_HEADER_MAX_RUNES {
   561  		mlog.Warn("Slack Import: Channel header exceeds the maximum length. It will be truncated when imported.", mlog.String("channel_display_name", channel.DisplayName))
   562  		channel.Header = truncateRunes(channel.Header, model.CHANNEL_HEADER_MAX_RUNES)
   563  	}
   564  
   565  	return channel
   566  }
   567  
   568  func (si *SlackImporter) slackAddChannels(teamId string, slackchannels []slackChannel, posts map[string][]slackPost, users map[string]*model.User, uploads map[string]*zip.File, botUser *model.User, importerLog *bytes.Buffer) map[string]*model.Channel {
   569  	// Write Header
   570  	importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.added"))
   571  	importerLog.WriteString("=================\r\n\r\n")
   572  
   573  	addedChannels := make(map[string]*model.Channel)
   574  	for _, sChannel := range slackchannels {
   575  		newChannel := model.Channel{
   576  			TeamId:      teamId,
   577  			Type:        sChannel.Type,
   578  			DisplayName: sChannel.Name,
   579  			Name:        slackConvertChannelName(sChannel.Name, sChannel.Id),
   580  			Purpose:     sChannel.Purpose.Value,
   581  			Header:      sChannel.Topic.Value,
   582  		}
   583  
   584  		// Direct message channels in Slack don't have a name so we set the id as name or else the messages won't get imported.
   585  		if newChannel.Type == model.CHANNEL_DIRECT {
   586  			sChannel.Name = sChannel.Id
   587  		}
   588  
   589  		newChannel = slackSanitiseChannelProperties(newChannel)
   590  
   591  		var mChannel *model.Channel
   592  		var err error
   593  		if mChannel, err = si.store.Channel().GetByName(teamId, sChannel.Name, true); err == nil {
   594  			// The channel already exists as an active channel. Merge with the existing one.
   595  			importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.merge", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
   596  		} else if _, nErr := si.store.Channel().GetDeletedByName(teamId, sChannel.Name); nErr == nil {
   597  			// The channel already exists but has been deleted. Generate a random string for the handle instead.
   598  			newChannel.Name = model.NewId()
   599  			newChannel = slackSanitiseChannelProperties(newChannel)
   600  		}
   601  
   602  		if mChannel == nil {
   603  			// Haven't found an existing channel to merge with. Try importing it as a new one.
   604  			mChannel = si.oldImportChannel(&newChannel, sChannel, users)
   605  			if mChannel == nil {
   606  				mlog.Warn("Slack Import: Unable to import Slack channel.", mlog.String("channel_display_name", newChannel.DisplayName))
   607  				importerLog.WriteString(i18n.T("api.slackimport.slack_add_channels.import_failed", map[string]interface{}{"DisplayName": newChannel.DisplayName}))
   608  				continue
   609  			}
   610  		}
   611  
   612  		// Members for direct and group channels are added during the creation of the channel in the oldImportChannel function
   613  		if sChannel.Type == model.CHANNEL_OPEN || sChannel.Type == model.CHANNEL_PRIVATE {
   614  			si.addSlackUsersToChannel(sChannel.Members, users, mChannel, importerLog)
   615  		}
   616  		importerLog.WriteString(newChannel.DisplayName + "\r\n")
   617  		addedChannels[sChannel.Id] = mChannel
   618  		si.slackAddPosts(teamId, mChannel, posts[sChannel.Name], users, uploads, botUser)
   619  	}
   620  
   621  	return addedChannels
   622  }
   623  
   624  //
   625  // -- Old SlackImport Functions --
   626  // Import functions are suitable for entering posts and users into the database without
   627  // some of the usual checks. (IsValid is still run)
   628  //
   629  
   630  func (si *SlackImporter) oldImportPost(post *model.Post) string {
   631  	// Workaround for empty messages, which may be the case if they are webhook posts.
   632  	firstIteration := true
   633  	firstPostId := ""
   634  	if post.ParentId != "" {
   635  		firstPostId = post.ParentId
   636  	}
   637  	maxPostSize := si.actions.MaxPostSize()
   638  	for messageRuneCount := utf8.RuneCountInString(post.Message); messageRuneCount > 0 || firstIteration; messageRuneCount = utf8.RuneCountInString(post.Message) {
   639  		var remainder string
   640  		if messageRuneCount > maxPostSize {
   641  			remainder = string(([]rune(post.Message))[maxPostSize:])
   642  			post.Message = truncateRunes(post.Message, maxPostSize)
   643  		} else {
   644  			remainder = ""
   645  		}
   646  
   647  		post.Hashtags, _ = model.ParseHashtags(post.Message)
   648  
   649  		post.RootId = firstPostId
   650  		post.ParentId = firstPostId
   651  
   652  		_, err := si.store.Post().Save(post)
   653  		if err != nil {
   654  			mlog.Debug("Error saving post.", mlog.String("user_id", post.UserId), mlog.String("message", post.Message))
   655  		}
   656  
   657  		if firstIteration {
   658  			if firstPostId == "" {
   659  				firstPostId = post.Id
   660  			}
   661  			for _, fileId := range post.FileIds {
   662  				if err := si.store.FileInfo().AttachToPost(fileId, post.Id, post.UserId); err != nil {
   663  					mlog.Error(
   664  						"Error attaching files to post.",
   665  						mlog.String("post_id", post.Id),
   666  						mlog.String("file_ids", strings.Join(post.FileIds, ",")),
   667  						mlog.String("user_id", post.UserId),
   668  						mlog.Err(err),
   669  					)
   670  				}
   671  			}
   672  			post.FileIds = nil
   673  		}
   674  
   675  		post.Id = ""
   676  		post.CreateAt++
   677  		post.Message = remainder
   678  		firstIteration = false
   679  	}
   680  	return firstPostId
   681  }
   682  
   683  func (si *SlackImporter) oldImportUser(team *model.Team, user *model.User) *model.User {
   684  	user.MakeNonNil()
   685  
   686  	user.Roles = model.SYSTEM_USER_ROLE_ID
   687  
   688  	ruser, nErr := si.store.User().Save(user)
   689  	if nErr != nil {
   690  		mlog.Debug("Error saving user.", mlog.Err(nErr))
   691  		return nil
   692  	}
   693  
   694  	if _, err := si.store.User().VerifyEmail(ruser.Id, ruser.Email); err != nil {
   695  		mlog.Warn("Failed to set email verified.", mlog.Err(err))
   696  	}
   697  
   698  	if _, err := si.actions.JoinUserToTeam(team, user, ""); err != nil {
   699  		mlog.Warn("Failed to join team when importing.", mlog.Err(err))
   700  	}
   701  
   702  	return ruser
   703  }
   704  
   705  func (si *SlackImporter) oldImportChannel(channel *model.Channel, sChannel slackChannel, users map[string]*model.User) *model.Channel {
   706  	switch {
   707  	case channel.Type == model.CHANNEL_DIRECT:
   708  		if len(sChannel.Members) < 2 {
   709  			return nil
   710  		}
   711  		u1 := users[sChannel.Members[0]]
   712  		u2 := users[sChannel.Members[1]]
   713  		if u1 == nil || u2 == nil {
   714  			mlog.Warn("Either or both of user ids not found in users.json. Ignoring.", mlog.String("id1", sChannel.Members[0]), mlog.String("id2", sChannel.Members[1]))
   715  			return nil
   716  		}
   717  		sc, err := si.actions.CreateDirectChannel(u1.Id, u2.Id)
   718  		if err != nil {
   719  			return nil
   720  		}
   721  
   722  		return sc
   723  	// check if direct channel has less than 8 members and if not import as private channel instead
   724  	case channel.Type == model.CHANNEL_GROUP && len(sChannel.Members) < 8:
   725  		members := make([]string, len(sChannel.Members))
   726  
   727  		for i := range sChannel.Members {
   728  			u := users[sChannel.Members[i]]
   729  			if u == nil {
   730  				mlog.Warn("User not found in users.json. Ignoring.", mlog.String("id", sChannel.Members[i]))
   731  				continue
   732  			}
   733  			members[i] = u.Id
   734  		}
   735  
   736  		creator := users[sChannel.Creator]
   737  		if creator == nil {
   738  			return nil
   739  		}
   740  		sc, err := si.actions.CreateGroupChannel(members)
   741  		if err != nil {
   742  			return nil
   743  		}
   744  
   745  		return sc
   746  	case channel.Type == model.CHANNEL_GROUP:
   747  		channel.Type = model.CHANNEL_PRIVATE
   748  		sc, err := si.actions.CreateChannel(channel, false)
   749  		if err != nil {
   750  			return nil
   751  		}
   752  
   753  		return sc
   754  	}
   755  
   756  	sc, err := si.store.Channel().Save(channel, *si.config.TeamSettings.MaxChannelsPerTeam)
   757  	if err != nil {
   758  		return nil
   759  	}
   760  
   761  	return sc
   762  }
   763  
   764  func (si *SlackImporter) oldImportFile(timestamp time.Time, file io.Reader, teamId string, channelId string, userId string, fileName string) (*model.FileInfo, error) {
   765  	buf := bytes.NewBuffer(nil)
   766  	io.Copy(buf, file)
   767  	data := buf.Bytes()
   768  
   769  	fileInfo, err := si.actions.DoUploadFile(timestamp, teamId, channelId, userId, fileName, data)
   770  	if err != nil {
   771  		return nil, err
   772  	}
   773  
   774  	if fileInfo.IsImage() && fileInfo.MimeType != "image/svg+xml" {
   775  		img, release, err := si.actions.PrepareImage(data)
   776  		if err != nil {
   777  			return nil, err
   778  		}
   779  		defer release()
   780  		si.actions.GenerateThumbnailImage(img, fileInfo.ThumbnailPath)
   781  		si.actions.GeneratePreviewImage(img, fileInfo.PreviewPath)
   782  	}
   783  
   784  	return fileInfo, nil
   785  }
   786  
   787  func (si *SlackImporter) oldImportIncomingWebhookPost(post *model.Post, props model.StringInterface) string {
   788  	linkWithTextRegex := regexp.MustCompile(`<([^<\|]+)\|([^>]+)>`)
   789  	post.Message = linkWithTextRegex.ReplaceAllString(post.Message, "[${2}](${1})")
   790  
   791  	post.AddProp("from_webhook", "true")
   792  
   793  	if _, ok := props["override_username"]; !ok {
   794  		post.AddProp("override_username", model.DEFAULT_WEBHOOK_USERNAME)
   795  	}
   796  
   797  	if len(props) > 0 {
   798  		for key, val := range props {
   799  			if key == "attachments" {
   800  				if attachments, success := val.([]*model.SlackAttachment); success {
   801  					model.ParseSlackAttachment(post, attachments)
   802  				}
   803  			} else if key != "from_webhook" {
   804  				post.AddProp(key, val)
   805  			}
   806  		}
   807  	}
   808  
   809  	return si.oldImportPost(post)
   810  }