github.com/dschalla/mattermost-server@v4.8.1-rc1+incompatible/cmd/platform/sampledata.go (about)

     1  // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
     2  // See License.txt for license information.
     3  package main
     4  
     5  import (
     6  	"encoding/json"
     7  	"errors"
     8  	"fmt"
     9  	"io/ioutil"
    10  	"math/rand"
    11  	"os"
    12  	"path"
    13  	"sort"
    14  	"strings"
    15  	"time"
    16  
    17  	"github.com/icrowley/fake"
    18  	"github.com/mattermost/mattermost-server/app"
    19  	"github.com/spf13/cobra"
    20  )
    21  
    22  var sampleDataCmd = &cobra.Command{
    23  	Use:   "sampledata",
    24  	Short: "Generate sample data",
    25  	RunE:  sampleDataCmdF,
    26  }
    27  
    28  func sliceIncludes(vs []string, t string) bool {
    29  	for _, v := range vs {
    30  		if v == t {
    31  			return true
    32  		}
    33  	}
    34  	return false
    35  }
    36  
    37  func randomPastTime(seconds int) int64 {
    38  	now := time.Now()
    39  	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.FixedZone("UTC", 0))
    40  	return today.Unix() - int64(rand.Intn(seconds*1000))
    41  }
    42  
    43  func randomEmoji() string {
    44  	emojis := []string{"+1", "-1", "heart", "blush"}
    45  	return emojis[rand.Intn(len(emojis))]
    46  }
    47  
    48  func randomReaction(users []string, parentCreateAt int64) app.ReactionImportData {
    49  	user := users[rand.Intn(len(users))]
    50  	emoji := randomEmoji()
    51  	date := parentCreateAt + int64(rand.Intn(100000))
    52  	return app.ReactionImportData{
    53  		User:      &user,
    54  		EmojiName: &emoji,
    55  		CreateAt:  &date,
    56  	}
    57  }
    58  
    59  func randomReply(users []string, parentCreateAt int64) app.ReplyImportData {
    60  	user := users[rand.Intn(len(users))]
    61  	message := randomMessage(users)
    62  	date := parentCreateAt + int64(rand.Intn(100000))
    63  	return app.ReplyImportData{
    64  		User:     &user,
    65  		Message:  &message,
    66  		CreateAt: &date,
    67  	}
    68  }
    69  
    70  func randomMessage(users []string) string {
    71  	var message string
    72  	switch rand.Intn(30) {
    73  	case 0:
    74  		mention := users[rand.Intn(len(users))]
    75  		message = "@" + mention + " " + fake.Sentence()
    76  	case 1:
    77  		switch rand.Intn(2) {
    78  		case 0:
    79  			mattermostVideos := []string{"Q4MgnxbpZas", "BFo7E9-Kc_E", "LsMLR-BHsKg", "MRmGDhlMhNA", "mUOPxT7VgWc"}
    80  			message = "https://www.youtube.com/watch?v=" + mattermostVideos[rand.Intn(len(mattermostVideos))]
    81  		case 1:
    82  			mattermostTweets := []string{"943119062334353408", "949370809528832005", "948539688171819009", "939122439115681792", "938061722027425797"}
    83  			message = "https://twitter.com/mattermosthq/status/" + mattermostTweets[rand.Intn(len(mattermostTweets))]
    84  		}
    85  	case 2:
    86  		message = ""
    87  		if rand.Intn(2) == 0 {
    88  			message += fake.Sentence()
    89  		}
    90  		for i := 0; i < rand.Intn(4)+1; i++ {
    91  			message += "\n  * " + fake.Word()
    92  		}
    93  	default:
    94  		if rand.Intn(2) == 0 {
    95  			message = fake.Sentence()
    96  		} else {
    97  			message = fake.Paragraph()
    98  		}
    99  		if rand.Intn(3) == 0 {
   100  			message += "\n" + fake.Sentence()
   101  		}
   102  		if rand.Intn(3) == 0 {
   103  			message += "\n" + fake.Sentence()
   104  		}
   105  		if rand.Intn(3) == 0 {
   106  			message += "\n" + fake.Sentence()
   107  		}
   108  	}
   109  	return message
   110  }
   111  
   112  func init() {
   113  	sampleDataCmd.Flags().Int64P("seed", "s", 1, "Seed used for generating the random data (Different seeds generate different data).")
   114  	sampleDataCmd.Flags().IntP("teams", "t", 2, "The number of sample teams.")
   115  	sampleDataCmd.Flags().Int("channels-per-team", 10, "The number of sample channels per team.")
   116  	sampleDataCmd.Flags().IntP("users", "u", 15, "The number of sample users.")
   117  	sampleDataCmd.Flags().Int("team-memberships", 2, "The number of sample team memberships per user.")
   118  	sampleDataCmd.Flags().Int("channel-memberships", 5, "The number of sample channel memberships per user in a team.")
   119  	sampleDataCmd.Flags().Int("posts-per-channel", 100, "The number of sample post per channel.")
   120  	sampleDataCmd.Flags().Int("direct-channels", 30, "The number of sample direct message channels.")
   121  	sampleDataCmd.Flags().Int("posts-per-direct-channel", 15, "The number of sample posts per direct message channel.")
   122  	sampleDataCmd.Flags().Int("group-channels", 15, "The number of sample group message channels.")
   123  	sampleDataCmd.Flags().Int("posts-per-group-channel", 30, "The number of sample posts per group message channel.")
   124  	sampleDataCmd.Flags().IntP("workers", "w", 2, "How many workers to run during the import.")
   125  	sampleDataCmd.Flags().String("profile-images", "", "Optional. Path to folder with images to randomly pick as user profile image.")
   126  	sampleDataCmd.Flags().StringP("bulk", "b", "", "Optional. Path to write a JSONL bulk file instead of loading into the database.")
   127  }
   128  
   129  func sampleDataCmdF(cmd *cobra.Command, args []string) error {
   130  	a, err := initDBCommandContextCobra(cmd)
   131  	if err != nil {
   132  		return err
   133  	}
   134  	seed, err := cmd.Flags().GetInt64("seed")
   135  	if err != nil {
   136  		return errors.New("Invalid seed parameter")
   137  	}
   138  	bulk, err := cmd.Flags().GetString("bulk")
   139  	if err != nil {
   140  		return errors.New("Invalid bulk parameter")
   141  	}
   142  	teams, err := cmd.Flags().GetInt("teams")
   143  	if err != nil || teams < 0 {
   144  		return errors.New("Invalid teams parameter")
   145  	}
   146  	channelsPerTeam, err := cmd.Flags().GetInt("channels-per-team")
   147  	if err != nil || channelsPerTeam < 0 {
   148  		return errors.New("Invalid channels-per-team parameter")
   149  	}
   150  	users, err := cmd.Flags().GetInt("users")
   151  	if err != nil || users < 0 {
   152  		return errors.New("Invalid users parameter")
   153  	}
   154  	teamMemberships, err := cmd.Flags().GetInt("team-memberships")
   155  	if err != nil || teamMemberships < 0 {
   156  		return errors.New("Invalid team-memberships parameter")
   157  	}
   158  	channelMemberships, err := cmd.Flags().GetInt("channel-memberships")
   159  	if err != nil || channelMemberships < 0 {
   160  		return errors.New("Invalid channel-memberships parameter")
   161  	}
   162  	postsPerChannel, err := cmd.Flags().GetInt("posts-per-channel")
   163  	if err != nil || postsPerChannel < 0 {
   164  		return errors.New("Invalid posts-per-channel parameter")
   165  	}
   166  	directChannels, err := cmd.Flags().GetInt("direct-channels")
   167  	if err != nil || directChannels < 0 {
   168  		return errors.New("Invalid direct-channels parameter")
   169  	}
   170  	postsPerDirectChannel, err := cmd.Flags().GetInt("posts-per-direct-channel")
   171  	if err != nil || postsPerDirectChannel < 0 {
   172  		return errors.New("Invalid posts-per-direct-channel parameter")
   173  	}
   174  	groupChannels, err := cmd.Flags().GetInt("group-channels")
   175  	if err != nil || groupChannels < 0 {
   176  		return errors.New("Invalid group-channels parameter")
   177  	}
   178  	postsPerGroupChannel, err := cmd.Flags().GetInt("posts-per-group-channel")
   179  	if err != nil || postsPerGroupChannel < 0 {
   180  		return errors.New("Invalid posts-per-group-channel parameter")
   181  	}
   182  	workers, err := cmd.Flags().GetInt("workers")
   183  	if err != nil {
   184  		return errors.New("Invalid workers parameter")
   185  	}
   186  	profileImagesPath, err := cmd.Flags().GetString("profile-images")
   187  	if err != nil {
   188  		return errors.New("Invalid profile-images parameter")
   189  	}
   190  	profileImages := []string{}
   191  	if profileImagesPath != "" {
   192  		profileImagesStat, err := os.Stat(profileImagesPath)
   193  		if os.IsNotExist(err) {
   194  			return errors.New("Profile images folder doesn't exists.")
   195  		}
   196  		if !profileImagesStat.IsDir() {
   197  			return errors.New("profile-images parameters must be a folder path.")
   198  		}
   199  		profileImagesFiles, err := ioutil.ReadDir(profileImagesPath)
   200  		if err != nil {
   201  			return errors.New("Invalid profile-images parameter")
   202  		}
   203  		for _, profileImage := range profileImagesFiles {
   204  			profileImages = append(profileImages, path.Join(profileImagesPath, profileImage.Name()))
   205  		}
   206  		sort.Strings(profileImages)
   207  	}
   208  
   209  	if workers < 1 {
   210  		return errors.New("You must have at least one worker.")
   211  	}
   212  	if teamMemberships > teams {
   213  		return errors.New("You can't have more team memberships than teams.")
   214  	}
   215  	if channelMemberships > channelsPerTeam {
   216  		return errors.New("You can't have more channel memberships than channels per team.")
   217  	}
   218  
   219  	var bulkFile *os.File
   220  	switch bulk {
   221  	case "":
   222  		bulkFile, err = ioutil.TempFile("", ".mattermost-sample-data-")
   223  		defer os.Remove(bulkFile.Name())
   224  		if err != nil {
   225  			return errors.New("Unable to open temporary file.")
   226  		}
   227  	case "-":
   228  		bulkFile = os.Stdout
   229  	default:
   230  		bulkFile, err = os.OpenFile(bulk, os.O_RDWR|os.O_CREATE, 0755)
   231  		if err != nil {
   232  			return errors.New("Unable to write into the \"" + bulk + "\" file.")
   233  		}
   234  	}
   235  
   236  	encoder := json.NewEncoder(bulkFile)
   237  	version := 1
   238  	encoder.Encode(app.LineImportData{Type: "version", Version: &version})
   239  
   240  	fake.Seed(seed)
   241  	rand.Seed(seed)
   242  
   243  	teamsAndChannels := make(map[string][]string)
   244  	for i := 0; i < teams; i++ {
   245  		teamLine := createTeam(i)
   246  		teamsAndChannels[*teamLine.Team.Name] = []string{}
   247  		encoder.Encode(teamLine)
   248  	}
   249  
   250  	teamsList := []string{}
   251  	for teamName := range teamsAndChannels {
   252  		teamsList = append(teamsList, teamName)
   253  	}
   254  	sort.Strings(teamsList)
   255  
   256  	for _, teamName := range teamsList {
   257  		for i := 0; i < channelsPerTeam; i++ {
   258  			channelLine := createChannel(i, teamName)
   259  			teamsAndChannels[teamName] = append(teamsAndChannels[teamName], *channelLine.Channel.Name)
   260  			encoder.Encode(channelLine)
   261  		}
   262  	}
   263  
   264  	allUsers := []string{}
   265  	for i := 0; i < users; i++ {
   266  		userLine := createUser(i, teamMemberships, channelMemberships, teamsAndChannels, profileImages)
   267  		encoder.Encode(userLine)
   268  		allUsers = append(allUsers, *userLine.User.Username)
   269  	}
   270  
   271  	for team, channels := range teamsAndChannels {
   272  		for _, channel := range channels {
   273  			for i := 0; i < postsPerChannel; i++ {
   274  				postLine := createPost(team, channel, allUsers)
   275  				encoder.Encode(postLine)
   276  			}
   277  		}
   278  	}
   279  
   280  	for i := 0; i < directChannels; i++ {
   281  		user1 := allUsers[rand.Intn(len(allUsers))]
   282  		user2 := allUsers[rand.Intn(len(allUsers))]
   283  		channelLine := createDirectChannel([]string{user1, user2})
   284  		encoder.Encode(channelLine)
   285  		for j := 0; j < postsPerDirectChannel; j++ {
   286  			postLine := createDirectPost([]string{user1, user2})
   287  			encoder.Encode(postLine)
   288  		}
   289  	}
   290  
   291  	for i := 0; i < groupChannels; i++ {
   292  		users := []string{}
   293  		totalUsers := 3 + rand.Intn(3)
   294  		for len(users) < totalUsers {
   295  			user := allUsers[rand.Intn(len(allUsers))]
   296  			if !sliceIncludes(users, user) {
   297  				users = append(users, user)
   298  			}
   299  		}
   300  		channelLine := createDirectChannel(users)
   301  		encoder.Encode(channelLine)
   302  		for j := 0; j < postsPerGroupChannel; j++ {
   303  			postLine := createDirectPost(users)
   304  			encoder.Encode(postLine)
   305  		}
   306  	}
   307  
   308  	if bulk == "" {
   309  		_, err := bulkFile.Seek(0, 0)
   310  		if err != nil {
   311  			return errors.New("Unable to read correctly the temporary file.")
   312  		}
   313  		importErr, lineNumber := a.BulkImport(bulkFile, false, workers)
   314  		if importErr != nil {
   315  			return errors.New(fmt.Sprintf("%s: %s, %s (line: %d)", importErr.Where, importErr.Message, importErr.DetailedError, lineNumber))
   316  		}
   317  	} else if bulk != "-" {
   318  		err := bulkFile.Close()
   319  		if err != nil {
   320  			return errors.New("Unable to close correctly the output file")
   321  		}
   322  	}
   323  
   324  	return nil
   325  }
   326  
   327  func createUser(idx int, teamMemberships int, channelMemberships int, teamsAndChannels map[string][]string, profileImages []string) app.LineImportData {
   328  	password := fmt.Sprintf("user-%d", idx)
   329  	email := fmt.Sprintf("user-%d@sample.mattermost.com", idx)
   330  	firstName := fake.FirstName()
   331  	lastName := fake.LastName()
   332  	username := fmt.Sprintf("%s.%s", strings.ToLower(firstName), strings.ToLower(lastName))
   333  	position := fake.JobTitle()
   334  	roles := "system_user"
   335  	if idx%5 == 0 {
   336  		roles = "system_admin system_user"
   337  	}
   338  
   339  	// The 75% of the users have custom profile image
   340  	var profileImage *string = nil
   341  	if rand.Intn(4) != 0 {
   342  		profileImageSelector := rand.Int()
   343  		if len(profileImages) > 0 {
   344  			profileImage = &profileImages[profileImageSelector%len(profileImages)]
   345  		}
   346  	}
   347  
   348  	useMilitaryTime := "false"
   349  	if idx != 0 && rand.Intn(2) == 0 {
   350  		useMilitaryTime = "true"
   351  	}
   352  
   353  	collapsePreviews := "false"
   354  	if idx != 0 && rand.Intn(2) == 0 {
   355  		collapsePreviews = "true"
   356  	}
   357  
   358  	messageDisplay := "clean"
   359  	if idx != 0 && rand.Intn(2) == 0 {
   360  		messageDisplay = "compact"
   361  	}
   362  
   363  	channelDisplayMode := "full"
   364  	if idx != 0 && rand.Intn(2) == 0 {
   365  		channelDisplayMode = "centered"
   366  	}
   367  
   368  	// Some users has nickname
   369  	nickname := ""
   370  	if rand.Intn(5) == 0 {
   371  		nickname = fake.Company()
   372  	}
   373  
   374  	// Half of users skip tutorial
   375  	tutorialStep := "999"
   376  	switch rand.Intn(6) {
   377  	case 1:
   378  		tutorialStep = "1"
   379  	case 2:
   380  		tutorialStep = "2"
   381  	case 3:
   382  		tutorialStep = "3"
   383  	}
   384  
   385  	teams := []app.UserTeamImportData{}
   386  	possibleTeams := []string{}
   387  	for teamName := range teamsAndChannels {
   388  		possibleTeams = append(possibleTeams, teamName)
   389  	}
   390  	sort.Strings(possibleTeams)
   391  	for x := 0; x < teamMemberships; x++ {
   392  		if len(possibleTeams) == 0 {
   393  			break
   394  		}
   395  		position := rand.Intn(len(possibleTeams))
   396  		team := possibleTeams[position]
   397  		possibleTeams = append(possibleTeams[:position], possibleTeams[position+1:]...)
   398  		if teamChannels, err := teamsAndChannels[team]; err == true {
   399  			teams = append(teams, createTeamMembership(channelMemberships, teamChannels, &team))
   400  		}
   401  	}
   402  
   403  	user := app.UserImportData{
   404  		ProfileImage:       profileImage,
   405  		Username:           &username,
   406  		Email:              &email,
   407  		Password:           &password,
   408  		Nickname:           &nickname,
   409  		FirstName:          &firstName,
   410  		LastName:           &lastName,
   411  		Position:           &position,
   412  		Roles:              &roles,
   413  		Teams:              &teams,
   414  		UseMilitaryTime:    &useMilitaryTime,
   415  		CollapsePreviews:   &collapsePreviews,
   416  		MessageDisplay:     &messageDisplay,
   417  		ChannelDisplayMode: &channelDisplayMode,
   418  		TutorialStep:       &tutorialStep,
   419  	}
   420  	return app.LineImportData{
   421  		Type: "user",
   422  		User: &user,
   423  	}
   424  }
   425  
   426  func createTeamMembership(numOfchannels int, teamChannels []string, teamName *string) app.UserTeamImportData {
   427  	roles := "team_user"
   428  	if rand.Intn(5) == 0 {
   429  		roles = "team_user team_admin"
   430  	}
   431  	channels := []app.UserChannelImportData{}
   432  	teamChannelsCopy := []string{}
   433  	for _, value := range teamChannels {
   434  		teamChannelsCopy = append(teamChannelsCopy, value)
   435  	}
   436  	for x := 0; x < numOfchannels; x++ {
   437  		if len(teamChannelsCopy) == 0 {
   438  			break
   439  		}
   440  		position := rand.Intn(len(teamChannelsCopy))
   441  		channelName := teamChannelsCopy[position]
   442  		teamChannelsCopy = append(teamChannelsCopy[:position], teamChannelsCopy[position+1:]...)
   443  		channels = append(channels, createChannelMembership(channelName))
   444  	}
   445  
   446  	return app.UserTeamImportData{
   447  		Name:     teamName,
   448  		Roles:    &roles,
   449  		Channels: &channels,
   450  	}
   451  }
   452  
   453  func createChannelMembership(channelName string) app.UserChannelImportData {
   454  	roles := "channel_user"
   455  	if rand.Intn(5) == 0 {
   456  		roles = "channel_user channel_admin"
   457  	}
   458  	favorite := rand.Intn(5) == 0
   459  
   460  	return app.UserChannelImportData{
   461  		Name:     &channelName,
   462  		Roles:    &roles,
   463  		Favorite: &favorite,
   464  	}
   465  }
   466  
   467  func createTeam(idx int) app.LineImportData {
   468  	displayName := fake.Word()
   469  	name := fmt.Sprintf("%s-%d", fake.Word(), idx)
   470  	allowOpenInvite := rand.Intn(2) == 0
   471  
   472  	description := fake.Paragraph()
   473  	if len(description) > 255 {
   474  		description = description[0:255]
   475  	}
   476  
   477  	teamType := "O"
   478  	if rand.Intn(2) == 0 {
   479  		teamType = "I"
   480  	}
   481  
   482  	team := app.TeamImportData{
   483  		DisplayName:     &displayName,
   484  		Name:            &name,
   485  		AllowOpenInvite: &allowOpenInvite,
   486  		Description:     &description,
   487  		Type:            &teamType,
   488  	}
   489  	return app.LineImportData{
   490  		Type: "team",
   491  		Team: &team,
   492  	}
   493  }
   494  
   495  func createChannel(idx int, teamName string) app.LineImportData {
   496  	displayName := fake.Word()
   497  	name := fmt.Sprintf("%s-%d", fake.Word(), idx)
   498  	header := fake.Paragraph()
   499  	purpose := fake.Paragraph()
   500  
   501  	if len(purpose) > 250 {
   502  		purpose = purpose[0:250]
   503  	}
   504  
   505  	channelType := "P"
   506  	if rand.Intn(2) == 0 {
   507  		channelType = "O"
   508  	}
   509  
   510  	channel := app.ChannelImportData{
   511  		Team:        &teamName,
   512  		Name:        &name,
   513  		DisplayName: &displayName,
   514  		Type:        &channelType,
   515  		Header:      &header,
   516  		Purpose:     &purpose,
   517  	}
   518  	return app.LineImportData{
   519  		Type:    "channel",
   520  		Channel: &channel,
   521  	}
   522  }
   523  
   524  func createPost(team string, channel string, allUsers []string) app.LineImportData {
   525  	message := randomMessage(allUsers)
   526  	create_at := randomPastTime(50000)
   527  	user := allUsers[rand.Intn(len(allUsers))]
   528  
   529  	// Some messages are flagged by an user
   530  	flagged_by := []string{}
   531  	if rand.Intn(10) == 0 {
   532  		flagged_by = append(flagged_by, allUsers[rand.Intn(len(allUsers))])
   533  	}
   534  
   535  	reactions := []app.ReactionImportData{}
   536  	if rand.Intn(10) == 0 {
   537  		for {
   538  			reactions = append(reactions, randomReaction(allUsers, create_at))
   539  			if rand.Intn(3) == 0 {
   540  				break
   541  			}
   542  		}
   543  	}
   544  
   545  	replies := []app.ReplyImportData{}
   546  	if rand.Intn(10) == 0 {
   547  		for {
   548  			replies = append(replies, randomReply(allUsers, create_at))
   549  			if rand.Intn(4) == 0 {
   550  				break
   551  			}
   552  		}
   553  	}
   554  
   555  	post := app.PostImportData{
   556  		Team:      &team,
   557  		Channel:   &channel,
   558  		User:      &user,
   559  		Message:   &message,
   560  		CreateAt:  &create_at,
   561  		FlaggedBy: &flagged_by,
   562  		Reactions: &reactions,
   563  		Replies:   &replies,
   564  	}
   565  	return app.LineImportData{
   566  		Type: "post",
   567  		Post: &post,
   568  	}
   569  }
   570  
   571  func createDirectChannel(members []string) app.LineImportData {
   572  	header := fake.Sentence()
   573  
   574  	channel := app.DirectChannelImportData{
   575  		Members: &members,
   576  		Header:  &header,
   577  	}
   578  	return app.LineImportData{
   579  		Type:          "direct_channel",
   580  		DirectChannel: &channel,
   581  	}
   582  }
   583  
   584  func createDirectPost(members []string) app.LineImportData {
   585  	message := randomMessage(members)
   586  	create_at := randomPastTime(50000)
   587  	user := members[rand.Intn(len(members))]
   588  
   589  	// Some messages are flagged by an user
   590  	flagged_by := []string{}
   591  	if rand.Intn(10) == 0 {
   592  		flagged_by = append(flagged_by, members[rand.Intn(len(members))])
   593  	}
   594  
   595  	reactions := []app.ReactionImportData{}
   596  	if rand.Intn(10) == 0 {
   597  		for {
   598  			reactions = append(reactions, randomReaction(members, create_at))
   599  			if rand.Intn(3) == 0 {
   600  				break
   601  			}
   602  		}
   603  	}
   604  
   605  	replies := []app.ReplyImportData{}
   606  	if rand.Intn(10) == 0 {
   607  		for {
   608  			replies = append(replies, randomReply(members, create_at))
   609  			if rand.Intn(4) == 0 {
   610  				break
   611  			}
   612  		}
   613  	}
   614  
   615  	post := app.DirectPostImportData{
   616  		ChannelMembers: &members,
   617  		User:           &user,
   618  		Message:        &message,
   619  		CreateAt:       &create_at,
   620  		FlaggedBy:      &flagged_by,
   621  		Reactions:      &reactions,
   622  		Replies:        &replies,
   623  	}
   624  	return app.LineImportData{
   625  		Type:       "direct_post",
   626  		DirectPost: &post,
   627  	}
   628  }