github.com/dschalla/mattermost-server@v4.8.1-rc1+incompatible/cmd/platform/sampledata.go (about) 1 // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 package main 4 5 import ( 6 "encoding/json" 7 "errors" 8 "fmt" 9 "io/ioutil" 10 "math/rand" 11 "os" 12 "path" 13 "sort" 14 "strings" 15 "time" 16 17 "github.com/icrowley/fake" 18 "github.com/mattermost/mattermost-server/app" 19 "github.com/spf13/cobra" 20 ) 21 22 var sampleDataCmd = &cobra.Command{ 23 Use: "sampledata", 24 Short: "Generate sample data", 25 RunE: sampleDataCmdF, 26 } 27 28 func sliceIncludes(vs []string, t string) bool { 29 for _, v := range vs { 30 if v == t { 31 return true 32 } 33 } 34 return false 35 } 36 37 func randomPastTime(seconds int) int64 { 38 now := time.Now() 39 today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.FixedZone("UTC", 0)) 40 return today.Unix() - int64(rand.Intn(seconds*1000)) 41 } 42 43 func randomEmoji() string { 44 emojis := []string{"+1", "-1", "heart", "blush"} 45 return emojis[rand.Intn(len(emojis))] 46 } 47 48 func randomReaction(users []string, parentCreateAt int64) app.ReactionImportData { 49 user := users[rand.Intn(len(users))] 50 emoji := randomEmoji() 51 date := parentCreateAt + int64(rand.Intn(100000)) 52 return app.ReactionImportData{ 53 User: &user, 54 EmojiName: &emoji, 55 CreateAt: &date, 56 } 57 } 58 59 func randomReply(users []string, parentCreateAt int64) app.ReplyImportData { 60 user := users[rand.Intn(len(users))] 61 message := randomMessage(users) 62 date := parentCreateAt + int64(rand.Intn(100000)) 63 return app.ReplyImportData{ 64 User: &user, 65 Message: &message, 66 CreateAt: &date, 67 } 68 } 69 70 func randomMessage(users []string) string { 71 var message string 72 switch rand.Intn(30) { 73 case 0: 74 mention := users[rand.Intn(len(users))] 75 message = "@" + mention + " " + fake.Sentence() 76 case 1: 77 switch rand.Intn(2) { 78 case 0: 79 mattermostVideos := []string{"Q4MgnxbpZas", "BFo7E9-Kc_E", "LsMLR-BHsKg", "MRmGDhlMhNA", "mUOPxT7VgWc"} 80 message = "https://www.youtube.com/watch?v=" + mattermostVideos[rand.Intn(len(mattermostVideos))] 81 case 1: 82 mattermostTweets := []string{"943119062334353408", "949370809528832005", "948539688171819009", "939122439115681792", "938061722027425797"} 83 message = "https://twitter.com/mattermosthq/status/" + mattermostTweets[rand.Intn(len(mattermostTweets))] 84 } 85 case 2: 86 message = "" 87 if rand.Intn(2) == 0 { 88 message += fake.Sentence() 89 } 90 for i := 0; i < rand.Intn(4)+1; i++ { 91 message += "\n * " + fake.Word() 92 } 93 default: 94 if rand.Intn(2) == 0 { 95 message = fake.Sentence() 96 } else { 97 message = fake.Paragraph() 98 } 99 if rand.Intn(3) == 0 { 100 message += "\n" + fake.Sentence() 101 } 102 if rand.Intn(3) == 0 { 103 message += "\n" + fake.Sentence() 104 } 105 if rand.Intn(3) == 0 { 106 message += "\n" + fake.Sentence() 107 } 108 } 109 return message 110 } 111 112 func init() { 113 sampleDataCmd.Flags().Int64P("seed", "s", 1, "Seed used for generating the random data (Different seeds generate different data).") 114 sampleDataCmd.Flags().IntP("teams", "t", 2, "The number of sample teams.") 115 sampleDataCmd.Flags().Int("channels-per-team", 10, "The number of sample channels per team.") 116 sampleDataCmd.Flags().IntP("users", "u", 15, "The number of sample users.") 117 sampleDataCmd.Flags().Int("team-memberships", 2, "The number of sample team memberships per user.") 118 sampleDataCmd.Flags().Int("channel-memberships", 5, "The number of sample channel memberships per user in a team.") 119 sampleDataCmd.Flags().Int("posts-per-channel", 100, "The number of sample post per channel.") 120 sampleDataCmd.Flags().Int("direct-channels", 30, "The number of sample direct message channels.") 121 sampleDataCmd.Flags().Int("posts-per-direct-channel", 15, "The number of sample posts per direct message channel.") 122 sampleDataCmd.Flags().Int("group-channels", 15, "The number of sample group message channels.") 123 sampleDataCmd.Flags().Int("posts-per-group-channel", 30, "The number of sample posts per group message channel.") 124 sampleDataCmd.Flags().IntP("workers", "w", 2, "How many workers to run during the import.") 125 sampleDataCmd.Flags().String("profile-images", "", "Optional. Path to folder with images to randomly pick as user profile image.") 126 sampleDataCmd.Flags().StringP("bulk", "b", "", "Optional. Path to write a JSONL bulk file instead of loading into the database.") 127 } 128 129 func sampleDataCmdF(cmd *cobra.Command, args []string) error { 130 a, err := initDBCommandContextCobra(cmd) 131 if err != nil { 132 return err 133 } 134 seed, err := cmd.Flags().GetInt64("seed") 135 if err != nil { 136 return errors.New("Invalid seed parameter") 137 } 138 bulk, err := cmd.Flags().GetString("bulk") 139 if err != nil { 140 return errors.New("Invalid bulk parameter") 141 } 142 teams, err := cmd.Flags().GetInt("teams") 143 if err != nil || teams < 0 { 144 return errors.New("Invalid teams parameter") 145 } 146 channelsPerTeam, err := cmd.Flags().GetInt("channels-per-team") 147 if err != nil || channelsPerTeam < 0 { 148 return errors.New("Invalid channels-per-team parameter") 149 } 150 users, err := cmd.Flags().GetInt("users") 151 if err != nil || users < 0 { 152 return errors.New("Invalid users parameter") 153 } 154 teamMemberships, err := cmd.Flags().GetInt("team-memberships") 155 if err != nil || teamMemberships < 0 { 156 return errors.New("Invalid team-memberships parameter") 157 } 158 channelMemberships, err := cmd.Flags().GetInt("channel-memberships") 159 if err != nil || channelMemberships < 0 { 160 return errors.New("Invalid channel-memberships parameter") 161 } 162 postsPerChannel, err := cmd.Flags().GetInt("posts-per-channel") 163 if err != nil || postsPerChannel < 0 { 164 return errors.New("Invalid posts-per-channel parameter") 165 } 166 directChannels, err := cmd.Flags().GetInt("direct-channels") 167 if err != nil || directChannels < 0 { 168 return errors.New("Invalid direct-channels parameter") 169 } 170 postsPerDirectChannel, err := cmd.Flags().GetInt("posts-per-direct-channel") 171 if err != nil || postsPerDirectChannel < 0 { 172 return errors.New("Invalid posts-per-direct-channel parameter") 173 } 174 groupChannels, err := cmd.Flags().GetInt("group-channels") 175 if err != nil || groupChannels < 0 { 176 return errors.New("Invalid group-channels parameter") 177 } 178 postsPerGroupChannel, err := cmd.Flags().GetInt("posts-per-group-channel") 179 if err != nil || postsPerGroupChannel < 0 { 180 return errors.New("Invalid posts-per-group-channel parameter") 181 } 182 workers, err := cmd.Flags().GetInt("workers") 183 if err != nil { 184 return errors.New("Invalid workers parameter") 185 } 186 profileImagesPath, err := cmd.Flags().GetString("profile-images") 187 if err != nil { 188 return errors.New("Invalid profile-images parameter") 189 } 190 profileImages := []string{} 191 if profileImagesPath != "" { 192 profileImagesStat, err := os.Stat(profileImagesPath) 193 if os.IsNotExist(err) { 194 return errors.New("Profile images folder doesn't exists.") 195 } 196 if !profileImagesStat.IsDir() { 197 return errors.New("profile-images parameters must be a folder path.") 198 } 199 profileImagesFiles, err := ioutil.ReadDir(profileImagesPath) 200 if err != nil { 201 return errors.New("Invalid profile-images parameter") 202 } 203 for _, profileImage := range profileImagesFiles { 204 profileImages = append(profileImages, path.Join(profileImagesPath, profileImage.Name())) 205 } 206 sort.Strings(profileImages) 207 } 208 209 if workers < 1 { 210 return errors.New("You must have at least one worker.") 211 } 212 if teamMemberships > teams { 213 return errors.New("You can't have more team memberships than teams.") 214 } 215 if channelMemberships > channelsPerTeam { 216 return errors.New("You can't have more channel memberships than channels per team.") 217 } 218 219 var bulkFile *os.File 220 switch bulk { 221 case "": 222 bulkFile, err = ioutil.TempFile("", ".mattermost-sample-data-") 223 defer os.Remove(bulkFile.Name()) 224 if err != nil { 225 return errors.New("Unable to open temporary file.") 226 } 227 case "-": 228 bulkFile = os.Stdout 229 default: 230 bulkFile, err = os.OpenFile(bulk, os.O_RDWR|os.O_CREATE, 0755) 231 if err != nil { 232 return errors.New("Unable to write into the \"" + bulk + "\" file.") 233 } 234 } 235 236 encoder := json.NewEncoder(bulkFile) 237 version := 1 238 encoder.Encode(app.LineImportData{Type: "version", Version: &version}) 239 240 fake.Seed(seed) 241 rand.Seed(seed) 242 243 teamsAndChannels := make(map[string][]string) 244 for i := 0; i < teams; i++ { 245 teamLine := createTeam(i) 246 teamsAndChannels[*teamLine.Team.Name] = []string{} 247 encoder.Encode(teamLine) 248 } 249 250 teamsList := []string{} 251 for teamName := range teamsAndChannels { 252 teamsList = append(teamsList, teamName) 253 } 254 sort.Strings(teamsList) 255 256 for _, teamName := range teamsList { 257 for i := 0; i < channelsPerTeam; i++ { 258 channelLine := createChannel(i, teamName) 259 teamsAndChannels[teamName] = append(teamsAndChannels[teamName], *channelLine.Channel.Name) 260 encoder.Encode(channelLine) 261 } 262 } 263 264 allUsers := []string{} 265 for i := 0; i < users; i++ { 266 userLine := createUser(i, teamMemberships, channelMemberships, teamsAndChannels, profileImages) 267 encoder.Encode(userLine) 268 allUsers = append(allUsers, *userLine.User.Username) 269 } 270 271 for team, channels := range teamsAndChannels { 272 for _, channel := range channels { 273 for i := 0; i < postsPerChannel; i++ { 274 postLine := createPost(team, channel, allUsers) 275 encoder.Encode(postLine) 276 } 277 } 278 } 279 280 for i := 0; i < directChannels; i++ { 281 user1 := allUsers[rand.Intn(len(allUsers))] 282 user2 := allUsers[rand.Intn(len(allUsers))] 283 channelLine := createDirectChannel([]string{user1, user2}) 284 encoder.Encode(channelLine) 285 for j := 0; j < postsPerDirectChannel; j++ { 286 postLine := createDirectPost([]string{user1, user2}) 287 encoder.Encode(postLine) 288 } 289 } 290 291 for i := 0; i < groupChannels; i++ { 292 users := []string{} 293 totalUsers := 3 + rand.Intn(3) 294 for len(users) < totalUsers { 295 user := allUsers[rand.Intn(len(allUsers))] 296 if !sliceIncludes(users, user) { 297 users = append(users, user) 298 } 299 } 300 channelLine := createDirectChannel(users) 301 encoder.Encode(channelLine) 302 for j := 0; j < postsPerGroupChannel; j++ { 303 postLine := createDirectPost(users) 304 encoder.Encode(postLine) 305 } 306 } 307 308 if bulk == "" { 309 _, err := bulkFile.Seek(0, 0) 310 if err != nil { 311 return errors.New("Unable to read correctly the temporary file.") 312 } 313 importErr, lineNumber := a.BulkImport(bulkFile, false, workers) 314 if importErr != nil { 315 return errors.New(fmt.Sprintf("%s: %s, %s (line: %d)", importErr.Where, importErr.Message, importErr.DetailedError, lineNumber)) 316 } 317 } else if bulk != "-" { 318 err := bulkFile.Close() 319 if err != nil { 320 return errors.New("Unable to close correctly the output file") 321 } 322 } 323 324 return nil 325 } 326 327 func createUser(idx int, teamMemberships int, channelMemberships int, teamsAndChannels map[string][]string, profileImages []string) app.LineImportData { 328 password := fmt.Sprintf("user-%d", idx) 329 email := fmt.Sprintf("user-%d@sample.mattermost.com", idx) 330 firstName := fake.FirstName() 331 lastName := fake.LastName() 332 username := fmt.Sprintf("%s.%s", strings.ToLower(firstName), strings.ToLower(lastName)) 333 position := fake.JobTitle() 334 roles := "system_user" 335 if idx%5 == 0 { 336 roles = "system_admin system_user" 337 } 338 339 // The 75% of the users have custom profile image 340 var profileImage *string = nil 341 if rand.Intn(4) != 0 { 342 profileImageSelector := rand.Int() 343 if len(profileImages) > 0 { 344 profileImage = &profileImages[profileImageSelector%len(profileImages)] 345 } 346 } 347 348 useMilitaryTime := "false" 349 if idx != 0 && rand.Intn(2) == 0 { 350 useMilitaryTime = "true" 351 } 352 353 collapsePreviews := "false" 354 if idx != 0 && rand.Intn(2) == 0 { 355 collapsePreviews = "true" 356 } 357 358 messageDisplay := "clean" 359 if idx != 0 && rand.Intn(2) == 0 { 360 messageDisplay = "compact" 361 } 362 363 channelDisplayMode := "full" 364 if idx != 0 && rand.Intn(2) == 0 { 365 channelDisplayMode = "centered" 366 } 367 368 // Some users has nickname 369 nickname := "" 370 if rand.Intn(5) == 0 { 371 nickname = fake.Company() 372 } 373 374 // Half of users skip tutorial 375 tutorialStep := "999" 376 switch rand.Intn(6) { 377 case 1: 378 tutorialStep = "1" 379 case 2: 380 tutorialStep = "2" 381 case 3: 382 tutorialStep = "3" 383 } 384 385 teams := []app.UserTeamImportData{} 386 possibleTeams := []string{} 387 for teamName := range teamsAndChannels { 388 possibleTeams = append(possibleTeams, teamName) 389 } 390 sort.Strings(possibleTeams) 391 for x := 0; x < teamMemberships; x++ { 392 if len(possibleTeams) == 0 { 393 break 394 } 395 position := rand.Intn(len(possibleTeams)) 396 team := possibleTeams[position] 397 possibleTeams = append(possibleTeams[:position], possibleTeams[position+1:]...) 398 if teamChannels, err := teamsAndChannels[team]; err == true { 399 teams = append(teams, createTeamMembership(channelMemberships, teamChannels, &team)) 400 } 401 } 402 403 user := app.UserImportData{ 404 ProfileImage: profileImage, 405 Username: &username, 406 Email: &email, 407 Password: &password, 408 Nickname: &nickname, 409 FirstName: &firstName, 410 LastName: &lastName, 411 Position: &position, 412 Roles: &roles, 413 Teams: &teams, 414 UseMilitaryTime: &useMilitaryTime, 415 CollapsePreviews: &collapsePreviews, 416 MessageDisplay: &messageDisplay, 417 ChannelDisplayMode: &channelDisplayMode, 418 TutorialStep: &tutorialStep, 419 } 420 return app.LineImportData{ 421 Type: "user", 422 User: &user, 423 } 424 } 425 426 func createTeamMembership(numOfchannels int, teamChannels []string, teamName *string) app.UserTeamImportData { 427 roles := "team_user" 428 if rand.Intn(5) == 0 { 429 roles = "team_user team_admin" 430 } 431 channels := []app.UserChannelImportData{} 432 teamChannelsCopy := []string{} 433 for _, value := range teamChannels { 434 teamChannelsCopy = append(teamChannelsCopy, value) 435 } 436 for x := 0; x < numOfchannels; x++ { 437 if len(teamChannelsCopy) == 0 { 438 break 439 } 440 position := rand.Intn(len(teamChannelsCopy)) 441 channelName := teamChannelsCopy[position] 442 teamChannelsCopy = append(teamChannelsCopy[:position], teamChannelsCopy[position+1:]...) 443 channels = append(channels, createChannelMembership(channelName)) 444 } 445 446 return app.UserTeamImportData{ 447 Name: teamName, 448 Roles: &roles, 449 Channels: &channels, 450 } 451 } 452 453 func createChannelMembership(channelName string) app.UserChannelImportData { 454 roles := "channel_user" 455 if rand.Intn(5) == 0 { 456 roles = "channel_user channel_admin" 457 } 458 favorite := rand.Intn(5) == 0 459 460 return app.UserChannelImportData{ 461 Name: &channelName, 462 Roles: &roles, 463 Favorite: &favorite, 464 } 465 } 466 467 func createTeam(idx int) app.LineImportData { 468 displayName := fake.Word() 469 name := fmt.Sprintf("%s-%d", fake.Word(), idx) 470 allowOpenInvite := rand.Intn(2) == 0 471 472 description := fake.Paragraph() 473 if len(description) > 255 { 474 description = description[0:255] 475 } 476 477 teamType := "O" 478 if rand.Intn(2) == 0 { 479 teamType = "I" 480 } 481 482 team := app.TeamImportData{ 483 DisplayName: &displayName, 484 Name: &name, 485 AllowOpenInvite: &allowOpenInvite, 486 Description: &description, 487 Type: &teamType, 488 } 489 return app.LineImportData{ 490 Type: "team", 491 Team: &team, 492 } 493 } 494 495 func createChannel(idx int, teamName string) app.LineImportData { 496 displayName := fake.Word() 497 name := fmt.Sprintf("%s-%d", fake.Word(), idx) 498 header := fake.Paragraph() 499 purpose := fake.Paragraph() 500 501 if len(purpose) > 250 { 502 purpose = purpose[0:250] 503 } 504 505 channelType := "P" 506 if rand.Intn(2) == 0 { 507 channelType = "O" 508 } 509 510 channel := app.ChannelImportData{ 511 Team: &teamName, 512 Name: &name, 513 DisplayName: &displayName, 514 Type: &channelType, 515 Header: &header, 516 Purpose: &purpose, 517 } 518 return app.LineImportData{ 519 Type: "channel", 520 Channel: &channel, 521 } 522 } 523 524 func createPost(team string, channel string, allUsers []string) app.LineImportData { 525 message := randomMessage(allUsers) 526 create_at := randomPastTime(50000) 527 user := allUsers[rand.Intn(len(allUsers))] 528 529 // Some messages are flagged by an user 530 flagged_by := []string{} 531 if rand.Intn(10) == 0 { 532 flagged_by = append(flagged_by, allUsers[rand.Intn(len(allUsers))]) 533 } 534 535 reactions := []app.ReactionImportData{} 536 if rand.Intn(10) == 0 { 537 for { 538 reactions = append(reactions, randomReaction(allUsers, create_at)) 539 if rand.Intn(3) == 0 { 540 break 541 } 542 } 543 } 544 545 replies := []app.ReplyImportData{} 546 if rand.Intn(10) == 0 { 547 for { 548 replies = append(replies, randomReply(allUsers, create_at)) 549 if rand.Intn(4) == 0 { 550 break 551 } 552 } 553 } 554 555 post := app.PostImportData{ 556 Team: &team, 557 Channel: &channel, 558 User: &user, 559 Message: &message, 560 CreateAt: &create_at, 561 FlaggedBy: &flagged_by, 562 Reactions: &reactions, 563 Replies: &replies, 564 } 565 return app.LineImportData{ 566 Type: "post", 567 Post: &post, 568 } 569 } 570 571 func createDirectChannel(members []string) app.LineImportData { 572 header := fake.Sentence() 573 574 channel := app.DirectChannelImportData{ 575 Members: &members, 576 Header: &header, 577 } 578 return app.LineImportData{ 579 Type: "direct_channel", 580 DirectChannel: &channel, 581 } 582 } 583 584 func createDirectPost(members []string) app.LineImportData { 585 message := randomMessage(members) 586 create_at := randomPastTime(50000) 587 user := members[rand.Intn(len(members))] 588 589 // Some messages are flagged by an user 590 flagged_by := []string{} 591 if rand.Intn(10) == 0 { 592 flagged_by = append(flagged_by, members[rand.Intn(len(members))]) 593 } 594 595 reactions := []app.ReactionImportData{} 596 if rand.Intn(10) == 0 { 597 for { 598 reactions = append(reactions, randomReaction(members, create_at)) 599 if rand.Intn(3) == 0 { 600 break 601 } 602 } 603 } 604 605 replies := []app.ReplyImportData{} 606 if rand.Intn(10) == 0 { 607 for { 608 replies = append(replies, randomReply(members, create_at)) 609 if rand.Intn(4) == 0 { 610 break 611 } 612 } 613 } 614 615 post := app.DirectPostImportData{ 616 ChannelMembers: &members, 617 User: &user, 618 Message: &message, 619 CreateAt: &create_at, 620 FlaggedBy: &flagged_by, 621 Reactions: &reactions, 622 Replies: &replies, 623 } 624 return app.LineImportData{ 625 Type: "direct_post", 626 DirectPost: &post, 627 } 628 }