github.com/demisto/mattermost-server@v4.9.0-rc3+incompatible/cmd/commands/sampledata.go (about) 1 // Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved. 2 // See License.txt for license information. 3 4 package commands 5 6 import ( 7 "encoding/json" 8 "errors" 9 "fmt" 10 "io/ioutil" 11 "math/rand" 12 "os" 13 "path" 14 "sort" 15 "strings" 16 "time" 17 18 "github.com/icrowley/fake" 19 "github.com/mattermost/mattermost-server/app" 20 "github.com/mattermost/mattermost-server/cmd" 21 "github.com/spf13/cobra" 22 ) 23 24 var SampleDataCmd = &cobra.Command{ 25 Use: "sampledata", 26 Short: "Generate sample data", 27 RunE: sampleDataCmdF, 28 } 29 30 func init() { 31 SampleDataCmd.Flags().Int64P("seed", "s", 1, "Seed used for generating the random data (Different seeds generate different data).") 32 SampleDataCmd.Flags().IntP("teams", "t", 2, "The number of sample teams.") 33 SampleDataCmd.Flags().Int("channels-per-team", 10, "The number of sample channels per team.") 34 SampleDataCmd.Flags().IntP("users", "u", 15, "The number of sample users.") 35 SampleDataCmd.Flags().Int("team-memberships", 2, "The number of sample team memberships per user.") 36 SampleDataCmd.Flags().Int("channel-memberships", 5, "The number of sample channel memberships per user in a team.") 37 SampleDataCmd.Flags().Int("posts-per-channel", 100, "The number of sample post per channel.") 38 SampleDataCmd.Flags().Int("direct-channels", 30, "The number of sample direct message channels.") 39 SampleDataCmd.Flags().Int("posts-per-direct-channel", 15, "The number of sample posts per direct message channel.") 40 SampleDataCmd.Flags().Int("group-channels", 15, "The number of sample group message channels.") 41 SampleDataCmd.Flags().Int("posts-per-group-channel", 30, "The number of sample posts per group message channel.") 42 SampleDataCmd.Flags().IntP("workers", "w", 2, "How many workers to run during the import.") 43 SampleDataCmd.Flags().String("profile-images", "", "Optional. Path to folder with images to randomly pick as user profile image.") 44 SampleDataCmd.Flags().StringP("bulk", "b", "", "Optional. Path to write a JSONL bulk file instead of loading into the database.") 45 cmd.RootCmd.AddCommand(SampleDataCmd) 46 } 47 48 func sliceIncludes(vs []string, t string) bool { 49 for _, v := range vs { 50 if v == t { 51 return true 52 } 53 } 54 return false 55 } 56 57 func randomPastTime(seconds int) int64 { 58 now := time.Now() 59 today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, time.FixedZone("UTC", 0)) 60 return today.Unix() - int64(rand.Intn(seconds*1000)) 61 } 62 63 func randomEmoji() string { 64 emojis := []string{"+1", "-1", "heart", "blush"} 65 return emojis[rand.Intn(len(emojis))] 66 } 67 68 func randomReaction(users []string, parentCreateAt int64) app.ReactionImportData { 69 user := users[rand.Intn(len(users))] 70 emoji := randomEmoji() 71 date := parentCreateAt + int64(rand.Intn(100000)) 72 return app.ReactionImportData{ 73 User: &user, 74 EmojiName: &emoji, 75 CreateAt: &date, 76 } 77 } 78 79 func randomReply(users []string, parentCreateAt int64) app.ReplyImportData { 80 user := users[rand.Intn(len(users))] 81 message := randomMessage(users) 82 date := parentCreateAt + int64(rand.Intn(100000)) 83 return app.ReplyImportData{ 84 User: &user, 85 Message: &message, 86 CreateAt: &date, 87 } 88 } 89 90 func randomMessage(users []string) string { 91 var message string 92 switch rand.Intn(30) { 93 case 0: 94 mention := users[rand.Intn(len(users))] 95 message = "@" + mention + " " + fake.Sentence() 96 case 1: 97 switch rand.Intn(2) { 98 case 0: 99 mattermostVideos := []string{"Q4MgnxbpZas", "BFo7E9-Kc_E", "LsMLR-BHsKg", "MRmGDhlMhNA", "mUOPxT7VgWc"} 100 message = "https://www.youtube.com/watch?v=" + mattermostVideos[rand.Intn(len(mattermostVideos))] 101 case 1: 102 mattermostTweets := []string{"943119062334353408", "949370809528832005", "948539688171819009", "939122439115681792", "938061722027425797"} 103 message = "https://twitter.com/mattermosthq/status/" + mattermostTweets[rand.Intn(len(mattermostTweets))] 104 } 105 case 2: 106 message = "" 107 if rand.Intn(2) == 0 { 108 message += fake.Sentence() 109 } 110 for i := 0; i < rand.Intn(4)+1; i++ { 111 message += "\n * " + fake.Word() 112 } 113 default: 114 if rand.Intn(2) == 0 { 115 message = fake.Sentence() 116 } else { 117 message = fake.Paragraph() 118 } 119 if rand.Intn(3) == 0 { 120 message += "\n" + fake.Sentence() 121 } 122 if rand.Intn(3) == 0 { 123 message += "\n" + fake.Sentence() 124 } 125 if rand.Intn(3) == 0 { 126 message += "\n" + fake.Sentence() 127 } 128 } 129 return message 130 } 131 132 func sampleDataCmdF(command *cobra.Command, args []string) error { 133 a, err := cmd.InitDBCommandContextCobra(command) 134 if err != nil { 135 return err 136 } 137 seed, err := command.Flags().GetInt64("seed") 138 if err != nil { 139 return errors.New("Invalid seed parameter") 140 } 141 bulk, err := command.Flags().GetString("bulk") 142 if err != nil { 143 return errors.New("Invalid bulk parameter") 144 } 145 teams, err := command.Flags().GetInt("teams") 146 if err != nil || teams < 0 { 147 return errors.New("Invalid teams parameter") 148 } 149 channelsPerTeam, err := command.Flags().GetInt("channels-per-team") 150 if err != nil || channelsPerTeam < 0 { 151 return errors.New("Invalid channels-per-team parameter") 152 } 153 users, err := command.Flags().GetInt("users") 154 if err != nil || users < 0 { 155 return errors.New("Invalid users parameter") 156 } 157 teamMemberships, err := command.Flags().GetInt("team-memberships") 158 if err != nil || teamMemberships < 0 { 159 return errors.New("Invalid team-memberships parameter") 160 } 161 channelMemberships, err := command.Flags().GetInt("channel-memberships") 162 if err != nil || channelMemberships < 0 { 163 return errors.New("Invalid channel-memberships parameter") 164 } 165 postsPerChannel, err := command.Flags().GetInt("posts-per-channel") 166 if err != nil || postsPerChannel < 0 { 167 return errors.New("Invalid posts-per-channel parameter") 168 } 169 directChannels, err := command.Flags().GetInt("direct-channels") 170 if err != nil || directChannels < 0 { 171 return errors.New("Invalid direct-channels parameter") 172 } 173 postsPerDirectChannel, err := command.Flags().GetInt("posts-per-direct-channel") 174 if err != nil || postsPerDirectChannel < 0 { 175 return errors.New("Invalid posts-per-direct-channel parameter") 176 } 177 groupChannels, err := command.Flags().GetInt("group-channels") 178 if err != nil || groupChannels < 0 { 179 return errors.New("Invalid group-channels parameter") 180 } 181 postsPerGroupChannel, err := command.Flags().GetInt("posts-per-group-channel") 182 if err != nil || postsPerGroupChannel < 0 { 183 return errors.New("Invalid posts-per-group-channel parameter") 184 } 185 workers, err := command.Flags().GetInt("workers") 186 if err != nil { 187 return errors.New("Invalid workers parameter") 188 } 189 profileImagesPath, err := command.Flags().GetString("profile-images") 190 if err != nil { 191 return errors.New("Invalid profile-images parameter") 192 } 193 profileImages := []string{} 194 if profileImagesPath != "" { 195 profileImagesStat, err := os.Stat(profileImagesPath) 196 if os.IsNotExist(err) { 197 return errors.New("Profile images folder doesn't exists.") 198 } 199 if !profileImagesStat.IsDir() { 200 return errors.New("profile-images parameters must be a folder path.") 201 } 202 profileImagesFiles, err := ioutil.ReadDir(profileImagesPath) 203 if err != nil { 204 return errors.New("Invalid profile-images parameter") 205 } 206 for _, profileImage := range profileImagesFiles { 207 profileImages = append(profileImages, path.Join(profileImagesPath, profileImage.Name())) 208 } 209 sort.Strings(profileImages) 210 } 211 212 if workers < 1 { 213 return errors.New("You must have at least one worker.") 214 } 215 if teamMemberships > teams { 216 return errors.New("You can't have more team memberships than teams.") 217 } 218 if channelMemberships > channelsPerTeam { 219 return errors.New("You can't have more channel memberships than channels per team.") 220 } 221 222 var bulkFile *os.File 223 switch bulk { 224 case "": 225 bulkFile, err = ioutil.TempFile("", ".mattermost-sample-data-") 226 defer os.Remove(bulkFile.Name()) 227 if err != nil { 228 return errors.New("Unable to open temporary file.") 229 } 230 case "-": 231 bulkFile = os.Stdout 232 default: 233 bulkFile, err = os.OpenFile(bulk, os.O_RDWR|os.O_CREATE, 0755) 234 if err != nil { 235 return errors.New("Unable to write into the \"" + bulk + "\" file.") 236 } 237 } 238 239 encoder := json.NewEncoder(bulkFile) 240 version := 1 241 encoder.Encode(app.LineImportData{Type: "version", Version: &version}) 242 243 fake.Seed(seed) 244 rand.Seed(seed) 245 246 teamsAndChannels := make(map[string][]string) 247 for i := 0; i < teams; i++ { 248 teamLine := createTeam(i) 249 teamsAndChannels[*teamLine.Team.Name] = []string{} 250 encoder.Encode(teamLine) 251 } 252 253 teamsList := []string{} 254 for teamName := range teamsAndChannels { 255 teamsList = append(teamsList, teamName) 256 } 257 sort.Strings(teamsList) 258 259 for _, teamName := range teamsList { 260 for i := 0; i < channelsPerTeam; i++ { 261 channelLine := createChannel(i, teamName) 262 teamsAndChannels[teamName] = append(teamsAndChannels[teamName], *channelLine.Channel.Name) 263 encoder.Encode(channelLine) 264 } 265 } 266 267 allUsers := []string{} 268 for i := 0; i < users; i++ { 269 userLine := createUser(i, teamMemberships, channelMemberships, teamsAndChannels, profileImages) 270 encoder.Encode(userLine) 271 allUsers = append(allUsers, *userLine.User.Username) 272 } 273 274 for team, channels := range teamsAndChannels { 275 for _, channel := range channels { 276 for i := 0; i < postsPerChannel; i++ { 277 postLine := createPost(team, channel, allUsers) 278 encoder.Encode(postLine) 279 } 280 } 281 } 282 283 for i := 0; i < directChannels; i++ { 284 user1 := allUsers[rand.Intn(len(allUsers))] 285 user2 := allUsers[rand.Intn(len(allUsers))] 286 channelLine := createDirectChannel([]string{user1, user2}) 287 encoder.Encode(channelLine) 288 for j := 0; j < postsPerDirectChannel; j++ { 289 postLine := createDirectPost([]string{user1, user2}) 290 encoder.Encode(postLine) 291 } 292 } 293 294 for i := 0; i < groupChannels; i++ { 295 users := []string{} 296 totalUsers := 3 + rand.Intn(3) 297 for len(users) < totalUsers { 298 user := allUsers[rand.Intn(len(allUsers))] 299 if !sliceIncludes(users, user) { 300 users = append(users, user) 301 } 302 } 303 channelLine := createDirectChannel(users) 304 encoder.Encode(channelLine) 305 for j := 0; j < postsPerGroupChannel; j++ { 306 postLine := createDirectPost(users) 307 encoder.Encode(postLine) 308 } 309 } 310 311 if bulk == "" { 312 _, err := bulkFile.Seek(0, 0) 313 if err != nil { 314 return errors.New("Unable to read correctly the temporary file.") 315 } 316 importErr, lineNumber := a.BulkImport(bulkFile, false, workers) 317 if importErr != nil { 318 return fmt.Errorf("%s: %s, %s (line: %d)", importErr.Where, importErr.Message, importErr.DetailedError, lineNumber) 319 } 320 } else if bulk != "-" { 321 err := bulkFile.Close() 322 if err != nil { 323 return errors.New("Unable to close correctly the output file") 324 } 325 } 326 327 return nil 328 } 329 330 func createUser(idx int, teamMemberships int, channelMemberships int, teamsAndChannels map[string][]string, profileImages []string) app.LineImportData { 331 password := fmt.Sprintf("user-%d", idx) 332 email := fmt.Sprintf("user-%d@sample.mattermost.com", idx) 333 firstName := fake.FirstName() 334 lastName := fake.LastName() 335 username := fmt.Sprintf("%s.%s", strings.ToLower(firstName), strings.ToLower(lastName)) 336 if idx == 0 { 337 username = "sysadmin" 338 password = "sysadmin" 339 email = "sysadmin@sample.mattermost.com" 340 } else if idx == 1 { 341 username = "user-1" 342 } 343 position := fake.JobTitle() 344 roles := "system_user" 345 if idx%5 == 0 { 346 roles = "system_admin system_user" 347 } 348 349 // The 75% of the users have custom profile image 350 var profileImage *string = nil 351 if rand.Intn(4) != 0 { 352 profileImageSelector := rand.Int() 353 if len(profileImages) > 0 { 354 profileImage = &profileImages[profileImageSelector%len(profileImages)] 355 } 356 } 357 358 useMilitaryTime := "false" 359 if idx != 0 && rand.Intn(2) == 0 { 360 useMilitaryTime = "true" 361 } 362 363 collapsePreviews := "false" 364 if idx != 0 && rand.Intn(2) == 0 { 365 collapsePreviews = "true" 366 } 367 368 messageDisplay := "clean" 369 if idx != 0 && rand.Intn(2) == 0 { 370 messageDisplay = "compact" 371 } 372 373 channelDisplayMode := "full" 374 if idx != 0 && rand.Intn(2) == 0 { 375 channelDisplayMode = "centered" 376 } 377 378 // Some users has nickname 379 nickname := "" 380 if rand.Intn(5) == 0 { 381 nickname = fake.Company() 382 } 383 384 // Half of users skip tutorial 385 tutorialStep := "999" 386 switch rand.Intn(6) { 387 case 1: 388 tutorialStep = "1" 389 case 2: 390 tutorialStep = "2" 391 case 3: 392 tutorialStep = "3" 393 } 394 395 teams := []app.UserTeamImportData{} 396 possibleTeams := []string{} 397 for teamName := range teamsAndChannels { 398 possibleTeams = append(possibleTeams, teamName) 399 } 400 sort.Strings(possibleTeams) 401 for x := 0; x < teamMemberships; x++ { 402 if len(possibleTeams) == 0 { 403 break 404 } 405 position := rand.Intn(len(possibleTeams)) 406 team := possibleTeams[position] 407 possibleTeams = append(possibleTeams[:position], possibleTeams[position+1:]...) 408 if teamChannels, err := teamsAndChannels[team]; err { 409 teams = append(teams, createTeamMembership(channelMemberships, teamChannels, &team)) 410 } 411 } 412 413 user := app.UserImportData{ 414 ProfileImage: profileImage, 415 Username: &username, 416 Email: &email, 417 Password: &password, 418 Nickname: &nickname, 419 FirstName: &firstName, 420 LastName: &lastName, 421 Position: &position, 422 Roles: &roles, 423 Teams: &teams, 424 UseMilitaryTime: &useMilitaryTime, 425 CollapsePreviews: &collapsePreviews, 426 MessageDisplay: &messageDisplay, 427 ChannelDisplayMode: &channelDisplayMode, 428 TutorialStep: &tutorialStep, 429 } 430 return app.LineImportData{ 431 Type: "user", 432 User: &user, 433 } 434 } 435 436 func createTeamMembership(numOfchannels int, teamChannels []string, teamName *string) app.UserTeamImportData { 437 roles := "team_user" 438 if rand.Intn(5) == 0 { 439 roles = "team_user team_admin" 440 } 441 channels := []app.UserChannelImportData{} 442 teamChannelsCopy := append([]string(nil), teamChannels...) 443 for x := 0; x < numOfchannels; x++ { 444 if len(teamChannelsCopy) == 0 { 445 break 446 } 447 position := rand.Intn(len(teamChannelsCopy)) 448 channelName := teamChannelsCopy[position] 449 teamChannelsCopy = append(teamChannelsCopy[:position], teamChannelsCopy[position+1:]...) 450 channels = append(channels, createChannelMembership(channelName)) 451 } 452 453 return app.UserTeamImportData{ 454 Name: teamName, 455 Roles: &roles, 456 Channels: &channels, 457 } 458 } 459 460 func createChannelMembership(channelName string) app.UserChannelImportData { 461 roles := "channel_user" 462 if rand.Intn(5) == 0 { 463 roles = "channel_user channel_admin" 464 } 465 favorite := rand.Intn(5) == 0 466 467 return app.UserChannelImportData{ 468 Name: &channelName, 469 Roles: &roles, 470 Favorite: &favorite, 471 } 472 } 473 474 func createTeam(idx int) app.LineImportData { 475 displayName := fake.Word() 476 name := fmt.Sprintf("%s-%d", fake.Word(), idx) 477 allowOpenInvite := rand.Intn(2) == 0 478 479 description := fake.Paragraph() 480 if len(description) > 255 { 481 description = description[0:255] 482 } 483 484 teamType := "O" 485 if rand.Intn(2) == 0 { 486 teamType = "I" 487 } 488 489 team := app.TeamImportData{ 490 DisplayName: &displayName, 491 Name: &name, 492 AllowOpenInvite: &allowOpenInvite, 493 Description: &description, 494 Type: &teamType, 495 } 496 return app.LineImportData{ 497 Type: "team", 498 Team: &team, 499 } 500 } 501 502 func createChannel(idx int, teamName string) app.LineImportData { 503 displayName := fake.Word() 504 name := fmt.Sprintf("%s-%d", fake.Word(), idx) 505 header := fake.Paragraph() 506 purpose := fake.Paragraph() 507 508 if len(purpose) > 250 { 509 purpose = purpose[0:250] 510 } 511 512 channelType := "P" 513 if rand.Intn(2) == 0 { 514 channelType = "O" 515 } 516 517 channel := app.ChannelImportData{ 518 Team: &teamName, 519 Name: &name, 520 DisplayName: &displayName, 521 Type: &channelType, 522 Header: &header, 523 Purpose: &purpose, 524 } 525 return app.LineImportData{ 526 Type: "channel", 527 Channel: &channel, 528 } 529 } 530 531 func createPost(team string, channel string, allUsers []string) app.LineImportData { 532 message := randomMessage(allUsers) 533 create_at := randomPastTime(50000) 534 user := allUsers[rand.Intn(len(allUsers))] 535 536 // Some messages are flagged by an user 537 flagged_by := []string{} 538 if rand.Intn(10) == 0 { 539 flagged_by = append(flagged_by, allUsers[rand.Intn(len(allUsers))]) 540 } 541 542 reactions := []app.ReactionImportData{} 543 if rand.Intn(10) == 0 { 544 for { 545 reactions = append(reactions, randomReaction(allUsers, create_at)) 546 if rand.Intn(3) == 0 { 547 break 548 } 549 } 550 } 551 552 replies := []app.ReplyImportData{} 553 if rand.Intn(10) == 0 { 554 for { 555 replies = append(replies, randomReply(allUsers, create_at)) 556 if rand.Intn(4) == 0 { 557 break 558 } 559 } 560 } 561 562 post := app.PostImportData{ 563 Team: &team, 564 Channel: &channel, 565 User: &user, 566 Message: &message, 567 CreateAt: &create_at, 568 FlaggedBy: &flagged_by, 569 Reactions: &reactions, 570 Replies: &replies, 571 } 572 return app.LineImportData{ 573 Type: "post", 574 Post: &post, 575 } 576 } 577 578 func createDirectChannel(members []string) app.LineImportData { 579 header := fake.Sentence() 580 581 channel := app.DirectChannelImportData{ 582 Members: &members, 583 Header: &header, 584 } 585 return app.LineImportData{ 586 Type: "direct_channel", 587 DirectChannel: &channel, 588 } 589 } 590 591 func createDirectPost(members []string) app.LineImportData { 592 message := randomMessage(members) 593 create_at := randomPastTime(50000) 594 user := members[rand.Intn(len(members))] 595 596 // Some messages are flagged by an user 597 flagged_by := []string{} 598 if rand.Intn(10) == 0 { 599 flagged_by = append(flagged_by, members[rand.Intn(len(members))]) 600 } 601 602 reactions := []app.ReactionImportData{} 603 if rand.Intn(10) == 0 { 604 for { 605 reactions = append(reactions, randomReaction(members, create_at)) 606 if rand.Intn(3) == 0 { 607 break 608 } 609 } 610 } 611 612 replies := []app.ReplyImportData{} 613 if rand.Intn(10) == 0 { 614 for { 615 replies = append(replies, randomReply(members, create_at)) 616 if rand.Intn(4) == 0 { 617 break 618 } 619 } 620 } 621 622 post := app.DirectPostImportData{ 623 ChannelMembers: &members, 624 User: &user, 625 Message: &message, 626 CreateAt: &create_at, 627 FlaggedBy: &flagged_by, 628 Reactions: &reactions, 629 Replies: &replies, 630 } 631 return app.LineImportData{ 632 Type: "direct_post", 633 DirectPost: &post, 634 } 635 }