github.com/olli-ai/jx/v2@v2.0.400-0.20210921045218-14731b4dd448/pkg/cmd/step/step_blog.go (about) 1 package step 2 3 import ( 4 "bufio" 5 "bytes" 6 "fmt" 7 "io" 8 "io/ioutil" 9 "os" 10 "path/filepath" 11 "sort" 12 "strconv" 13 "strings" 14 "time" 15 16 "github.com/olli-ai/jx/v2/pkg/cmd/opts/step" 17 18 releases2 "github.com/olli-ai/jx/v2/pkg/gits/releases" 19 20 v1 "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io/v1" 21 "github.com/jenkins-x/jx-logging/pkg/log" 22 "github.com/olli-ai/jx/v2/pkg/chats" 23 "github.com/olli-ai/jx/v2/pkg/cmd/helper" 24 "github.com/olli-ai/jx/v2/pkg/cmd/opts" 25 "github.com/olli-ai/jx/v2/pkg/cmd/templates" 26 "github.com/olli-ai/jx/v2/pkg/config" 27 "github.com/olli-ai/jx/v2/pkg/gits" 28 "github.com/olli-ai/jx/v2/pkg/issues" 29 "github.com/olli-ai/jx/v2/pkg/reports" 30 "github.com/olli-ai/jx/v2/pkg/util" 31 "github.com/spf13/cobra" 32 ) 33 34 var ( 35 stepBlogLong = templates.LongDesc(` 36 Generates charts for a project 37 `) 38 39 stepBlogExample = templates.Examples(` 40 # create charts for the cuect 41 jx step chart 42 43 `) 44 45 ignoreNewUsers = map[string]bool{ 46 "GitHub": true, 47 } 48 ) 49 50 // StepBlogOptions contains the command line flags 51 type StepBlogOptions struct { 52 step.StepOptions 53 54 FromDate string 55 ToDate string 56 Dir string 57 BlogOutputDir string 58 BlogName string 59 CombineMinorReleases bool 60 DeveloperChannelMemberCount int 61 UserChannelMemberCount int 62 63 State StepBlogState 64 } 65 66 type StepBlogState struct { 67 GitInfo *gits.GitRepository 68 GitProvider gits.GitProvider 69 Tracker issues.IssueProvider 70 Release *v1.Release 71 BlogFileName string 72 DeveloperChatMetricsName string 73 UserChatMetricsName string 74 Buffer *bytes.Buffer 75 Writer *bufio.Writer 76 HistoryService *reports.ProjectHistoryService 77 History *reports.ProjectHistory 78 NewContributors map[string]*v1.UserDetails 79 NewCommitters map[string]*v1.UserDetails 80 } 81 82 // NewCmdStepBlog Creates a new Command object 83 func NewCmdStepBlog(commonOpts *opts.CommonOptions) *cobra.Command { 84 options := &StepBlogOptions{ 85 StepOptions: step.StepOptions{ 86 CommonOptions: commonOpts, 87 }, 88 } 89 90 cmd := &cobra.Command{ 91 Use: "blog", 92 Short: "Creates a blog post with changes, metrics and charts showing improvements", 93 Long: stepBlogLong, 94 Example: stepBlogExample, 95 Run: func(cmd *cobra.Command, args []string) { 96 options.Cmd = cmd 97 options.Args = args 98 err := options.Run() 99 helper.CheckErr(err) 100 }, 101 } 102 103 cmd.Flags().StringVarP(&options.Dir, "dir", "d", "", "The directory to query to find the projects .git directory") 104 cmd.Flags().StringVarP(&options.FromDate, "from-date", "f", "", "The date to create the charts from. Defaults to a week before the to date. Should be a format: "+util.DateFormat) 105 cmd.Flags().StringVarP(&options.ToDate, "to-date", "t", "", "The date to query up to. Defaults to now. Should be a format: "+util.DateFormat) 106 cmd.Flags().StringVarP(&options.BlogOutputDir, "blog-dir", "", "", "The Hugo-style blog source code to generate the charts into") 107 cmd.Flags().StringVarP(&options.BlogName, "blog-name", "n", "", "The blog name") 108 cmd.Flags().BoolVarP(&options.CombineMinorReleases, "combine-minor", "c", true, "If enabled lets combine minor releases together to simplify the charts") 109 cmd.Flags().IntVarP(&options.DeveloperChannelMemberCount, "dev-channel-members", "", 0, "If no chat bots can connect to your chat server you can pass in the counts for the developer channel here") 110 cmd.Flags().IntVarP(&options.UserChannelMemberCount, "user-channel-members", "", 0, "If no chat bots can connect to your chat server you can pass in the counts for the user channel here") 111 return cmd 112 } 113 114 // Run implements this command 115 func (o *StepBlogOptions) Run() error { 116 o.State = StepBlogState{ 117 NewContributors: map[string]*v1.UserDetails{}, 118 NewCommitters: map[string]*v1.UserDetails{}, 119 } 120 pc, _, err := config.LoadProjectConfig(o.Dir) 121 if err != nil { 122 return err 123 } 124 outDir := o.BlogOutputDir 125 if outDir != "" { 126 if o.BlogName == "" { 127 t := time.Now() 128 o.BlogName = "changes-" + strconv.Itoa(t.Day()) + "-" + strings.ToLower(t.Month().String()) + "-" + strconv.Itoa(t.Year()) 129 } 130 historyFile := filepath.Join(o.BlogOutputDir, "data", "projectHistory.yml") 131 o.State.HistoryService, o.State.History, err = reports.NewProjectHistoryService(historyFile) 132 if err != nil { 133 return err 134 } 135 136 err = o.generateChangelog() 137 if err != nil { 138 return err 139 } 140 } else { 141 gitInfo, gitProvider, tracker, err := o.CreateGitProvider(o.Dir) 142 if err != nil { 143 return err 144 } 145 if gitInfo == nil { 146 return fmt.Errorf("Could not find a .git folder in the current directory so could not determine the current project") 147 } 148 o.State.GitInfo = gitInfo 149 o.State.GitProvider = gitProvider 150 o.State.Tracker = tracker 151 } 152 153 err = o.downloadsReport(o.State.GitProvider, o.State.GitInfo.Organisation, o.State.GitInfo.Name) 154 if err != nil { 155 return err 156 } 157 if pc.Chat != nil { 158 err = o.loadChatMetrics(pc.Chat) 159 if err != nil { 160 return err 161 } 162 } 163 return o.addReportsToBlog() 164 } 165 166 func (o *StepBlogOptions) downloadsReport(provider gits.GitProvider, owner string, repo string) error { 167 releases, err := provider.ListReleases(owner, repo) 168 if err != nil { 169 return err 170 } 171 if len(releases) == 0 { 172 log.Logger().Warnf("No releases found for %s/%s/n", owner, repo) 173 return nil 174 } 175 if o.CombineMinorReleases { 176 releases = o.combineMinorReleases(releases) 177 } 178 release := o.State.Release 179 history := o.State.History 180 if history != nil { 181 history.DownloadMetrics(o.ToDate, releases2.ReleaseDownloadCount(releases)) 182 if release != nil { 183 spec := &release.Spec 184 issuesClosed := len(spec.Issues) 185 queryClosedIssueCount, err := o.queryClosedIssues() 186 if err != nil { 187 log.Logger().Warnf("Failed to query closed issues: %s", err) 188 } 189 if queryClosedIssueCount > issuesClosed { 190 issuesClosed = queryClosedIssueCount 191 } 192 history.IssueMetrics(o.ToDate, issuesClosed) 193 history.PullRequestMetrics(o.ToDate, len(spec.PullRequests)) 194 history.CommitMetrics(o.ToDate, len(spec.Commits)) 195 } 196 197 repo, err := provider.GetRepository(owner, repo) 198 if err != nil { 199 log.Logger().Warnf("Failed to load the repository %s", err) 200 } else { 201 history.StarsMetrics(o.ToDate, repo.Stars) 202 } 203 } 204 205 report := o.createBarReport("downloads", "Version", "Downloads") 206 207 for _, release := range releases { 208 report.AddNumber(release.Name, release.DownloadCount) 209 } 210 return report.Render() 211 } 212 213 // createBarReport creates the new report instance 214 func (o *StepBlogOptions) createBarReport(name string, legends ...string) reports.BarReport { 215 outDir := o.BlogOutputDir 216 if outDir != "" { 217 blogName := o.BlogName 218 if blogName == "" { 219 t := time.Now() 220 blogName = fmt.Sprintf("changes-%d-%s-%d", t.Day(), t.Month().String(), t.Year()) 221 } 222 223 jsDir := filepath.Join(outDir, "static", "news", blogName) 224 err := os.MkdirAll(jsDir, util.DefaultWritePermissions) 225 if err != nil { 226 log.Logger().Warnf("Could not create directory %s: %s", jsDir, err) 227 } 228 jsFileName := filepath.Join(jsDir, name+".js") 229 jsLinkURI := filepath.Join("/news", blogName, name+".js") 230 state := &o.State 231 if state.Buffer == nil { 232 var buffer bytes.Buffer 233 state.Buffer = &buffer 234 } 235 if state.Writer == nil { 236 state.Writer = bufio.NewWriter(state.Buffer) 237 } 238 state.Buffer.WriteString(` 239 240 ## ` + strings.Title(name) + ` 241 242 `) 243 return reports.NewBlogBarReport(name, state.Writer, jsFileName, jsLinkURI) 244 } 245 return reports.NewTableBarReport(o.CreateTable(), legends...) 246 } 247 248 func (options *StepBlogOptions) combineMinorReleases(releases []*gits.GitRelease) []*gits.GitRelease { 249 answer := []*gits.GitRelease{} 250 m := map[string]*gits.GitRelease{} 251 for _, release := range releases { 252 name := release.Name 253 if name != "" { 254 idx := strings.LastIndex(name, ".") 255 if idx > 0 { 256 name = name[0:idx] + ".x" 257 } 258 } 259 cur := m[name] 260 if cur == nil { 261 copy := *release 262 copy.Name = name 263 m[name] = © 264 answer = append(answer, ©) 265 } else { 266 cur.DownloadCount += release.DownloadCount 267 } 268 } 269 return answer 270 } 271 272 func (o *StepBlogOptions) generateChangelog() error { 273 blogFile := filepath.Join(o.BlogOutputDir, "content", "news", o.BlogName+".md") 274 previousDate := o.FromDate 275 now := time.Now() 276 if previousDate == "" { 277 // default to 4 weeks ago 278 t := now.Add(-time.Hour * 24 * 7 * 4) 279 previousDate = util.FormatDate(t) 280 o.FromDate = previousDate 281 } 282 if o.ToDate == "" { 283 o.ToDate = util.FormatDate(now) 284 } 285 options := &StepChangelogOptions{ 286 StepOptions: o.StepOptions, 287 Dir: o.Dir, 288 Version: "Changes since " + previousDate, 289 // TODO add time now and previous time 290 PreviousDate: previousDate, 291 OutputMarkdownFile: blogFile, 292 } 293 err := options.Run() 294 if err != nil { 295 return err 296 } 297 state := &o.State 298 output := &options.State 299 state.GitInfo = output.GitInfo 300 state.GitProvider = output.GitProvider 301 state.Tracker = output.Tracker 302 state.Release = output.Release 303 state.BlogFileName = blogFile 304 return nil 305 } 306 307 func (o *StepBlogOptions) addReportsToBlog() error { 308 state := &o.State 309 if state.BlogFileName != "" { 310 data, err := ioutil.ReadFile(state.BlogFileName) 311 if err != nil { 312 return err 313 } 314 toDate := o.ToDate 315 fromDate := o.FromDate 316 committersText := o.createNewCommitters() 317 318 prefix := `--- 319 title: "Changes for ` + toDate + `" 320 date: ` + time.Now().Format(time.RFC3339) + ` 321 description: "Whats new for ` + toDate + `" 322 categories: [blog] 323 keywords: [] 324 slug: "changes-` + strings.Replace(toDate, " ", "-", -1) + `" 325 aliases: [] 326 author: jenkins-x-bot 327 --- 328 329 ## Changes for ` + toDate + ` 330 331 This blog outlines the changes on the project from ` + fromDate + ` to ` + toDate + `. 332 333 ` + o.createMetricsSummary() + ` 334 335 [View Charts](#charts) 336 337 ` + committersText 338 339 postfix := "" 340 if state.Writer != nil { 341 state.Writer.Flush() 342 postfix = ` 343 344 ## Charts 345 346 ` + state.Buffer.String() + ` 347 348 This blog post was generated via the [jx step blog](https://jenkins-x.io/commands/jx_step_blog/) command from [Jenkins X](https://jenkins-x.io/). 349 ` 350 351 } 352 changelog := strings.TrimSpace(string(data)) 353 changelog = strings.TrimPrefix(changelog, "## Changes") 354 text := prefix + changelog + postfix 355 err = ioutil.WriteFile(state.BlogFileName, []byte(text), util.DefaultWritePermissions) 356 if err != nil { 357 return err 358 } 359 } 360 historyService := state.HistoryService 361 if historyService != nil { 362 return historyService.SaveHistory() 363 } 364 return nil 365 } 366 367 func (o *StepBlogOptions) createMetricsSummary() string { 368 var buffer bytes.Buffer 369 out := bufio.NewWriter(&buffer) 370 _, report := o.report() 371 if report != nil { 372 developerChatMetricsName := o.State.DeveloperChatMetricsName 373 if developerChatMetricsName == "" { 374 developerChatMetricsName = "Developer Chat Members" 375 } 376 userChatMetricsName := o.State.UserChatMetricsName 377 if userChatMetricsName == "" { 378 userChatMetricsName = "User Chat Members" 379 } 380 381 fmt.Fprintf(out, "| Metrics | Changes | Total |\n") 382 fmt.Fprintf(out, "| :---------- | -------:| -----:|\n") 383 o.printMetrics(out, "Downloads", &report.DownloadMetrics) 384 o.printMetrics(out, "Stars", &report.StarsMetrics) 385 o.printMetrics(out, "New Committers", &report.NewCommitterMetrics) 386 o.printMetrics(out, "New Contributors", &report.NewContributorMetrics) 387 o.printMetrics(out, developerChatMetricsName, &report.DeveloperChatMetrics) 388 o.printMetrics(out, userChatMetricsName, &report.UserChatMetrics) 389 o.printMetrics(out, "Issues Closed", &report.IssueMetrics) 390 o.printMetrics(out, "Pull Requests Merged", &report.PullRequestMetrics) 391 o.printMetrics(out, "Commits", &report.CommitMetrics) 392 } 393 out.Flush() 394 return buffer.String() 395 } 396 397 func (o *StepBlogOptions) report() (*reports.ProjectHistory, *reports.ProjectReport) { 398 history := o.State.History 399 if history != nil { 400 toDate := o.ToDate 401 report := history.FindReport(toDate) 402 if report == nil { 403 log.Logger().Warnf("No report for date %s", toDate) 404 } 405 return history, report 406 } 407 return nil, nil 408 } 409 410 func (o *StepBlogOptions) printMetrics(out io.Writer, name string, metrics *reports.CountMetrics) { 411 count := metrics.Count 412 total := metrics.Total 413 if count > 0 || total > 0 { 414 fmt.Fprintf(out, "| %s | **%d** | **%d** |\n", name, count, total) 415 } 416 } 417 418 func (o *StepBlogOptions) createNewCommitters() string { 419 release := o.State.Release 420 if release != nil { 421 spec := &release.Spec 422 // TODO commits typically don't have a login so lets ignore for now 423 // and assume that they show up in the PRs 424 /* 425 for _, commit := range spec.Commits { 426 o.addCommitter(commit.Committer) 427 o.addCommitter(commit.Author) 428 } 429 */ 430 for _, pr := range spec.PullRequests { 431 o.addCommitter(pr.User) 432 o.addCommitters(pr.Assignees) 433 } 434 for _, issue := range spec.Issues { 435 o.addContributor(issue.User) 436 o.addContributors(issue.Assignees) 437 } 438 } else { 439 log.Logger().Warnf("No Release!") 440 } 441 history, _ := o.report() 442 if history != nil { 443 history.NewContributorMetrics(o.ToDate, len(o.State.NewContributors)) 444 history.NewCommitterMetrics(o.ToDate, len(o.State.NewCommitters)) 445 446 // now lets remove the current contributors 447 for _, user := range history.Committers { 448 delete(o.State.NewCommitters, user) 449 } 450 for _, user := range history.Contributors { 451 delete(o.State.NewContributors, user) 452 } 453 454 // now lets add the new users to the history for the next blog 455 for _, user := range o.State.NewCommitters { 456 history.Committers = append(history.Committers, user.Login) 457 } 458 for _, user := range o.State.NewContributors { 459 history.Contributors = append(history.Contributors, user.Login) 460 } 461 } 462 463 var buffer bytes.Buffer 464 out := bufio.NewWriter(&buffer) 465 o.printUserMap(out, "committers", o.State.NewCommitters) 466 o.printUserMap(out, "contributors", o.State.NewContributors) 467 out.Flush() //nolint:errcheck 468 return buffer.String() 469 } 470 471 //nolint:errcheck 472 func (o *StepBlogOptions) printUserMap(out io.Writer, role string, newUsers map[string]*v1.UserDetails) { 473 if len(newUsers) > 0 { 474 out.Write([]byte(` 475 476 ## New ` + strings.Title(role) + ` 477 478 Welcome to our new ` + role + `! 479 480 `)) 481 482 keys := []string{} 483 for k := range newUsers { 484 keys = append(keys, k) 485 } 486 sort.Strings(keys) 487 for _, k := range keys { 488 user := newUsers[k] 489 if user != nil { 490 out.Write([]byte("* " + o.formatUser(user) + "\n")) //nolint:errcheck 491 } 492 } 493 } 494 } 495 496 func (o *StepBlogOptions) addCommitters(users []v1.UserDetails) { 497 for _, u := range users { 498 user := u 499 o.addCommitter(&user) 500 } 501 } 502 503 func (o *StepBlogOptions) addContributors(users []v1.UserDetails) { 504 for _, u := range users { 505 user := u 506 o.addContributor(&user) 507 } 508 } 509 510 func (o *StepBlogOptions) addContributor(user *v1.UserDetails) { 511 o.addUser(user, &o.State.NewContributors) 512 } 513 514 func (o *StepBlogOptions) addCommitter(user *v1.UserDetails) { 515 o.addUser(user, &o.State.NewCommitters) 516 o.addContributor(user) 517 } 518 519 func (o *StepBlogOptions) addUser(user *v1.UserDetails, newUsers *map[string]*v1.UserDetails) { 520 if user != nil { 521 key := user.Login 522 if key == "" { 523 key = user.Name 524 } 525 oldUser := (*newUsers)[key] 526 if key != "" && !ignoreNewUsers[key] && oldUser == nil || user.URL != "" { 527 (*newUsers)[key] = user 528 } 529 } 530 } 531 532 func (o *StepBlogOptions) formatUser(user *v1.UserDetails) string { 533 u := user.URL 534 login := user.Login 535 if u == "" { 536 u = util.UrlJoin(o.State.GitProvider.ServerURL(), login) 537 } 538 name := user.Name 539 if name == "" { 540 name = login 541 } 542 if name == "" { 543 name = u 544 } 545 if name == "" { 546 return "" 547 } 548 prefix := "" 549 avatar := user.AvatarURL 550 if avatar != "" { 551 prefix = "<img class='avatar' src='" + avatar + "' height='32' width='32'> " 552 } 553 label := prefix + name 554 if u != "" { 555 return "<a href='" + u + "' title='" + user.Name + "'>" + label + "</a>" 556 } 557 return label 558 } 559 560 func (o *StepBlogOptions) loadChatMetrics(chatConfig *config.ChatConfig) error { 561 u := chatConfig.URL 562 if u == "" { 563 return nil 564 } 565 history, _ := o.report() 566 if history == nil { 567 return nil 568 } 569 const membersPostfix = " members" 570 devChannel := chatConfig.DeveloperChannel 571 if devChannel != "" { 572 count := o.DeveloperChannelMemberCount 573 if count > 0 { 574 o.State.DeveloperChatMetricsName = devChannel + membersPostfix 575 } else { 576 metrics, err := o.getChannelMetrics(chatConfig, devChannel) 577 if err != nil { 578 log.Logger().Warnf("Failed to get chat metrics for channel %s: %s", devChannel, err) 579 return nil 580 } 581 count = metrics.MemberCount 582 o.State.DeveloperChatMetricsName = metrics.ToMarkdown() + membersPostfix 583 } 584 history.DeveloperChatMetrics(o.ToDate, count) 585 } 586 userChannel := chatConfig.UserChannel 587 if userChannel != "" { 588 count := o.UserChannelMemberCount 589 if count > 0 { 590 o.State.UserChatMetricsName = userChannel + membersPostfix 591 } else { 592 metrics, err := o.getChannelMetrics(chatConfig, userChannel) 593 if err != nil { 594 log.Logger().Warnf("Failed to get chat metrics for channel %s: %s", userChannel, err) 595 return nil 596 } 597 count = metrics.MemberCount 598 o.State.UserChatMetricsName = metrics.ToMarkdown() + membersPostfix 599 } 600 history.UserChatMetrics(o.ToDate, count) 601 } 602 return nil 603 } 604 605 func (o *StepBlogOptions) getChannelMetrics(chatConfig *config.ChatConfig, channelName string) (*chats.ChannelMetrics, error) { 606 provider, err := o.CreateChatProvider(chatConfig) 607 if err != nil { 608 return nil, err 609 } 610 return provider.GetChannelMetrics(channelName) 611 } 612 613 func (o *StepBlogOptions) queryClosedIssues() (int, error) { 614 fromDate := o.FromDate 615 if fromDate == "" { 616 return 0, fmt.Errorf("No from date specified!") 617 } 618 t, err := util.ParseDate(fromDate) 619 if err != nil { 620 return 0, fmt.Errorf("Failed to parse from date: %s: %s", fromDate, err) 621 } 622 issues, err := o.State.Tracker.SearchIssuesClosedSince(t) 623 count := len(issues) 624 return count, err 625 } 626 627 // CreateChatProvider creates a new chart provider from the given configuration 628 func (o *StepBlogOptions) CreateChatProvider(chatConfig *config.ChatConfig) (chats.ChatProvider, error) { 629 u := chatConfig.URL 630 if u == "" { 631 return nil, nil 632 } 633 authConfigSvc, err := o.CreateChatAuthConfigService("") 634 if err != nil { 635 return nil, err 636 } 637 config := authConfigSvc.Config() 638 639 server := config.GetOrCreateServer(u) 640 userAuth, err := config.PickServerUserAuth(server, "user to access the chat service at "+u, o.BatchMode, "", o.GetIOFileHandles()) 641 if err != nil { 642 return nil, err 643 } 644 return chats.CreateChatProvider(server.Kind, server, userAuth, o.BatchMode) 645 }