golang.org/x/build@v0.0.0-20240506185731-218518f32b70/cmd/coordinator/internal/dashboard/handler.go

// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build linux || darwin

// Package dashboard contains the implementation of the build dashboard for the Coordinator.
package dashboard

import (
	"bytes"
	"context"
	_ "embed"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"sort"
	"strconv"
	"strings"
	"time"

	"cloud.google.com/go/datastore"
	"golang.org/x/build/cmd/coordinator/internal/lucipoll"
	"golang.org/x/build/dashboard"
	"golang.org/x/build/internal/releasetargets"
	"golang.org/x/build/maintner/maintnerd/apipb"
	"google.golang.org/grpc"
)

type data struct {
	Branch    string
	Builders  []*builder
	Commits   []*commit
	Dashboard struct {
		Name string
	}
	Package    dashPackage
	Pagination *struct{}
	TagState   []struct{}
}

// MaintnerClient is a subset of apipb.MaintnerServiceClient.
type MaintnerClient interface {
	// GetDashboard is extracted from apipb.MaintnerServiceClient.
	GetDashboard(ctx context.Context, in *apipb.DashboardRequest, opts ...grpc.CallOption) (*apipb.DashboardResponse, error)
}

type luciClient interface {
	PostSubmitSnapshot() lucipoll.Snapshot
}

type Handler struct {
	// Datastore is a client used for fetching build status. If nil, it uses in-memory storage of build status.
	Datastore *datastore.Client
	// Maintner is a client for Maintner, used for fetching lists of commits.
	Maintner MaintnerClient
	// LUCI is a client for LUCI, used for fetching build results from there.
	LUCI luciClient

	// memoryResults is an in-memory storage of CI results. Used in development and testing for datastore data.
	memoryResults map[string][]string
}

func (d *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	var showLUCI = true
	if legacyOnly, _ := strconv.ParseBool(req.URL.Query().Get("legacyonly")); legacyOnly {
		showLUCI = false
	}

	var luci lucipoll.Snapshot
	if d.LUCI != nil && showLUCI {
		luci = d.LUCI.PostSubmitSnapshot()
	}

	dd := &data{
		Builders: d.getBuilders(dashboard.Builders, luci),
		Commits:  d.commits(req.Context(), luci),
		Package:  dashPackage{Name: "Go"},
	}

	var buf bytes.Buffer
	if err := templ.Execute(&buf, dd); err != nil {
		log.Printf("handleDashboard: error rendering template: %v", err)
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}
	buf.WriteTo(w)
}

func (d *Handler) commits(ctx context.Context, luci lucipoll.Snapshot) []*commit {
	resp, err := d.Maintner.GetDashboard(ctx, &apipb.DashboardRequest{})
	if err != nil {
		log.Printf("handleDashboard: error fetching from maintner: %v", err)
		return nil
	}
	var commits []*commit
	for _, c := range resp.GetCommits() {
		commits = append(commits, &commit{
			Desc: c.Title,
			Hash: c.Commit,
			Time: time.Unix(c.CommitTimeSec, 0).Format("02 Jan 15:04"),
			User: formatGitAuthor(c.AuthorName, c.AuthorEmail),
		})
	}
	d.getResults(ctx, commits, luci)
	return commits
}

// getResults populates result data on commits, fetched from Datastore or in-memory storage
// and, if luci is non-zero, also from LUCI.
func (d *Handler) getResults(ctx context.Context, commits []*commit, luci lucipoll.Snapshot) {
	if d.Datastore != nil {
		getDatastoreResults(ctx, d.Datastore, commits, "go")
	} else {
		for _, c := range commits {
			if result, ok := d.memoryResults[c.Hash]; ok {
				c.ResultData = result
			}
		}
	}
	appendLUCIResults(luci, commits, "go")
}

func (d *Handler) getBuilders(conf map[string]*dashboard.BuildConfig, luci lucipoll.Snapshot) []*builder {
	bm := make(map[string]builder)
	for _, b := range conf {
		if !b.BuildsRepoPostSubmit("go", "master", "master") {
			continue
		}
		if dashboard.BuildersPortedToLUCI[b.Name] && len(luci.Builders) > 0 {
			// Don't display old builders that have been ported
			// to LUCI if willing to show LUCI builders as well.
			continue
		}
		db := bm[b.GOOS()]
		db.OS = b.GOOS()
		db.Archs = append(db.Archs, &arch{
			os: b.GOOS(), Arch: b.GOARCH(),
			Name: b.Name,
			// Tag is the part after "os-arch", if any, without leading dash.
			Tag: strings.TrimPrefix(strings.TrimPrefix(b.Name, fmt.Sprintf("%s-%s", b.GOOS(), b.GOARCH())), "-"),
		})
		bm[b.GOOS()] = db
	}

	for _, b := range luci.Builders {
		if b.Repo != "go" || b.GoBranch != "master" {
			continue
		}
		db := bm[b.Target.GOOS]
		db.OS = b.Target.GOOS
		tagFriendly := b.Name + "-🐇"
		if after, ok := strings.CutPrefix(tagFriendly, fmt.Sprintf("gotip-%s-%s_", b.Target.GOOS, b.Target.GOARCH)); ok {
			// Convert os-arch_osversion-mod1-mod2 (an underscore at start of "_osversion")
			// to have os-arch-osversion-mod1-mod2 (a dash at start of "-osversion") form.
			// The tag computation below uses this to find both "osversion-mod1" or "mod1".
			tagFriendly = fmt.Sprintf("gotip-%s-%s-", b.Target.GOOS, b.Target.GOARCH) + after
		}
		db.Archs = append(db.Archs, &arch{
			os: b.Target.GOOS, Arch: b.Target.GOARCH,
			Name: b.Name,
			// Tag is the part after "os-arch", if any, without leading dash.
			Tag: strings.TrimPrefix(strings.TrimPrefix(tagFriendly, fmt.Sprintf("gotip-%s-%s", b.Target.GOOS, b.Target.GOARCH)), "-"),
		})
		bm[b.Target.GOOS] = db
	}

	var builders builderSlice
	for _, db := range bm {
		db := db
		sort.Sort(&db.Archs)
		builders = append(builders, &db)
	}
	sort.Sort(builders)
	return builders
}

type arch struct {
	os, Arch string
	Name     string
	Tag      string
}

func (a arch) FirstClass() bool { return releasetargets.IsFirstClass(a.os, a.Arch) }

type archSlice []*arch

func (d archSlice) Len() int {
	return len(d)
}

// Less sorts first-class ports first, then it sorts by name.
func (d archSlice) Less(i, j int) bool {
	iFirst, jFirst := d[i].FirstClass(), d[j].FirstClass()
	if iFirst && !jFirst {
		return true
	}
	if !iFirst && jFirst {
		return false
	}
	return d[i].Name < d[j].Name
}

func (d archSlice) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

type builder struct {
	Active      bool
	Archs       archSlice
	OS          string
	Unsupported bool
}

func (b *builder) FirstClass() bool {
	for _, a := range b.Archs {
		if a.FirstClass() {
			return true
		}
	}
	return false
}

func (b *builder) FirstClassArchs() archSlice {
	var as archSlice
	for _, a := range b.Archs {
		if a.FirstClass() {
			as = append(as, a)
		}
	}
	return as
}

type builderSlice []*builder

func (d builderSlice) Len() int {
	return len(d)
}

// Less sorts first-class ports first, then it sorts by name.
func (d builderSlice) Less(i, j int) bool {
	iFirst, jFirst := d[i].FirstClass(), d[j].FirstClass()
	if iFirst && !jFirst {
		return true
	}
	if !iFirst && jFirst {
		return false
	}
	return d[i].OS < d[j].OS
}

func (d builderSlice) Swap(i, j int) {
	d[i], d[j] = d[j], d[i]
}

type dashPackage struct {
	Name string
	Path string
}

type commit struct {
	Desc string
	Hash string
	// ResultData is a copy of the [Commit.ResultData] field from datastore,
	// with an additional rule that the second '|'-separated value may be "infra_failure"
	// to indicate a problem with the infrastructure rather than the code being tested.
	//
	// It can also have the form of "builder|BuildingURL" for in progress builds.
	ResultData []string
	Time       string
	User       string
}

// ShortUser returns a shortened version of a user string.
func (c *commit) ShortUser() string {
	user := c.User
	if i, j := strings.Index(user, "<"), strings.Index(user, ">"); 0 <= i && i < j {
		user = user[i+1 : j]
	}
	if i := strings.Index(user, "@"); i >= 0 {
		return user[:i]
	}
	return user
}

func (c *commit) ResultForBuilder(builder string) *result {
	for _, rd := range c.ResultData {
		segs := strings.Split(rd, "|")
		if len(segs) == 2 && segs[0] == builder {
			return &result{
				BuildingURL: segs[1],
			}
		}
		if len(segs) < 4 {
			continue
		}
		if segs[0] == builder {
			return &result{
				OK:      segs[1] == "true",
				Noise:   segs[1] == "infra_failure",
				LogHash: segs[2],
			}
		}
	}
	return nil
}

type result struct {
	BuildingURL string
	OK          bool
	Noise       bool
	LogHash     string
}

func (r result) LogURL() string {
	if strings.HasPrefix(r.LogHash, "https://") {
		return r.LogHash
	} else {
		return "https://build.golang.org/log/" + r.LogHash
	}
}

// formatGitAuthor formats the git author name and email (as split by
// maintner) back into the unified string how they're stored in a git
// commit, so the shortUser func (used by the HTML template) can parse
// back out the email part's username later. Maybe we could plumb down
// the parsed proto into the template later.
func formatGitAuthor(name, email string) string {
	name = strings.TrimSpace(name)
	email = strings.TrimSpace(email)
	if name != "" && email != "" {
		return fmt.Sprintf("%s <%s>", name, email)
	}
	if name != "" {
		return name
	}
	return "<" + email + ">"
}

//go:embed dashboard.html
var dashboardTemplate string

var templ = template.Must(
	template.New("dashboard.html").Funcs(template.FuncMap{
		"shortHash": shortHash,
	}).Parse(dashboardTemplate),
)

// shortHash returns a short version of a hash.
func shortHash(hash string) string {
	if len(hash) > 7 {
		hash = hash[:7]
	}
	return hash
}

// A Commit describes an individual commit in a package.
//
// Each Commit entity is a descendant of its associated Package entity.
// In other words, all Commits with the same PackagePath belong to the same
// datastore entity group.
type Commit struct {
	PackagePath string // (empty for main repo commits)
	Hash        string

	// ResultData is the Data string of each build Result for this Commit.
	// For non-Go commits, only the Results for the current Go tip, weekly,
	// and release Tags are stored here. This is purely de-normalized data.
	// The complete data set is stored in Result entities.
	//
	// Each string is formatted as builder|OK|LogHash|GoHash.
	ResultData []string `datastore:",noindex"`
}