golang.org/x/build@v0.0.0-20240506185731-218518f32b70/internal/dash/dash.go

// Copyright 2022 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package dash reads build.golang.org's dashboards.
package dash

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"sort"
	"sync"
	"time"
)

// A Board is a single dashboard.
type Board struct {
	Repo      string   // repo being displayed: "go", "arch", and so on
	Branch    string   // branch in repo
	Builders  []string // builder columns
	Revisions []*Line  // commit lines, newest to oldest
}

// A Line is a single commit line on a Board b.
type Line struct {
	Repo       string    // same as b.Repo
	Branch     string    // same as b.Branch
	Revision   string    // revision of Repo
	GoRevision string    // for Repo != "go", revision of go repo being used
	GoBranch   string    // for Repo != "go", branch of go repo being used
	Date       time.Time // date of commit
	Author     string    // author of commit
	Desc       string    // commit description

	// // Results[i] reports b.Builders[i]'s result:
	// "" (not run), "ok" (passed), or the URL of the failure log
	// ("https://build.golang.org/log/...")
	Results []string
}

// Read reads and returns all the dashboards on build.golang.org
// (for the main repo, the main repo release branches, and subrepos),
// including all results up to the given time limit.
// It guarantees that all the returned boards will have the same b.Builders slices,
// so that any line.Results[i] even for different boards refers to a consistent
// builder for a given i.
func Read(limit time.Time) ([]*Board, error) {
	return Update(nil, limit)
}

// Update is like Read but takes a starting set of boards from
// a previous call to Read or Update and avoids redownloading
// information from those boards.
// It does not modify the boards passed in as input.
func Update(old []*Board, limit time.Time) ([]*Board, error) {
	// Read the front page to derive the Go repo branches and subrepos.
	_, goBranches, repos, err := readPage("", "", 0)
	if err != nil {
		return nil, err
	}
	repos = append([]string{"go"}, repos...)

	// Build cache of existing boards.
	type key struct{ repo, branch string }
	cache := make(map[key]*Board)
	for _, b := range old {
		cache[key{b.Repo, b.Branch}] = b
	}

	// For each repo and branch, fetch that repo's list of board pages.
	var boards []*Board
	var errors []error
	var wg sync.WaitGroup
	for _, r := range repos {
		r := r
		branches := []string{""}
		if r == "go" {
			branches = goBranches
		}
		for _, branch := range branches {
			branch := branch
			if branch == "master" || branch == "main" {
				branch = ""
			}
			// Only read up to what we already have in old, respecting limit.
			old := cache[key{r, branch}]
			oldLimit := limit
			if old != nil && len(old.Revisions) > 0 && old.Revisions[0].Date.After(limit) {
				oldLimit = old.Revisions[0].Date
			}
			i := len(boards)
			boards = append(boards, nil)
			errors = append(errors, nil)
			wg.Add(1)
			go func() {
				defer wg.Done()
				boards[i], errors[i] = readRepo(r, branch, oldLimit)
				if errors[i] == nil {
					boards[i] = update(boards[i], old, limit)
				}
			}()
		}
	}
	wg.Wait()

	for _, err := range errors {
		if err != nil {
			return nil, err
		}
	}

	// Remap all the boards to have a consistent Builders array.
	// It is slightly inefficient that readRepo does this remap as well,
	// but all the downloads take more time.
	remap(boards)

	return boards, nil
}

// update returns the result of merging b and old,
// discarding revisions older than limit and removing duplicates.
// It modifies b but not old.
func update(b, old *Board, limit time.Time) *Board {
	if old == nil || !same(b.Builders, old.Builders) {
		if old == nil {
			old = new(Board)
		} else {
			old = old.clone()
		}
		remap([]*Board{b, old})
	}

	type key struct {
		rev   string
		gorev string
	}
	have := make(map[key]bool)
	keep := b.Revisions[:0]
	for _, list := range [][]*Line{b.Revisions, old.Revisions} {
		for _, r := range list {
			if !r.Date.Before(limit) && !have[key{r.Revision, r.GoRevision}] {
				have[key{r.Revision, r.GoRevision}] = true
				keep = append(keep, r)
			}
		}
	}
	b.Revisions = keep
	return b
}

// clone returns a deep copy of b.
func (b *Board) clone() *Board {
	b1 := &Board{
		Repo:      b.Repo,
		Branch:    b.Branch,
		Builders:  make([]string, len(b.Builders)),
		Revisions: make([]*Line, len(b.Revisions)),
	}
	copy(b1.Builders, b.Builders)
	for i := range b1.Revisions {
		r := new(Line)
		*r = *b.Revisions[i]
		results := make([]string, len(r.Results))
		copy(results, r.Results)
		r.Results = results
		b1.Revisions[i] = r
	}
	return b1
}

// readRepo reads and returns the pages for the given repo and branch,
// stopping when it finds a page that contains no results newer than limit.
func readRepo(repo, branch string, limit time.Time) (*Board, error) {
	path := ""
	if repo != "go" {
		path = "golang.org/x/" + repo
	}
	var pages []*Board
	for page := 0; ; page++ {
		b, _, _, err := readPage(path, branch, page)
		if err != nil {
			return merge(pages), err
		}

		// If there's nothing new enough on the whole page, stop.
		keep := b.Revisions[:0]
		for _, r := range b.Revisions {
			if !r.Date.Before(limit) {
				keep = append(keep, r)
			}
		}
		if len(keep) == 0 {
			break
		}
		b.Revisions = keep
		b.Repo = repo
		b.Branch = branch
		pages = append(pages, b)
	}
	return merge(pages), nil
}

// merge merges all the pages into a single board.
func merge(pages []*Board) *Board {
	if len(pages) == 0 {
		return new(Board)
	}

	remap(pages)
	for _, b := range pages {
		if !same(b.Builders, pages[0].Builders) || b.Repo != pages[0].Repo || b.Branch != pages[0].Branch {
			panic("misuse of merge")
		}
	}

	merged := &Board{Repo: pages[0].Repo, Branch: pages[0].Branch, Builders: pages[0].Builders}
	for _, b := range pages {
		merged.Revisions = append(merged.Revisions, b.Revisions...)
	}
	return merged
}

// remap remaps all the results in all the boards
// to use a consistent set of Builders.
func remap(boards []*Board) {
	// Collect list of all builders across all boards.
	var builders []string
	index := make(map[string]int)
	for _, b := range boards {
		for _, builder := range b.Builders {
			if index[builder] == 0 {
				index[builder] = 1
				builders = append(builders, builder)
			}
		}
	}
	sort.Strings(builders)
	for i, builder := range builders {
		index[builder] = i
	}

	// Remap.
	for _, b := range boards {
		for _, r := range b.Revisions {
			results := make([]string, len(builders))
			for i, ok := range r.Results {
				results[index[b.Builders[i]]] = ok
			}
			r.Results = results
		}
		b.Builders = builders
	}
}

// readPage reads the build.golang.org page for repo, branch.
// It returns the board on that page.
// When repo == "go" and branch == "" and page == 0,
// build.golang.org also sends back information about the
// other go repo branches and the subrepos.
// readPage("go", "", 0) returns those lists of go branches
// and subrepos as extra results.
func readPage(repo, branch string, page int) (b *Board, branches, repos []string, err error) {
	if repo == "" {
		repo = "go"
	}
	u := "https://build.golang.org/?mode=json&repo=" + url.QueryEscape(repo) + "&branch=" + url.QueryEscape(branch) + "&page=" + fmt.Sprint(page)
	log.Printf("read %v", u)
	resp, err := http.Get(u)
	if err != nil {
		return nil, nil, nil, fmt.Errorf("%s page %d: %v", repo, page, err)
	}
	data, err := io.ReadAll(resp.Body)
	resp.Body.Close()
	if err != nil {
		return nil, nil, nil, fmt.Errorf("%s page %d: %v", repo, page, err)
	}
	if resp.StatusCode != 200 {
		return nil, nil, nil, fmt.Errorf("%s page %d: %s\n%s", repo, page, resp.Status, data)
	}

	b = new(Board)
	if err := json.Unmarshal(data, b); err != nil {
		return nil, nil, nil, fmt.Errorf("%s page %d: %v", repo, page, err)
	}

	// Use empty string consistently to denote master/main branch.
	for _, r := range b.Revisions {
		if r.Branch == "master" || r.Branch == "main" {
			r.Branch = ""
		}
		if r.GoBranch == "master" || r.GoBranch == "main" {
			r.GoBranch = ""
		}
	}

	// https://build.golang.org/?mode=json (main repo, no branch, page 0)
	// sends back a bit about the subrepos too. Filter that out.
	if repo == "go" {
		var save []*Line
		for _, r := range b.Revisions {
			if r.Repo == "go" {
				save = append(save, r)
			} else {
				branches = append(branches, r.GoBranch)
				repos = append(repos, r.Repo)
			}
		}
		b.Revisions = save
		branches = uniq(branches)
		repos = uniq(repos)
	}

	return b, branches, repos, nil
}

// same reports whether x and y are the same slice.
func same(x, y []string) bool {
	if len(x) != len(y) {
		return false
	}
	for i, s := range x {
		if y[i] != s {
			return false
		}
	}
	return true
}

// uniq sorts and removes duplicates from list, returning the result.
// uniq reuses list's storage for its result.
func uniq(list []string) []string {
	sort.Strings(list)
	keep := list[:0]
	for _, s := range list {
		if len(keep) == 0 || s != keep[len(keep)-1] {
			keep = append(keep, s)
		}
	}
	return keep
}