go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/luci_notify/notify/tree_status.go

// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package notify

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"sync"
	"time"

	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/data/stringset"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/lhttp"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry/transient"
	"go.chromium.org/luci/common/sync/parallel"
	"go.chromium.org/luci/gae/service/datastore"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/server/auth"
	tspb "go.chromium.org/luci/tree_status/proto/v1"

	"go.chromium.org/luci/luci_notify/config"
)

const botUsername = "luci-notify@appspot.gserviceaccount.com"
const legacyBotUsername = "buildbot@chromium.org"

type treeStatus struct {
	username  string
	message   string
	key       int64
	status    config.TreeCloserStatus
	timestamp time.Time
}

type treeStatusClient interface {
	getStatus(c context.Context, host string) (*treeStatus, error)
	postStatus(c context.Context, host, message string, prevKey int64, treeName string, status config.TreeCloserStatus) error
}

type httpTreeStatusClient struct {
	getFunc  func(context.Context, string) ([]byte, error)
	postFunc func(context.Context, string) error
	client   tspb.TreeStatusClient
}

func NewHTTPTreeStatusClient(ctx context.Context, luciTreeStatusHost string) (*httpTreeStatusClient, error) {
	transport, err := auth.GetRPCTransport(ctx, auth.AsSelf)
	if err != nil {
		return nil, err
	}
	rpcOpts := prpc.DefaultOptions()
	rpcOpts.Insecure = lhttp.IsLocalHost(luciTreeStatusHost)
	prpcClient := &prpc.Client{
		C:                     &http.Client{Transport: transport},
		Host:                  luciTreeStatusHost,
		Options:               rpcOpts,
		MaxConcurrentRequests: 100,
	}

	return &httpTreeStatusClient{
		getFunc:  getHttp,
		postFunc: postHttp,
		client:   tspb.NewTreeStatusPRPCClient(prpcClient),
	}, nil
}

func (ts *httpTreeStatusClient) getStatus(c context.Context, host string) (*treeStatus, error) {
	// TODO(mwarton): transition to the new tree status app RPC after the migration.
	respJSON, err := ts.getFunc(c, fmt.Sprintf("https://%s/current?format=json", host))
	if err != nil {
		return nil, err
	}

	var r struct {
		Username        string
		CanCommitFreely bool `json:"can_commit_freely"`
		Key             int64
		Date            string
		Message         string
	}
	if err = json.Unmarshal(respJSON, &r); err != nil {
		return nil, errors.Annotate(err, "failed to unmarshal JSON").Err()
	}

	var status config.TreeCloserStatus = config.Closed
	if r.CanCommitFreely {
		status = config.Open
	}

	// Similar to RFC3339, but not quite the same. No time zone is specified,
	// so this will default to UTC, which is correct here.
	const dateFormat = "2006-01-02 15:04:05.999999"
	t, err := time.Parse(dateFormat, r.Date)
	if err != nil {
		return nil, errors.Annotate(err, "failed to parse date from tree status").Err()
	}

	return &treeStatus{
		username:  r.Username,
		message:   r.Message,
		key:       r.Key,
		status:    status,
		timestamp: t,
	}, nil
}

func (ts *httpTreeStatusClient) postStatus(ctx context.Context, host, message string, prevKey int64, treeName string, status config.TreeCloserStatus) error {
	// During the tree status migration, we will update both the old (HTTP) status
	// and the new (PRPC) status.  We always attempt to update both despite whatever
	// errors may occur.
	// TODO(mwarton): Remove the HTTP post after the migration.
	logging.Infof(ctx, "Updating status for %s: %q", host, message)

	q := url.Values{}
	q.Add("message", message)
	q.Add("last_status_key", strconv.FormatInt(prevKey, 10))
	u := url.URL{
		Host:     host,
		Scheme:   "https",
		Path:     "/",
		RawQuery: q.Encode(),
	}

	httpErr := ts.postFunc(ctx, u.String())

	generalState := tspb.GeneralState_OPEN
	if status == config.Closed {
		generalState = tspb.GeneralState_CLOSED
	}
	request := &tspb.CreateStatusRequest{
		Parent: fmt.Sprintf("trees/%s/status", treeName),
		Status: &tspb.Status{
			GeneralState: generalState,
			Message:      message,
		},
	}
	_, prpcErr := ts.client.CreateStatus(ctx, request)

	// If the PRPC worked, we don't really mind what the status of the HTTP request was, as
	// we expect it to start failing during the migration.
	if prpcErr == nil {
		if httpErr != nil {
			logging.Infof(ctx, "Error updating status by HTTP: %s", httpErr)
		}
		return nil
	}
	// Log any PRPC errors, but allow HTTP success to override us during the migration.
	logging.Errorf(ctx, "Error updating status by PRPC: %s", prpcErr)
	return httpErr

}

func getHttp(c context.Context, url string) ([]byte, error) {
	response, err := makeHttpRequest(c, url, "GET")
	if err != nil {
		return nil, err
	}

	defer response.Body.Close()
	bytes, err := io.ReadAll(response.Body)
	if err != nil {
		return nil, errors.Annotate(err, "failed to read response body from %q", url).Err()
	}

	return bytes, nil
}

func postHttp(c context.Context, url string) error {
	response, err := makeHttpRequest(c, url, "POST")
	if err != nil {
		return err
	}

	response.Body.Close()

	// If the operation succeeded, the status app will apply the update, and
	// then redirect back to the main page. Let's also check for a 200, as this
	// is a reasonable response and we don't want to depend too heavily on
	// particular implementation details.
	if response.StatusCode == http.StatusFound || response.StatusCode == http.StatusOK {
		return nil
	}
	return fmt.Errorf("POST to %q returned unexpected status code %d", url, response.StatusCode)
}

func makeHttpRequest(c context.Context, url, method string) (*http.Response, error) {
	transport, err := auth.GetRPCTransport(c, auth.AsSelf)
	if err != nil {
		return nil, err
	}

	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return nil, err
	}
	req = req.WithContext(c)

	response, err := (&http.Client{Transport: transport}).Do(req)
	if err != nil {
		return nil, errors.Annotate(err, "%s request to %q failed", method, url).Err()
	}

	return response, nil
}

// UpdateTreeStatus is the HTTP handler triggered by cron when it's time to
// check tree closers and update tree status if necessary.
func UpdateTreeStatus(ctx context.Context) error {
	ctx, cancel := context.WithTimeout(ctx, time.Minute)
	defer cancel()

	settings, err := config.FetchSettings(ctx)
	if err != nil {
		return errors.Annotate(err, "fetching settings").Err()
	}
	client, err := NewHTTPTreeStatusClient(ctx, settings.LuciTreeStatusHost)
	if err != nil {
		return errors.Annotate(err, "creating tree status client").Err()
	}

	return transient.Tag.Apply(updateTrees(ctx, client))
}

// updateTrees fetches all TreeClosers from datastore, uses this to determine if
// any trees should be opened or closed, and makes the necessary updates.
func updateTrees(c context.Context, ts treeStatusClient) error {
	// The goal here is, for every project, to atomically fetch the config
	// for that project along with all TreeClosers within it. So if the
	// project config and the set of TreeClosers are updated at the same
	// time, we should always see either both updates, or neither. Also, we
	// want to do it without XG transactions.
	//
	// First we fetch keys for all the projects. Second, for every project,
	// we fetch the full config and all TreeClosers in a transaction. Since
	// these two steps aren't within a transaction, it's possible that
	// changes have occurred in between. But all cases are dealt with:
	//
	// * Updates to project config or TreeClosers aren't a problem since we
	//   only fetch them in the second step anyway.
	// * Deletions of projects are fine, since if we don't find them in the
	//   second fetch we just ignore that project and carry on.
	// * New projects are ignored, and picked up the next time we run.
	q := datastore.NewQuery("Project").KeysOnly(true)
	var projects []*config.Project
	if err := datastore.GetAll(c, q, &projects); err != nil {
		return errors.Annotate(err, "failed to get project keys").Err()
	}

	// Guards access to both treeClosers and closingEnabledProjects.
	mu := sync.Mutex{}
	var treeClosers []*config.TreeCloser
	closingEnabledProjects := stringset.New(0)

	err := parallel.WorkPool(32, func(ch chan<- func() error) {
		for _, project := range projects {
			project := project
			ch <- func() error {
				return datastore.RunInTransaction(c, func(c context.Context) error {
					switch err := datastore.Get(c, project); {
					// The project was deleted since the previous time we fetched it just above.
					// In this case, just move on, since the project is no more.
					case err == datastore.ErrNoSuchEntity:
						logging.Infof(c, "Project %s removed between queries, ignoring it", project.Name)
						return nil
					case err != nil:
						return errors.Annotate(err, "failed to get project").Tag(transient.Tag).Err()
					}

					q := datastore.NewQuery("TreeCloser").Ancestor(datastore.KeyForObj(c, project))
					var treeClosersForProject []*config.TreeCloser
					if err := datastore.GetAll(c, q, &treeClosersForProject); err != nil {
						return errors.Annotate(err, "failed to get tree closers").Tag(transient.Tag).Err()
					}

					mu.Lock()
					defer mu.Unlock()
					logging.Debugf(c, "Appending tree closers for project: %v", project)
					treeClosers = append(treeClosers, treeClosersForProject...)
					if project.TreeClosingEnabled {
						closingEnabledProjects.Add(project.Name)
					}

					return nil
				}, nil)
			}
		}
	})
	if err != nil {
		return err
	}

	logging.Debugf(c, "closingEnabledProjects: %v", closingEnabledProjects)
	return parallel.WorkPool(32, func(ch chan<- func() error) {
		for host, treeClosers := range groupTreeClosers(treeClosers) {
			host, treeClosers := host, treeClosers
			ch <- func() error {
				c := logging.SetField(c, "tree-status-host", host)
				return updateHost(c, ts, host, treeClosers, closingEnabledProjects, treeNameOrDefault(treeClosers))
			}
		}
	})
}

func groupTreeClosers(treeClosers []*config.TreeCloser) map[string][]*config.TreeCloser {
	byHost := map[string][]*config.TreeCloser{}
	for _, tc := range treeClosers {
		byHost[tc.TreeStatusHost] = append(byHost[tc.TreeStatusHost], tc)
	}

	return byHost
}

func treeNameOrDefault(treeClosers []*config.TreeCloser) string {
	for _, closer := range treeClosers {
		if closer.TreeCloser.TreeName != "" {
			return closer.TreeCloser.TreeName
		}
	}
	for _, closer := range treeClosers {
		if closer.TreeStatusHost != "" {
			return strings.TrimSuffix(strings.TrimSuffix(closer.TreeStatusHost, ".appspot.com"), "-status")
		}
		if closer.TreeCloser.TreeStatusHost != "" {
			return strings.TrimSuffix(strings.TrimSuffix(closer.TreeCloser.TreeStatusHost, ".appspot.com"), "-status")
		}
	}
	panic("Should not have gotten here, invalid project configuration contains neither host nor tree name")
}

func tcProject(tc *config.TreeCloser) string {
	return tc.BuilderKey.Parent().StringID()
}

func updateHost(c context.Context, ts treeStatusClient, host string, treeClosers []*config.TreeCloser, closingEnabledProjects stringset.Set, treeName string) error {
	treeStatus, err := ts.getStatus(c, host)
	if err != nil {
		return err
	}

	if treeStatus.status == config.Closed && treeStatus.username != botUsername && treeStatus.username != legacyBotUsername {
		// Don't do anything if the tree was manually closed.
		logging.Debugf(c, "Tree is closed and last update was from non-bot user %s; not doing anything", treeStatus.username)
		return nil
	}

	logging.Debugf(c, "Scanning treeClosers for any belonging to a project with tree closing enabled: %v", treeClosers)
	anyEnabled := false
	for _, tc := range treeClosers {
		if closingEnabledProjects.Has(tcProject(tc)) {
			logging.Debugf(c, "Found such a treeCloser: %v", tc)
			anyEnabled = true
			break
		}
	}
	logging.Debugf(c, "anyEnabled = %v", anyEnabled)

	anyFailingBuild := false
	anyNewBuild := false
	var oldestClosed *config.TreeCloser
	for _, tc := range treeClosers {
		// If any TreeClosers are from projects with tree closing enabled,
		// ignore any TreeClosers *not* from such projects. In general we don't
		// expect different projects to close the same tree, so we're okay with
		// not seeing dry run logging for these TreeClosers in this rare case.
		if anyEnabled && !closingEnabledProjects.Has(tcProject(tc)) {
			continue
		}

		// For opening the tree, we need to make sure *all* builders are
		// passing, not just those that have had new builds. Otherwise we'll
		// open the tree after any new green build, even if the builder that
		// caused us to close it is still failing.
		if tc.Status == config.Closed {
			logging.Debugf(c, "Found failing builder with message: %s", tc.Message)
			anyFailingBuild = true
		}

		// Only pay attention to failing builds from after the last update to
		// the tree. Otherwise we'll close the tree even after people manually
		// open it.
		if tc.Timestamp.Before(treeStatus.timestamp) {
			continue
		}

		anyNewBuild = true

		if tc.Status == config.Closed && (oldestClosed == nil || tc.Timestamp.Before(oldestClosed.Timestamp)) {
			logging.Debugf(c, "Updating oldest failing builder")
			oldestClosed = tc
		}
	}

	var newStatus config.TreeCloserStatus
	if !anyNewBuild {
		// Don't do anything if all the builds are older than the last update
		// to the tree - nothing has changed, so there's no reason to take any
		// action.
		logging.Debugf(c, "No builds newer than last tree update (%s); not doing anything",
			treeStatus.timestamp.Format(time.RFC1123Z))
		return nil
	}
	if !anyFailingBuild {
		// We can open the tree, as no builders are failing, including builders
		// that haven't run since the last update to the tree.
		logging.Debugf(c, "No failing builders; new status is Open")
		newStatus = config.Open
	} else if oldestClosed != nil {
		// We can close the tree, as at least one builder has failed since the
		// last update to the tree.
		logging.Debugf(c, "At least one failing builder; new status is Closed")
		newStatus = config.Closed
	} else {
		// Some builders are failing, but they were already failing before the
		// last update. Don't do anything, so as not to close the tree after a
		// sheriff has manually opened it.
		logging.Debugf(c, "At least one failing builder, but there's a more recent update; not doing anything")
		return nil
	}

	if treeStatus.status == newStatus {
		// Don't do anything if the current status is already correct.
		logging.Debugf(c, "Current status is already correct; not doing anything")
		return nil
	}

	var message string
	if newStatus == config.Open {
		message = fmt.Sprintf("Tree is open (Automatic: %s)", randomMessage(c))
	} else {
		message = fmt.Sprintf("Tree is closed (Automatic: %s)", oldestClosed.Message)
	}

	if anyEnabled {
		return ts.postStatus(c, host, message, treeStatus.key, treeName, newStatus)
	}
	logging.Infof(c, "Would update status for %s to %q", host, message)
	return nil
}

// Want more messages? CLs welcome!
var messages = []string{
	"('o')",
	"(｡>﹏<｡)",
	"☃",
	"☀ Tree is open ☀",
	"٩◔̯◔۶",
	"☺",
	"(´・ω・`)",
	"(｀・ω・´)",
	"(΄◞ิ౪◟ิ‵ )",
	"(╹◡╹)",
	"♩‿♩",
	"(/･ω･)/",
	" ʅ(◔౪◔ ) ʃ",
	"ᕙ(`▿´)ᕗ",
	"ヽ(^o^)丿",
	"\\(･ω･)/",
	"＼(^o^)／",
	"ｷﾀ━━━━(ﾟ∀ﾟ)━━━━ｯ!!",
	"ヽ(^。^)ノ",
	"(ﾟдﾟ)",
	"ヽ(´ω`*人*´ω`)ノ",
	" ﾟ+｡:.ﾟヽ(*´∀`)ﾉﾟ.:｡+ﾟ",
	"(゜ー゜＊）ネッ！",
	" ♪d(´▽｀)b♪オールオッケィ♪",
	"(ﾉ≧∀≦)ﾉ・‥…",
	"☆（ゝω・）vｷｬﾋﾟ",
	"ლ(╹◡╹ლ)",
	"ƪ(•̃͡ε•̃͡)∫ʃ",
	"(•_•)",
	"( ་ ⍸ ་ )",
	"(☉౪ ⊙)",
	"˙ ͜ʟ˙",
	"( ఠൠఠ )",
	"☆.｡.:*･ﾟ☆.｡.:*･ﾟ☆祝☆ﾟ･*:.｡.☆ﾟ･*:.｡.☆",
	"༼ꉺɷꉺ༽",
	"◉_◉",
	"ϵ( ‘Θ’ )϶",
	"ヾ(⌐■_■)ノ♪",
	"(◡‿◡✿)",
	"★.:ﾟ+｡☆ (●´v｀○)bｫﾒﾃﾞﾄd(○´v｀●)☆.:ﾟ+｡★",
	"(☆.☆)",
	"ｵﾒﾃﾞﾄｰ♪c(*ﾟｰ^)ﾉ*･'ﾟ☆｡.:*:･'☆'･:*:.",
	"☆.。.:*・°☆.。.:*・°☆",
	"ʕ •ᴥ•ʔ",
	"☼.☼",
	"⊂(・(ェ)・)⊃",
	"(ﾉ≧∇≦)ﾉ ﾐ ┸━┸",
	"¯\\_(ツ)_/¯",
	"UwU",
	"Paç fat!",
	"Sretno",
	"Hodně štěstí!",
	"Held og lykke!",
	"Veel geluk!",
	"Edu!",
	"lykkyä tykö",
	"Viel Glück!",
	"Καλή τύχη!",
	"Sok szerencsét kivánok!",
	"Gangi þér vel!",
	"Go n-éirí an t-ádh leat!",
	"Buona fortuna!",
	"Laimīgs gadījums!",
	"Sėkmės!",
	"Vill Gléck!",
	"Со среќа!",
	"Powodzenia!",
	"Boa sorte!",
	"Noroc!",
	"Срећно",
	"Veľa šťastia!",
	"Lycka till!",
	"Bona sort!",
	"Zorte on!",
	"Góða eydnu",
	"¡Boa fortuna!",
	"Bona fortuna!",
	"Xewqat sbieħ",
	"Aigh vie!",
	"Pob lwc!",
	" موفق باشيد",
	"İyi şanslar!",
	"Bonŝancon!",
	"祝你好运！",
	"祝你好運！",
	"頑張って！",
	"សំណាងល្អ ",
	"행운을 빌어요",
	"शुभ कामना ",
	"โชคดี!",
	"Chúc may mắn!",
	"بالتوفيق!",
	"Sterkte!",
	"Ke o lakaletsa mohlohonolo",
	"Uve nemhanza yakanaka",
	"Kila la kheri!",
	"Amathamsanqa",
	"Ngikufisela iwela!",
	"Bonne chance!",
	"¡Buena suerte!",
	"Good luck!",
	"Semoga Beruntung!",
	"Selamat Maju Jaya!",
	"Ia manuia",
	"Suwertehin ka sana",
	"Հաջողությո'ւն",
	"Іске сәт",
	"Амжилт хүсье",
	"удачі!",
	"Da legst di nieda!",
	"Gell, da schaugst?",
	"Ois Guade",
	"शुभ कामना!",
	"நல் வாழ்த்துக்கள் ",
	"అంతా శుభం కలగాలి! ",
	":')",
	":'D",
	"`,;)",
	"Tree is open (^O^)",
	"Thượng lộ bình an",
	"Tree is open now (ง '̀͜ '́ )ง",
	"ヽ(^o^)ノ",
	"Ahoy all is good!",
	"All's right with the world!",
	"Aloha",
}

func randomMessage(c context.Context) string {
	message := messages[mathrand.Intn(c, len(messages))]
	if message[len(message)-1] == ')' {
		return message + " "
	}
	return message
}