go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/luci_notify/notify/tree_status_test.go (about)

     1  // Copyright 2020 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package notify
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"strings"
    21  	"sync"
    22  	"testing"
    23  	"time"
    24  
    25  	grpc "google.golang.org/grpc"
    26  
    27  	"go.chromium.org/luci/common/errors"
    28  	"go.chromium.org/luci/common/logging"
    29  	"go.chromium.org/luci/common/logging/memlogger"
    30  	"go.chromium.org/luci/gae/impl/memory"
    31  	"go.chromium.org/luci/gae/service/datastore"
    32  	tspb "go.chromium.org/luci/tree_status/proto/v1"
    33  
    34  	notifypb "go.chromium.org/luci/luci_notify/api/config"
    35  	"go.chromium.org/luci/luci_notify/common"
    36  	"go.chromium.org/luci/luci_notify/config"
    37  
    38  	. "github.com/smartystreets/goconvey/convey"
    39  )
    40  
    41  // fakeTreeStatusClient simulates the behaviour of a real tree status instance,
    42  // but locally, in-memory.
    43  type fakeTreeStatusClient struct {
    44  	statusForHosts map[string]treeStatus
    45  	nextKey        int64
    46  	mu             sync.Mutex
    47  }
    48  
    49  func (ts *fakeTreeStatusClient) getStatus(c context.Context, host string) (*treeStatus, error) {
    50  	ts.mu.Lock()
    51  	defer ts.mu.Unlock()
    52  
    53  	status, exists := ts.statusForHosts[host]
    54  	if exists {
    55  		return &status, nil
    56  	}
    57  	return nil, errors.New(fmt.Sprintf("No status for host %s", host))
    58  }
    59  
    60  func (ts *fakeTreeStatusClient) postStatus(c context.Context, host, message string, prevKey int64, treeName string, status config.TreeCloserStatus) error {
    61  	ts.mu.Lock()
    62  	defer ts.mu.Unlock()
    63  
    64  	currStatus, exists := ts.statusForHosts[host]
    65  	if exists && currStatus.key != prevKey {
    66  		return errors.New(fmt.Sprintf(
    67  			"prevKey %q passed to postStatus doesn't match previously stored key %q",
    68  			prevKey, currStatus.key))
    69  	}
    70  
    71  	key := ts.nextKey
    72  	ts.nextKey++
    73  
    74  	var messageStatus config.TreeCloserStatus
    75  	if strings.Contains(message, "close") {
    76  		messageStatus = config.Closed
    77  	} else {
    78  		messageStatus = config.Open
    79  	}
    80  	if messageStatus != status {
    81  		return errors.Reason("message status does not match provided status").Err()
    82  	}
    83  
    84  	ts.statusForHosts[host] = treeStatus{
    85  		"buildbot@chromium.org", message, key, messageStatus, time.Now(),
    86  	}
    87  	return nil
    88  }
    89  
    90  func TestUpdateTrees(t *testing.T) {
    91  	Convey("Test environment", t, func() {
    92  		c := memory.Use(context.Background())
    93  		c = common.SetAppIDForTest(c, "luci-notify-test")
    94  
    95  		datastore.GetTestable(c).Consistent(true)
    96  		c = memlogger.Use(c)
    97  		log := logging.Get(c).(*memlogger.MemLogger)
    98  
    99  		project1 := &config.Project{Name: "chromium", TreeClosingEnabled: true}
   100  		project1Key := datastore.KeyForObj(c, project1)
   101  		builder1 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder1"}
   102  		builder2 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder2"}
   103  		builder3 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder3"}
   104  		builder4 := &config.Builder{ProjectKey: project1Key, ID: "ci/builder4"}
   105  
   106  		project2 := &config.Project{Name: "infra", TreeClosingEnabled: false}
   107  		project2Key := datastore.KeyForObj(c, project2)
   108  		builder5 := &config.Builder{ProjectKey: project2Key, ID: "ci/builder5"}
   109  		builder6 := &config.Builder{ProjectKey: project2Key, ID: "ci/builder6"}
   110  
   111  		So(datastore.Put(c, project1, builder1, builder2, builder3, builder4, project2, builder5, builder6), ShouldBeNil)
   112  
   113  		earlierTime := time.Now().AddDate(-1, 0, 0).UTC()
   114  		evenEarlierTime := time.Now().AddDate(-2, 0, 0).UTC()
   115  
   116  		cleanup := func() {
   117  			var treeClosers []*config.TreeCloser
   118  			So(datastore.GetAll(c, datastore.NewQuery("TreeClosers"), &treeClosers), ShouldBeNil)
   119  			datastore.Delete(c, treeClosers)
   120  		}
   121  
   122  		// Helper function for basic tests. Sets an initial tree state, adds two tree closers
   123  		// for the tree, and checks that updateTrees sets the tree to the correct state.
   124  		testUpdateTrees := func(initialTreeStatus, builder1Status, builder2Status, expectedStatus config.TreeCloserStatus) {
   125  			var statusMessage string
   126  			if initialTreeStatus == config.Open {
   127  				statusMessage = "Open for business"
   128  			} else {
   129  				statusMessage = "Closed up"
   130  			}
   131  			ts := fakeTreeStatusClient{
   132  				statusForHosts: map[string]treeStatus{
   133  					"chromium-status.appspot.com": {
   134  						username:  botUsername,
   135  						message:   statusMessage,
   136  						key:       -1,
   137  						status:    initialTreeStatus,
   138  						timestamp: earlierTime,
   139  					},
   140  				},
   141  			}
   142  
   143  			So(datastore.Put(c, &config.TreeCloser{
   144  				BuilderKey:     datastore.KeyForObj(c, builder1),
   145  				TreeStatusHost: "chromium-status.appspot.com",
   146  				TreeCloser:     notifypb.TreeCloser{},
   147  				Status:         builder1Status,
   148  				Timestamp:      time.Now().UTC(),
   149  			}), ShouldBeNil)
   150  			So(datastore.Put(c, &config.TreeCloser{
   151  				BuilderKey:     datastore.KeyForObj(c, builder2),
   152  				TreeStatusHost: "chromium-status.appspot.com",
   153  				TreeCloser:     notifypb.TreeCloser{},
   154  				Status:         builder2Status,
   155  				Timestamp:      time.Now().UTC(),
   156  			}), ShouldBeNil)
   157  			defer cleanup()
   158  
   159  			So(updateTrees(c, &ts), ShouldBeNil)
   160  
   161  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   162  			So(err, ShouldBeNil)
   163  			So(status.status, ShouldEqual, expectedStatus)
   164  		}
   165  
   166  		Convey("Open, both TCs failing, closes", func() {
   167  			testUpdateTrees(config.Open, config.Closed, config.Closed, config.Closed)
   168  		})
   169  
   170  		Convey("Open, 1 failing & 1 passing TC, closes", func() {
   171  			testUpdateTrees(config.Open, config.Closed, config.Open, config.Closed)
   172  		})
   173  
   174  		Convey("Open, both TCs passing, stays open", func() {
   175  			testUpdateTrees(config.Open, config.Open, config.Open, config.Open)
   176  		})
   177  
   178  		Convey("Closed, both TCs failing, stays closed", func() {
   179  			testUpdateTrees(config.Closed, config.Closed, config.Closed, config.Closed)
   180  		})
   181  
   182  		Convey("Closed, 1 failing & 1 passing TC, stays closed", func() {
   183  			testUpdateTrees(config.Closed, config.Closed, config.Open, config.Closed)
   184  		})
   185  
   186  		Convey("Closed, both TCs, stays closed", func() {
   187  			testUpdateTrees(config.Closed, config.Closed, config.Open, config.Closed)
   188  		})
   189  
   190  		Convey("Closed manually, doesn't re-open", func() {
   191  			ts := fakeTreeStatusClient{
   192  				statusForHosts: map[string]treeStatus{
   193  					"chromium-status.appspot.com": {
   194  						username:  "somedev@chromium.org",
   195  						message:   "Closed because of reasons",
   196  						key:       -1,
   197  						status:    config.Closed,
   198  						timestamp: earlierTime,
   199  					},
   200  				},
   201  			}
   202  
   203  			So(datastore.Put(c, &config.TreeCloser{
   204  				BuilderKey:     datastore.KeyForObj(c, builder1),
   205  				TreeStatusHost: "chromium-status.appspot.com",
   206  				TreeCloser:     notifypb.TreeCloser{},
   207  				Status:         config.Open,
   208  				Timestamp:      time.Now().UTC(),
   209  			}), ShouldBeNil)
   210  			defer cleanup()
   211  
   212  			So(updateTrees(c, &ts), ShouldBeNil)
   213  
   214  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   215  			So(err, ShouldBeNil)
   216  			So(status.status, ShouldEqual, config.Closed)
   217  		})
   218  
   219  		Convey("Opened manually, stays open with no new failures", func() {
   220  			ts := fakeTreeStatusClient{
   221  				statusForHosts: map[string]treeStatus{
   222  					"chromium-status.appspot.com": {
   223  						username:  "somedev@chromium.org",
   224  						message:   "Opened, because I feel like it",
   225  						key:       -1,
   226  						status:    config.Open,
   227  						timestamp: earlierTime,
   228  					},
   229  				},
   230  			}
   231  
   232  			So(datastore.Put(c, &config.TreeCloser{
   233  				BuilderKey:     datastore.KeyForObj(c, builder1),
   234  				TreeStatusHost: "chromium-status.appspot.com",
   235  				TreeCloser:     notifypb.TreeCloser{},
   236  				Status:         config.Closed,
   237  				Timestamp:      evenEarlierTime,
   238  			}), ShouldBeNil)
   239  			defer cleanup()
   240  
   241  			So(updateTrees(c, &ts), ShouldBeNil)
   242  
   243  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   244  			So(err, ShouldBeNil)
   245  			So(status.status, ShouldEqual, config.Open)
   246  			So(status.message, ShouldEqual, "Opened, because I feel like it")
   247  		})
   248  
   249  		Convey("Opened manually, closes on new failure", func() {
   250  			ts := fakeTreeStatusClient{
   251  				statusForHosts: map[string]treeStatus{
   252  					"chromium-status.appspot.com": {
   253  						username:  "somedev@chromium.org",
   254  						message:   "Opened, because I feel like it",
   255  						key:       -1,
   256  						status:    config.Open,
   257  						timestamp: earlierTime,
   258  					},
   259  				},
   260  			}
   261  
   262  			So(datastore.Put(c, &config.TreeCloser{
   263  				BuilderKey:     datastore.KeyForObj(c, builder1),
   264  				TreeStatusHost: "chromium-status.appspot.com",
   265  				TreeCloser:     notifypb.TreeCloser{},
   266  				Status:         config.Closed,
   267  				Timestamp:      time.Now().UTC(),
   268  			}), ShouldBeNil)
   269  			defer cleanup()
   270  
   271  			So(updateTrees(c, &ts), ShouldBeNil)
   272  
   273  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   274  			So(err, ShouldBeNil)
   275  			So(status.status, ShouldEqual, config.Closed)
   276  		})
   277  
   278  		Convey("Multiple trees", func() {
   279  			ts := fakeTreeStatusClient{
   280  				statusForHosts: map[string]treeStatus{
   281  					"chromium-status.appspot.com": {
   282  						username:  botUsername,
   283  						message:   "Closed up",
   284  						key:       -1,
   285  						status:    config.Closed,
   286  						timestamp: evenEarlierTime,
   287  					},
   288  					"v8-status.appspot.com": {
   289  						username:  botUsername,
   290  						message:   "Open for business",
   291  						key:       -1,
   292  						status:    config.Open,
   293  						timestamp: evenEarlierTime,
   294  					},
   295  				},
   296  			}
   297  
   298  			So(datastore.Put(c, &config.TreeCloser{
   299  				BuilderKey:     datastore.KeyForObj(c, builder1),
   300  				TreeStatusHost: "chromium-status.appspot.com",
   301  				TreeCloser:     notifypb.TreeCloser{},
   302  				Status:         config.Open,
   303  				Timestamp:      time.Now().UTC(),
   304  			}), ShouldBeNil)
   305  			So(datastore.Put(c, &config.TreeCloser{
   306  				BuilderKey:     datastore.KeyForObj(c, builder2),
   307  				TreeStatusHost: "chromium-status.appspot.com",
   308  				TreeCloser:     notifypb.TreeCloser{},
   309  				Status:         config.Open,
   310  				Timestamp:      time.Now().UTC(),
   311  			}), ShouldBeNil)
   312  			So(datastore.Put(c, &config.TreeCloser{
   313  				BuilderKey:     datastore.KeyForObj(c, builder3),
   314  				TreeStatusHost: "chromium-status.appspot.com",
   315  				TreeCloser:     notifypb.TreeCloser{},
   316  				Status:         config.Open,
   317  				Timestamp:      time.Now().UTC(),
   318  			}), ShouldBeNil)
   319  
   320  			So(datastore.Put(c, &config.TreeCloser{
   321  				BuilderKey:     datastore.KeyForObj(c, builder2),
   322  				TreeStatusHost: "v8-status.appspot.com",
   323  				TreeCloser:     notifypb.TreeCloser{},
   324  				Status:         config.Open,
   325  				Timestamp:      time.Now().UTC(),
   326  			}), ShouldBeNil)
   327  			So(datastore.Put(c, &config.TreeCloser{
   328  				BuilderKey:     datastore.KeyForObj(c, builder3),
   329  				TreeStatusHost: "v8-status.appspot.com",
   330  				TreeCloser:     notifypb.TreeCloser{},
   331  				Status:         config.Open,
   332  				Timestamp:      time.Now().UTC(),
   333  			}), ShouldBeNil)
   334  			So(datastore.Put(c, &config.TreeCloser{
   335  				BuilderKey:     datastore.KeyForObj(c, builder4),
   336  				TreeStatusHost: "v8-status.appspot.com",
   337  				TreeCloser:     notifypb.TreeCloser{},
   338  				Status:         config.Closed,
   339  				Timestamp:      earlierTime,
   340  				Message:        "Correct message",
   341  			}), ShouldBeNil)
   342  
   343  			defer cleanup()
   344  
   345  			So(updateTrees(c, &ts), ShouldBeNil)
   346  
   347  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   348  			So(err, ShouldBeNil)
   349  			So(status.status, ShouldEqual, config.Open)
   350  			So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
   351  
   352  			status, err = ts.getStatus(c, "v8-status.appspot.com")
   353  			So(err, ShouldBeNil)
   354  			So(status.status, ShouldEqual, config.Closed)
   355  			So(status.message, ShouldEqual, "Tree is closed (Automatic: Correct message)")
   356  		})
   357  
   358  		Convey("Doesn't close when build is older than last status update", func() {
   359  			ts := fakeTreeStatusClient{
   360  				statusForHosts: map[string]treeStatus{
   361  					"chromium-status.appspot.com": {
   362  						username:  "somedev@chromium.org",
   363  						message:   "Opened, because I feel like it",
   364  						key:       -1,
   365  						status:    config.Open,
   366  						timestamp: earlierTime,
   367  					},
   368  				},
   369  			}
   370  
   371  			So(datastore.Put(c, &config.TreeCloser{
   372  				BuilderKey:     datastore.KeyForObj(c, builder1),
   373  				TreeStatusHost: "chromium-status.appspot.com",
   374  				TreeCloser:     notifypb.TreeCloser{},
   375  				Status:         config.Closed,
   376  				Timestamp:      evenEarlierTime,
   377  			}), ShouldBeNil)
   378  			defer cleanup()
   379  
   380  			So(updateTrees(c, &ts), ShouldBeNil)
   381  
   382  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   383  			So(err, ShouldBeNil)
   384  			So(status.status, ShouldEqual, config.Open)
   385  			So(status.message, ShouldEqual, "Opened, because I feel like it")
   386  		})
   387  
   388  		Convey("Doesn't open when build is older than last status update", func() {
   389  			// This test replicates the likely state just after we've
   390  			// automatically closed the tree: the tree is closed with
   391  			// our username, and there is some failing TreeCloser older
   392  			// than the status update.
   393  			ts := fakeTreeStatusClient{
   394  				statusForHosts: map[string]treeStatus{
   395  					"chromium-status.appspot.com": {
   396  						username:  botUsername,
   397  						message:   "Tree is closed (Automatic: some builder failed)",
   398  						key:       -1,
   399  						status:    config.Closed,
   400  						timestamp: earlierTime,
   401  					},
   402  				},
   403  			}
   404  
   405  			So(datastore.Put(c, &config.TreeCloser{
   406  				BuilderKey:     datastore.KeyForObj(c, builder1),
   407  				TreeStatusHost: "chromium-status.appspot.com",
   408  				TreeCloser:     notifypb.TreeCloser{},
   409  				Status:         config.Open,
   410  				Timestamp:      evenEarlierTime,
   411  			}, &config.TreeCloser{
   412  				BuilderKey:     datastore.KeyForObj(c, builder2),
   413  				TreeStatusHost: "chromium-status.appspot.com",
   414  				TreeCloser:     notifypb.TreeCloser{},
   415  				Status:         config.Closed,
   416  				Timestamp:      evenEarlierTime,
   417  			}), ShouldBeNil)
   418  			defer cleanup()
   419  
   420  			So(updateTrees(c, &ts), ShouldBeNil)
   421  
   422  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   423  			So(err, ShouldBeNil)
   424  			So(status.status, ShouldEqual, config.Closed)
   425  			So(status.message, ShouldEqual, "Tree is closed (Automatic: some builder failed)")
   426  		})
   427  
   428  		Convey("Doesn't open when a builder is still failing", func() {
   429  			// This test replicates the likely state after we've automatically
   430  			// closed the tree, but some other builder has had a successful
   431  			// build.
   432  			ts := fakeTreeStatusClient{
   433  				statusForHosts: map[string]treeStatus{
   434  					"chromium-status.appspot.com": {
   435  						username:  botUsername,
   436  						message:   "Tree is closed (Automatic: some builder failed)",
   437  						key:       -1,
   438  						status:    config.Closed,
   439  						timestamp: earlierTime,
   440  					},
   441  				},
   442  			}
   443  
   444  			So(datastore.Put(c, &config.TreeCloser{
   445  				BuilderKey:     datastore.KeyForObj(c, builder1),
   446  				TreeStatusHost: "chromium-status.appspot.com",
   447  				TreeCloser:     notifypb.TreeCloser{},
   448  				Status:         config.Open,
   449  				Timestamp:      time.Now().UTC(),
   450  			}, &config.TreeCloser{
   451  				BuilderKey:     datastore.KeyForObj(c, builder2),
   452  				TreeStatusHost: "chromium-status.appspot.com",
   453  				TreeCloser:     notifypb.TreeCloser{},
   454  				Status:         config.Closed,
   455  				Timestamp:      evenEarlierTime,
   456  			}), ShouldBeNil)
   457  			defer cleanup()
   458  
   459  			So(updateTrees(c, &ts), ShouldBeNil)
   460  
   461  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   462  			So(err, ShouldBeNil)
   463  			So(status.status, ShouldEqual, config.Closed)
   464  			So(status.message, ShouldEqual, "Tree is closed (Automatic: some builder failed)")
   465  		})
   466  
   467  		Convey("Multiple projects", func() {
   468  			ts := fakeTreeStatusClient{
   469  				statusForHosts: map[string]treeStatus{
   470  					"chromium-status.appspot.com": {
   471  						username:  botUsername,
   472  						message:   "Tree is closed (Automatic: some builder failed)",
   473  						key:       -1,
   474  						status:    config.Closed,
   475  						timestamp: earlierTime,
   476  					},
   477  					"infra-status.appspot.com": {
   478  						username:  botUsername,
   479  						message:   "Tree is open (Automatic: Yes!)",
   480  						key:       -1,
   481  						status:    config.Open,
   482  						timestamp: earlierTime,
   483  					},
   484  				},
   485  			}
   486  
   487  			So(datastore.Put(c, &config.TreeCloser{
   488  				BuilderKey:     datastore.KeyForObj(c, builder1),
   489  				TreeStatusHost: "chromium-status.appspot.com",
   490  				TreeCloser:     notifypb.TreeCloser{},
   491  				Status:         config.Open,
   492  				Timestamp:      time.Now().UTC(),
   493  			}, &config.TreeCloser{
   494  				BuilderKey:     datastore.KeyForObj(c, builder5),
   495  				TreeStatusHost: "infra-status.appspot.com",
   496  				TreeCloser:     notifypb.TreeCloser{},
   497  				Status:         config.Closed,
   498  				Timestamp:      time.Now().UTC(),
   499  				Message:        "Close it up!",
   500  			}), ShouldBeNil)
   501  			defer cleanup()
   502  
   503  			So(updateTrees(c, &ts), ShouldBeNil)
   504  
   505  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   506  			So(err, ShouldBeNil)
   507  			So(status.status, ShouldEqual, config.Open)
   508  			So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
   509  
   510  			status, err = ts.getStatus(c, "infra-status.appspot.com")
   511  			So(err, ShouldBeNil)
   512  			So(status.status, ShouldEqual, config.Open)
   513  			So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
   514  
   515  			hasExpectedLog := false
   516  			for _, log := range log.Messages() {
   517  				if log.Level == logging.Info {
   518  					hasExpectedLog = true
   519  					So(log.Msg, ShouldEqual, `Would update status for infra-status.appspot.com to "Tree is closed (Automatic: Close it up!)"`)
   520  				}
   521  			}
   522  
   523  			So(hasExpectedLog, ShouldBeTrue)
   524  		})
   525  
   526  		Convey("Multiple projects, overlapping tree status hosts", func() {
   527  			ts := fakeTreeStatusClient{
   528  				statusForHosts: map[string]treeStatus{
   529  					"chromium-status.appspot.com": {
   530  						username:  botUsername,
   531  						message:   "Tree is open (Flake)",
   532  						key:       -1,
   533  						status:    config.Open,
   534  						timestamp: earlierTime,
   535  					},
   536  					"infra-status.appspot.com": {
   537  						username:  botUsername,
   538  						message:   "Tree is closed (Automatic: Some builder failed)",
   539  						key:       -1,
   540  						status:    config.Closed,
   541  						timestamp: earlierTime,
   542  					},
   543  				},
   544  			}
   545  
   546  			So(datastore.Put(c, &config.TreeCloser{
   547  				BuilderKey:     datastore.KeyForObj(c, builder1),
   548  				TreeStatusHost: "chromium-status.appspot.com",
   549  				TreeCloser:     notifypb.TreeCloser{},
   550  				Status:         config.Open,
   551  				Timestamp:      time.Now().UTC(),
   552  			}, &config.TreeCloser{
   553  				BuilderKey:     datastore.KeyForObj(c, builder5),
   554  				TreeStatusHost: "chromium-status.appspot.com",
   555  				TreeCloser:     notifypb.TreeCloser{},
   556  				Status:         config.Closed,
   557  				Timestamp:      time.Now().UTC(),
   558  			}, &config.TreeCloser{
   559  				BuilderKey:     datastore.KeyForObj(c, builder2),
   560  				TreeStatusHost: "infra-status.appspot.com",
   561  				TreeCloser:     notifypb.TreeCloser{},
   562  				Status:         config.Open,
   563  				Timestamp:      time.Now().UTC(),
   564  			}, &config.TreeCloser{
   565  				BuilderKey:     datastore.KeyForObj(c, builder6),
   566  				TreeStatusHost: "infra-status.appspot.com",
   567  				TreeCloser:     notifypb.TreeCloser{},
   568  				Status:         config.Closed,
   569  				Timestamp:      time.Now().UTC(),
   570  			}), ShouldBeNil)
   571  			defer cleanup()
   572  
   573  			So(updateTrees(c, &ts), ShouldBeNil)
   574  
   575  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   576  			So(err, ShouldBeNil)
   577  			So(status.status, ShouldEqual, config.Open)
   578  			So(status.message, ShouldEqual, "Tree is open (Flake)")
   579  
   580  			status, err = ts.getStatus(c, "infra-status.appspot.com")
   581  			So(err, ShouldBeNil)
   582  			So(status.status, ShouldEqual, config.Open)
   583  			So(status.message, ShouldStartWith, "Tree is open (Automatic: ")
   584  		})
   585  	})
   586  }
   587  
   588  func TestHttpTreeStatusClient(t *testing.T) {
   589  	Convey("Test environment for httpTreeStatusClient", t, func() {
   590  		c := memory.Use(context.Background())
   591  		c = common.SetAppIDForTest(c, "luci-notify-test")
   592  
   593  		// Real responses, with usernames redacted and readable formatting applied.
   594  		responses := map[string]string{
   595  			"https://chromium-status.appspot.com/current?format=json": `{
   596  				"username": "someone@google.com",
   597  				"can_commit_freely": false,
   598  				"general_state": "throttled",
   599  				"key": 5656890264518656,
   600  				"date": "2020-03-31 05:33:52.682351",
   601  				"message": "Tree is throttled (win rel 32 appears to be a goma flake. the other builds seem to be charging ahead OK. will fully open / fully close if win32 does/doesn't improve)"
   602  			}`,
   603  			"https://v8-status.appspot.com/current?format=json": `{
   604  				"username": "someone-else@google.com",
   605  				"can_commit_freely": true,
   606  				"general_state": "open",
   607  				"key": 5739466035560448,
   608  				"date": "2020-04-02 15:21:39.981072",
   609  				"message": "open (flake?)"
   610  			}`,
   611  		}
   612  
   613  		get := func(_ context.Context, url string) ([]byte, error) {
   614  			if s, e := responses[url]; e {
   615  				return []byte(s), nil
   616  			} else {
   617  				return nil, fmt.Errorf("Key not present: %q", url)
   618  			}
   619  		}
   620  
   621  		var postUrls []string
   622  		post := func(_ context.Context, url string) error {
   623  			postUrls = append(postUrls, url)
   624  			return nil
   625  		}
   626  
   627  		fakePrpcClient := &fakePRPCTreeStatusClient{}
   628  		ts := httpTreeStatusClient{get, post, fakePrpcClient}
   629  
   630  		Convey("getStatus, open tree", func() {
   631  			status, err := ts.getStatus(c, "chromium-status.appspot.com")
   632  			So(err, ShouldBeNil)
   633  
   634  			expectedTime := time.Date(2020, time.March, 31, 5, 33, 52, 682351000, time.UTC)
   635  			So(status, ShouldResemble, &treeStatus{
   636  				username:  "someone@google.com",
   637  				message:   "Tree is throttled (win rel 32 appears to be a goma flake. the other builds seem to be charging ahead OK. will fully open / fully close if win32 does/doesn't improve)",
   638  				key:       5656890264518656,
   639  				status:    config.Closed,
   640  				timestamp: expectedTime,
   641  			})
   642  		})
   643  
   644  		Convey("getStatus, closed tree", func() {
   645  			status, err := ts.getStatus(c, "v8-status.appspot.com")
   646  			So(err, ShouldBeNil)
   647  
   648  			expectedTime := time.Date(2020, time.April, 2, 15, 21, 39, 981072000, time.UTC)
   649  			So(status, ShouldResemble, &treeStatus{
   650  				username:  "someone-else@google.com",
   651  				message:   "open (flake?)",
   652  				key:       5739466035560448,
   653  				status:    config.Open,
   654  				timestamp: expectedTime,
   655  			})
   656  		})
   657  
   658  		Convey("postStatus", func() {
   659  			err := ts.postStatus(c, "dart-status.appspot.com", "open for business", 1234, "dart", config.Open)
   660  			So(err, ShouldBeNil)
   661  
   662  			So(postUrls, ShouldHaveLength, 1)
   663  			So(postUrls[0], ShouldEqual, "https://dart-status.appspot.com/?last_status_key=1234&message=open+for+business")
   664  			So(fakePrpcClient.latestStatus, ShouldNotBeNil)
   665  			So(fakePrpcClient.latestStatus.Message, ShouldEqual, "open for business")
   666  			So(fakePrpcClient.latestStatus.GeneralState, ShouldEqual, tspb.GeneralState_OPEN)
   667  		})
   668  	})
   669  }
   670  
   671  type fakePRPCTreeStatusClient struct {
   672  	latestStatus *tspb.Status
   673  }
   674  
   675  func (c *fakePRPCTreeStatusClient) ListStatus(ctx context.Context, in *tspb.ListStatusRequest, opts ...grpc.CallOption) (*tspb.ListStatusResponse, error) {
   676  	return nil, errors.Reason("Not implemented").Err()
   677  }
   678  
   679  func (c *fakePRPCTreeStatusClient) GetStatus(ctx context.Context, in *tspb.GetStatusRequest, opts ...grpc.CallOption) (*tspb.Status, error) {
   680  	return nil, errors.Reason("Not implemented").Err()
   681  }
   682  
   683  func (c *fakePRPCTreeStatusClient) CreateStatus(ctx context.Context, in *tspb.CreateStatusRequest, opts ...grpc.CallOption) (*tspb.Status, error) {
   684  	c.latestStatus = in.Status
   685  	return in.Status, nil
   686  }