github.com/shashidharatd/test-infra@v0.0.0-20171006011030-71304e1ca560/mungegithub/mungers/submit-queue_test.go (about)

     1  /*
     2  Copyright 2015 The Kubernetes Authors.
     3  
     4  Licensed under the Apache License, Version 2.0 (the "License");
     5  you may not use this file except in compliance with the License.
     6  You may obtain a copy of the License at
     7  
     8      http://www.apache.org/licenses/LICENSE-2.0
     9  
    10  Unless required by applicable law or agreed to in writing, software
    11  distributed under the License is distributed on an "AS IS" BASIS,
    12  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    13  See the License for the specific language governing permissions and
    14  limitations under the License.
    15  */
    16  
    17  package mungers
    18  
    19  import (
    20  	"encoding/json"
    21  	"fmt"
    22  	"net/http"
    23  	"net/http/httptest"
    24  	"runtime"
    25  	"strconv"
    26  	"strings"
    27  	"sync"
    28  	"testing"
    29  	"time"
    30  
    31  	utilclock "k8s.io/kubernetes/pkg/util/clock"
    32  
    33  	"k8s.io/contrib/test-utils/utils"
    34  	"k8s.io/test-infra/mungegithub/features"
    35  	github_util "k8s.io/test-infra/mungegithub/github"
    36  	github_test "k8s.io/test-infra/mungegithub/github/testing"
    37  	"k8s.io/test-infra/mungegithub/mungeopts"
    38  	"k8s.io/test-infra/mungegithub/mungers/e2e"
    39  	fake_e2e "k8s.io/test-infra/mungegithub/mungers/e2e/fake"
    40  	"k8s.io/test-infra/mungegithub/mungers/mungerutil"
    41  	"k8s.io/test-infra/mungegithub/options"
    42  	"k8s.io/test-infra/mungegithub/sharedmux"
    43  
    44  	"github.com/google/go-github/github"
    45  )
    46  
    47  func stringPtr(val string) *string    { return &val }
    48  func boolPtr(val bool) *bool          { return &val }
    49  func intPtr(val int) *int             { return &val }
    50  func slicePtr(val []string) *[]string { return &val }
    51  
    52  const (
    53  	someUserName        = "someUserName"
    54  	doNotMergeMilestone = "some-milestone-you-should-not-merge"
    55  
    56  	notRequiredReTestContext1 = "someNotRequiredForRetest1"
    57  	notRequiredReTestContext2 = "someNotRequiredForRetest2"
    58  	requiredReTestContext1    = "someRequiredRetestContext1"
    59  	requiredReTestContext2    = "someRequiredRetestContext2"
    60  )
    61  
    62  var (
    63  	someJobNames = []string{"foo", "bar"}
    64  )
    65  
    66  func ValidPR() *github.PullRequest {
    67  	return github_test.PullRequest(someUserName, false, true, true)
    68  }
    69  
    70  func UnMergeablePR() *github.PullRequest {
    71  	return github_test.PullRequest(someUserName, false, true, false)
    72  }
    73  
    74  func UndeterminedMergeablePR() *github.PullRequest {
    75  	return github_test.PullRequest(someUserName, false, false, false)
    76  }
    77  
    78  func MasterCommit() *github.RepositoryCommit {
    79  	masterSHA := "mastersha"
    80  	return &github.RepositoryCommit{
    81  		SHA: &masterSHA,
    82  	}
    83  }
    84  
    85  func BareIssue() *github.Issue {
    86  	return github_test.Issue(someUserName, 1, []string{}, true)
    87  }
    88  
    89  func LGTMIssue() *github.Issue {
    90  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel}, true)
    91  }
    92  
    93  func LGTMApprovedIssue() *github.Issue {
    94  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel}, true)
    95  }
    96  
    97  func LGTMApprovedCLAHumanApprovedIssue() *github.Issue {
    98  	return github_test.Issue(someUserName, 1, []string{claHumanLabel, lgtmLabel, approvedLabel}, true)
    99  }
   100  
   101  func CriticalFixLGTMApprovedIssue() *github.Issue {
   102  	return github_test.Issue(someUserName, 1, []string{criticalFixLabel, cncfClaYesLabel, lgtmLabel, approvedLabel}, true)
   103  }
   104  
   105  func OnlyApprovedIssue() *github.Issue {
   106  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, approvedLabel}, true)
   107  }
   108  
   109  func DoNotMergeIssue() *github.Issue {
   110  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, doNotMergeLabel}, true)
   111  }
   112  
   113  func CherrypickUnapprovedIssue() *github.Issue {
   114  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, cherrypickUnapprovedLabel}, true)
   115  }
   116  
   117  func BlockedPathsIssue() *github.Issue {
   118  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, blockedPathsLabel}, true)
   119  }
   120  
   121  func DeprecatedMissingReleaseNoteIssue() *github.Issue {
   122  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, deprecatedReleaseNoteLabelNeeded}, true)
   123  }
   124  
   125  func MissingReleaseNoteIssue() *github.Issue {
   126  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, releaseNoteLabelNeeded}, true)
   127  }
   128  
   129  func WorkInProgressIssue() *github.Issue {
   130  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, wipLabel}, true)
   131  }
   132  
   133  func HoldLabelIssue() *github.Issue {
   134  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, holdLabel}, true)
   135  }
   136  
   137  func AdditionalLabelIssue(label string) *github.Issue {
   138  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, label}, true)
   139  }
   140  
   141  func DoNotMergeMilestoneIssue() *github.Issue {
   142  	issue := github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel}, true)
   143  	milestone := &github.Milestone{
   144  		Title: stringPtr(doNotMergeMilestone),
   145  	}
   146  	issue.Milestone = milestone
   147  	return issue
   148  }
   149  
   150  func NoCLAIssue() *github.Issue {
   151  	return github_test.Issue(someUserName, 1, []string{lgtmLabel}, true)
   152  }
   153  
   154  func NoLGTMIssue() *github.Issue {
   155  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel}, true)
   156  }
   157  
   158  func NoRetestIssue() *github.Issue {
   159  	return github_test.Issue(someUserName, 1, []string{cncfClaYesLabel, lgtmLabel, approvedLabel, retestNotRequiredLabel}, true)
   160  }
   161  
   162  func OldLGTMEvents() []*github.IssueEvent {
   163  	return github_test.Events([]github_test.LabelTime{
   164  		{User: "bob", Label: approvedLabel, Time: 20},
   165  		{User: "bob", Label: lgtmLabel, Time: 6},
   166  		{User: "bob", Label: lgtmLabel, Time: 7},
   167  		{User: "bob", Label: lgtmLabel, Time: 8},
   168  	})
   169  }
   170  
   171  func NewLGTMEvents() []*github.IssueEvent {
   172  	return github_test.Events([]github_test.LabelTime{
   173  		{User: "bob", Label: approvedLabel, Time: 20},
   174  		{User: "bob", Label: lgtmLabel, Time: 10},
   175  		{User: "bob", Label: lgtmLabel, Time: 11},
   176  		{User: "bob", Label: lgtmLabel, Time: 12},
   177  	})
   178  }
   179  
   180  func OverlappingLGTMEvents() []*github.IssueEvent {
   181  	return github_test.Events([]github_test.LabelTime{
   182  		{User: "bob", Label: approvedLabel, Time: 20},
   183  		{User: "bob", Label: lgtmLabel, Time: 8},
   184  		{User: "bob", Label: lgtmLabel, Time: 9},
   185  		{User: "bob", Label: lgtmLabel, Time: 10},
   186  	})
   187  }
   188  
   189  func OldApprovedEvents() []*github.IssueEvent {
   190  	return github_test.Events([]github_test.LabelTime{
   191  		{User: "bob", Label: approvedLabel, Time: 6},
   192  		{User: "bob", Label: lgtmLabel, Time: 10},
   193  		{User: "bob", Label: lgtmLabel, Time: 11},
   194  		{User: "bob", Label: lgtmLabel, Time: 12},
   195  	})
   196  }
   197  
   198  // Commits returns a slice of github.RepositoryCommit of len==3 which
   199  // happened at times 7, 8, 9
   200  func Commits() []*github.RepositoryCommit {
   201  	return github_test.Commits(3, 7)
   202  }
   203  
   204  func SuccessStatus() *github.CombinedStatus {
   205  	return github_test.Status("mysha", []string{requiredReTestContext1, requiredReTestContext2, notRequiredReTestContext1, notRequiredReTestContext2}, nil, nil, nil)
   206  }
   207  
   208  func RetestFailStatus() *github.CombinedStatus {
   209  	return github_test.Status("mysha", []string{requiredReTestContext1, notRequiredReTestContext1, notRequiredReTestContext2}, []string{requiredReTestContext2}, nil, nil)
   210  }
   211  
   212  func NoRetestFailStatus() *github.CombinedStatus {
   213  	return github_test.Status("mysha", []string{requiredReTestContext1, requiredReTestContext2, notRequiredReTestContext1}, []string{notRequiredReTestContext2}, nil, nil)
   214  }
   215  
   216  func LastBuildNumber() int {
   217  	return 42
   218  }
   219  
   220  func SuccessGCS() utils.FinishedFile {
   221  	return utils.FinishedFile{
   222  		Result:    "SUCCESS",
   223  		Timestamp: uint64(time.Now().Unix()),
   224  	}
   225  }
   226  
   227  func FailGCS() utils.FinishedFile {
   228  	return utils.FinishedFile{
   229  		Result:    "FAILURE",
   230  		Timestamp: uint64(time.Now().Unix()),
   231  	}
   232  }
   233  
   234  func getJUnit(testsNo int, failuresNo int) []byte {
   235  	return []byte(fmt.Sprintf("%v\n<testsuite tests=\"%v\" failures=\"%v\" time=\"1234\">\n</testsuite>",
   236  		e2e.ExpectedXMLHeader, testsNo, failuresNo))
   237  }
   238  
   239  func getTestSQ(startThreads bool, config *github_util.Config, server *httptest.Server) *SubmitQueue {
   240  	// TODO: Remove this line when we fix the plumbing regarding the fake/real e2e tester.
   241  	sharedmux.Admin = sharedmux.NewConcurrentMux(http.NewServeMux())
   242  	sq := new(SubmitQueue)
   243  	sq.opts = options.New()
   244  
   245  	feats := &features.Features{
   246  		Server: &features.ServerFeature{
   247  			Enabled: false,
   248  		},
   249  	}
   250  
   251  	sq.GateApproved = true
   252  	sq.GateCLA = true
   253  	sq.NonBlockingJobNames = someJobNames
   254  	sq.DoNotMergeMilestones = []string{doNotMergeMilestone}
   255  
   256  	mungeopts.RequiredContexts.Merge = []string{notRequiredReTestContext1, notRequiredReTestContext2}
   257  	mungeopts.RequiredContexts.Retest = []string{requiredReTestContext1, requiredReTestContext2}
   258  	mungeopts.PRMaxWaitTime = 2 * time.Hour
   259  
   260  	sq.githubE2EQueue = map[int]*github_util.MungeObject{}
   261  	sq.githubE2EPollTime = time.Millisecond
   262  
   263  	sq.clock = utilclock.NewFakeClock(time.Time{})
   264  	sq.lastMergeTime = sq.clock.Now()
   265  	sq.lastE2EStable = true
   266  	sq.prStatus = map[string]submitStatus{}
   267  	sq.lgtmTimeCache = mungerutil.NewLabelTimeCache(lgtmLabel)
   268  
   269  	sq.startTime = sq.clock.Now()
   270  	sq.healthHistory = make([]healthRecord, 0)
   271  
   272  	sq.e2e = &fake_e2e.FakeE2ETester{JobNames: sq.NonBlockingJobNames}
   273  
   274  	if startThreads {
   275  		sq.internalInitialize(config, feats, server.URL)
   276  		sq.EachLoop()
   277  	}
   278  	return sq
   279  }
   280  
   281  func TestQueueOrder(t *testing.T) {
   282  	timeBase := time.Now()
   283  	time2 := timeBase.Add(6 * time.Minute).Unix()
   284  	time3 := timeBase.Add(5 * time.Minute).Unix()
   285  	time4 := timeBase.Add(4 * time.Minute).Unix()
   286  	time5 := timeBase.Add(3 * time.Minute).Unix()
   287  	time6 := timeBase.Add(2 * time.Minute).Unix()
   288  	labelEvents := map[int][]github_test.LabelTime{
   289  		2: {{User: "me", Label: lgtmLabel, Time: time2}},
   290  		3: {{User: "me", Label: lgtmLabel, Time: time3}},
   291  		4: {{User: "me", Label: lgtmLabel, Time: time4}},
   292  		5: {{User: "me", Label: lgtmLabel, Time: time5}},
   293  		6: {{User: "me", Label: lgtmLabel, Time: time6}},
   294  	}
   295  
   296  	tests := []struct {
   297  		name          string
   298  		issues        []*github.Issue
   299  		issueToEvents map[int][]github_test.LabelTime
   300  		expected      []int
   301  	}{
   302  		{
   303  			name: "Just prNum",
   304  			issues: []*github.Issue{
   305  				github_test.Issue(someUserName, 2, nil, true),
   306  				github_test.Issue(someUserName, 3, nil, true),
   307  				github_test.Issue(someUserName, 4, nil, true),
   308  				github_test.Issue(someUserName, 5, nil, true),
   309  			},
   310  			issueToEvents: labelEvents,
   311  			expected:      []int{5, 4, 3, 2},
   312  		},
   313  		{
   314  			name: "With a priority label",
   315  			issues: []*github.Issue{
   316  				github_test.Issue(someUserName, 2, []string{multirebaseLabel}, true),
   317  				github_test.Issue(someUserName, 3, []string{multirebaseLabel}, true),
   318  				github_test.Issue(someUserName, 4, []string{criticalFixLabel}, true),
   319  				github_test.Issue(someUserName, 5, nil, true),
   320  			},
   321  			issueToEvents: labelEvents,
   322  			expected:      []int{4, 3, 2, 5},
   323  		},
   324  		{
   325  			name: "With two priority labels",
   326  			issues: []*github.Issue{
   327  				github_test.Issue(someUserName, 2, []string{fixLabel, criticalFixLabel}, true),
   328  				github_test.Issue(someUserName, 3, []string{fixLabel}, true),
   329  				github_test.Issue(someUserName, 4, []string{criticalFixLabel}, true),
   330  				github_test.Issue(someUserName, 5, nil, true),
   331  			},
   332  			issueToEvents: labelEvents,
   333  			expected:      []int{4, 2, 3, 5},
   334  		},
   335  		{
   336  			name: "With unrelated labels",
   337  			issues: []*github.Issue{
   338  				github_test.Issue(someUserName, 2, []string{fixLabel, criticalFixLabel}, true),
   339  				github_test.Issue(someUserName, 3, []string{fixLabel, "kind/design"}, true),
   340  				github_test.Issue(someUserName, 4, []string{criticalFixLabel}, true),
   341  				github_test.Issue(someUserName, 5, []string{lgtmLabel, "kind/new-api"}, true),
   342  			},
   343  			issueToEvents: labelEvents,
   344  			expected:      []int{4, 2, 3, 5},
   345  		},
   346  		{
   347  			name: "With invalid priority label",
   348  			issues: []*github.Issue{
   349  				github_test.Issue(someUserName, 2, []string{fixLabel, criticalFixLabel}, true),
   350  				github_test.Issue(someUserName, 3, []string{fixLabel, "kind/design", "priority/high"}, true),
   351  				github_test.Issue(someUserName, 4, []string{criticalFixLabel, "priorty/bob"}, true),
   352  				github_test.Issue(someUserName, 5, nil, true),
   353  			},
   354  			issueToEvents: labelEvents,
   355  			expected:      []int{4, 2, 3, 5},
   356  		},
   357  		{
   358  			name: "Unlabeled counts as below",
   359  			issues: []*github.Issue{
   360  				github_test.Issue(someUserName, 2, nil, true),
   361  				github_test.Issue(someUserName, 3, []string{blocksOthersLabel}, true),
   362  				github_test.Issue(someUserName, 4, []string{multirebaseLabel}, true),
   363  				github_test.Issue(someUserName, 5, nil, true),
   364  			},
   365  			issueToEvents: labelEvents,
   366  			expected:      []int{4, 3, 5, 2},
   367  		},
   368  		{
   369  			name: "retestNotRequiredLabel counts above everything except criticalFixLabel",
   370  			issues: []*github.Issue{
   371  				github_test.Issue(someUserName, 2, nil, true),
   372  				github_test.Issue(someUserName, 3, []string{blocksOthersLabel}, true),
   373  				github_test.Issue(someUserName, 4, []string{fixLabel}, true),
   374  				github_test.Issue(someUserName, 5, []string{criticalFixLabel}, true),
   375  				github_test.Issue(someUserName, 6, []string{blocksOthersLabel, retestNotRequiredLabel}, true),
   376  			},
   377  			issueToEvents: labelEvents,
   378  			expected:      []int{5, 6, 4, 3, 2},
   379  		},
   380  	}
   381  	for testNum, test := range tests {
   382  		config := &github_util.Config{Org: "o", Project: "r"}
   383  		client, server, mux := github_test.InitServer(t, nil, nil, github_test.MultiIssueEvents(test.issueToEvents, "labeled"), nil, nil, nil, nil)
   384  		config.SetClient(client)
   385  		sq := getTestSQ(false, config, server)
   386  		for i := range test.issues {
   387  			issue := test.issues[i]
   388  			github_test.ServeIssue(t, mux, issue)
   389  
   390  			issueNum := *issue.Number
   391  			obj, err := config.GetObject(issueNum)
   392  			if err != nil {
   393  				t.Fatalf("%d:%q unable to get issue: %v", testNum, test.name, err)
   394  			}
   395  			sq.githubE2EQueue[issueNum] = obj
   396  		}
   397  		actual := sq.orderedE2EQueue()
   398  		if len(actual) != len(test.expected) {
   399  			t.Fatalf("%d:%q len(actual):%v != len(expected):%v", testNum, test.name, actual, test.expected)
   400  		}
   401  		for i, a := range actual {
   402  			e := test.expected[i]
   403  			if a != e {
   404  				t.Errorf("%d:%q a[%d]:%d != e[%d]:%d", testNum, test.name, i, a, i, e)
   405  			}
   406  		}
   407  		server.Close()
   408  	}
   409  }
   410  
   411  func TestValidateLGTMAfterPush(t *testing.T) {
   412  	tests := []struct {
   413  		issueEvents []*github.IssueEvent
   414  		commits     []*github.RepositoryCommit
   415  		shouldPass  bool
   416  	}{
   417  		{
   418  			issueEvents: NewLGTMEvents(), // Label >= time.Unix(10)
   419  			commits:     Commits(),       // Modified at time.Unix(7), 8, and 9
   420  			shouldPass:  true,
   421  		},
   422  		{
   423  			issueEvents: OldLGTMEvents(), // Label <= time.Unix(8)
   424  			commits:     Commits(),       // Modified at time.Unix(7), 8, and 9
   425  			shouldPass:  false,
   426  		},
   427  		{
   428  			issueEvents: OverlappingLGTMEvents(), // Labeled at 8, 9, and 10
   429  			commits:     Commits(),               // Modified at time.Unix(7), 8, and 9
   430  			shouldPass:  true,
   431  		},
   432  	}
   433  	for testNum, test := range tests {
   434  		config := &github_util.Config{Org: "o", Project: "r"}
   435  		client, server, _ := github_test.InitServer(t, nil, nil, test.issueEvents, test.commits, nil, nil, nil)
   436  		config.SetClient(client)
   437  
   438  		obj := github_util.NewTestObject(config, BareIssue(), nil, nil, nil)
   439  
   440  		if _, ok := obj.GetCommits(); !ok {
   441  			t.Errorf("Unexpected error getting filled commits")
   442  		}
   443  
   444  		if _, ok := obj.GetEvents(); !ok {
   445  			t.Errorf("Unexpected error getting events commits")
   446  		}
   447  
   448  		lastModifiedTime, ok1 := obj.LastModifiedTime()
   449  		lgtmTime, ok2 := obj.LabelTime(lgtmLabel)
   450  
   451  		if !ok1 || !ok2 || lastModifiedTime == nil || lgtmTime == nil {
   452  			t.Errorf("unexpected lastModifiedTime or lgtmTime == nil")
   453  		}
   454  
   455  		ok := !lastModifiedTime.After(*lgtmTime)
   456  
   457  		if ok != test.shouldPass {
   458  			t.Errorf("%d: expected: %v, saw: %v", testNum, test.shouldPass, ok)
   459  		}
   460  		server.Close()
   461  	}
   462  }
   463  
   464  func setStatus(status *github.RepoStatus, success bool) {
   465  	if success {
   466  		status.State = stringPtr("success")
   467  	} else {
   468  		status.State = stringPtr("failure")
   469  	}
   470  }
   471  
   472  func addStatus(context string, success bool, ciStatus *github.CombinedStatus) {
   473  	status := github.RepoStatus{
   474  		Context: stringPtr(context),
   475  	}
   476  	setStatus(&status, success)
   477  	ciStatus.Statuses = append(ciStatus.Statuses, status)
   478  }
   479  
   480  // fakeRunGithubE2ESuccess imitates jenkins running
   481  func fakeRunGithubE2ESuccess(ciStatus *github.CombinedStatus, context1Pass, context2Pass bool) {
   482  	ciStatus.State = stringPtr("pending")
   483  	for id := range ciStatus.Statuses {
   484  		status := &ciStatus.Statuses[id]
   485  		if *status.Context == requiredReTestContext1 || *status.Context == requiredReTestContext2 {
   486  			status.State = stringPtr("pending")
   487  		}
   488  	}
   489  	// short sleep like the test is running
   490  	time.Sleep(500 * time.Millisecond)
   491  	if context1Pass && context2Pass {
   492  		ciStatus.State = stringPtr("success")
   493  	}
   494  	foundContext1 := false
   495  	foundContext2 := false
   496  	for id := range ciStatus.Statuses {
   497  		status := &ciStatus.Statuses[id]
   498  		if *status.Context == requiredReTestContext1 {
   499  			setStatus(status, context1Pass)
   500  			foundContext1 = true
   501  		}
   502  		if *status.Context == requiredReTestContext2 {
   503  			setStatus(status, context2Pass)
   504  			foundContext2 = true
   505  		}
   506  	}
   507  	if !foundContext1 {
   508  		addStatus(jenkinsE2EContext, context1Pass, ciStatus)
   509  	}
   510  	if !foundContext2 {
   511  		addStatus(jenkinsUnitContext, context2Pass, ciStatus)
   512  	}
   513  }
   514  
   515  func TestSubmitQueue(t *testing.T) {
   516  	runtime.GOMAXPROCS(runtime.NumCPU())
   517  
   518  	// Since we testing, don't rateLimit api calls. Go hog wild
   519  	github_util.SetCombinedStatusLifetime(1)
   520  
   521  	tests := []struct {
   522  		name             string // because when the fail, counting is hard
   523  		pr               *github.PullRequest
   524  		issue            *github.Issue
   525  		commits          []*github.RepositoryCommit
   526  		events           []*github.IssueEvent
   527  		additionalLabels []string
   528  		ciStatus         *github.CombinedStatus
   529  		lastBuildNumber  int
   530  		gcsResult        utils.FinishedFile
   531  		retest1Pass      bool
   532  		retest2Pass      bool
   533  		mergeAfterQueued bool
   534  		reason           string
   535  		state            string // what the github status context should be for the PR HEAD
   536  
   537  		emergencyMergeStop bool
   538  		isMerged           bool
   539  
   540  		imHeadSHA      string
   541  		imBaseSHA      string
   542  		masterCommit   *github.RepositoryCommit
   543  		retestsAvoided int // desired output
   544  	}{
   545  		// Should pass because the entire thing was run and good (with cncf-cla: yes)
   546  		{
   547  			name:            "Test1",
   548  			pr:              ValidPR(),
   549  			issue:           LGTMApprovedIssue(),
   550  			events:          NewLGTMEvents(),
   551  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   552  			ciStatus:        SuccessStatus(),
   553  			lastBuildNumber: LastBuildNumber(),
   554  			gcsResult:       SuccessGCS(),
   555  			retest1Pass:     true,
   556  			retest2Pass:     true,
   557  			reason:          merged,
   558  			state:           "success",
   559  			isMerged:        true,
   560  		},
   561  		// Should pass because the entire thing was run and good (with cla: human-approved)
   562  		{
   563  			name:            "Test1",
   564  			pr:              ValidPR(),
   565  			issue:           LGTMApprovedCLAHumanApprovedIssue(),
   566  			events:          NewLGTMEvents(),
   567  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   568  			ciStatus:        SuccessStatus(),
   569  			lastBuildNumber: LastBuildNumber(),
   570  			gcsResult:       SuccessGCS(),
   571  			retest1Pass:     true,
   572  			retest2Pass:     true,
   573  			reason:          merged,
   574  			state:           "success",
   575  			isMerged:        true,
   576  		},
   577  		{
   578  			name:            "Test1+NoLgtm",
   579  			pr:              ValidPR(),
   580  			issue:           OnlyApprovedIssue(),
   581  			events:          NewLGTMEvents(),
   582  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   583  			ciStatus:        SuccessStatus(),
   584  			lastBuildNumber: LastBuildNumber(),
   585  			gcsResult:       SuccessGCS(),
   586  			retest1Pass:     true,
   587  			retest2Pass:     true,
   588  			reason:          noLGTM,
   589  			state:           "pending",
   590  			isMerged:        false,
   591  		},
   592  		// Entire thing was run and good, but emergency merge stop in progress
   593  		{
   594  			name:               "Test1+emergencyStop",
   595  			pr:                 ValidPR(),
   596  			issue:              LGTMApprovedIssue(),
   597  			events:             NewLGTMEvents(),
   598  			commits:            Commits(), // Modified at time.Unix(7), 8, and 9
   599  			ciStatus:           SuccessStatus(),
   600  			lastBuildNumber:    LastBuildNumber(),
   601  			gcsResult:          SuccessGCS(),
   602  			retest1Pass:        true,
   603  			retest2Pass:        true,
   604  			emergencyMergeStop: true,
   605  			isMerged:           false,
   606  			reason:             e2eFailure,
   607  			state:              "success",
   608  		},
   609  		// Should pass without running tests because we had a previous run.
   610  		// TODO: Add a proper test to make sure we don't shuffle queue when we can just merge a PR
   611  		{
   612  			name:            "Test1+prevsuccess",
   613  			pr:              ValidPR(),
   614  			issue:           LGTMApprovedIssue(),
   615  			events:          NewLGTMEvents(),
   616  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   617  			ciStatus:        SuccessStatus(),
   618  			lastBuildNumber: LastBuildNumber(),
   619  			gcsResult:       SuccessGCS(),
   620  			retest1Pass:     true,
   621  			retest2Pass:     true,
   622  			reason:          merged,
   623  			state:           "success",
   624  			isMerged:        true,
   625  			retestsAvoided:  1,
   626  			imHeadSHA:       "mysha", // Set by ValidPR
   627  			imBaseSHA:       "mastersha",
   628  			masterCommit:    MasterCommit(),
   629  		},
   630  		// Should list as 'merged' but the merge should happen before it gets e2e tested
   631  		// and we should bail early instead of waiting for a test that will never come.
   632  		{
   633  			name:            "Test2",
   634  			pr:              ValidPR(),
   635  			issue:           LGTMApprovedIssue(),
   636  			events:          NewLGTMEvents(),
   637  			commits:         Commits(),
   638  			ciStatus:        SuccessStatus(),
   639  			lastBuildNumber: LastBuildNumber(),
   640  			gcsResult:       SuccessGCS(),
   641  			// The test should never run, but if it does, make sure it fails
   642  			mergeAfterQueued: true,
   643  			reason:           mergedByHand,
   644  			state:            "success",
   645  		},
   646  		// Should merge even though retest1Pass would have failed before of `retestNotRequiredLabel`
   647  		{
   648  			name:            "merge because of retestNotRequired",
   649  			pr:              ValidPR(),
   650  			issue:           NoRetestIssue(),
   651  			events:          NewLGTMEvents(),
   652  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   653  			ciStatus:        SuccessStatus(),
   654  			lastBuildNumber: LastBuildNumber(),
   655  			gcsResult:       SuccessGCS(),
   656  			retest1Pass:     false,
   657  			retest2Pass:     false,
   658  			reason:          mergedSkippedRetest,
   659  			state:           "success",
   660  			isMerged:        true,
   661  		},
   662  		// Fail because PR can't automatically merge
   663  		{
   664  			name:   "Test5",
   665  			pr:     UnMergeablePR(),
   666  			issue:  LGTMApprovedIssue(),
   667  			reason: unmergeable,
   668  			state:  "pending",
   669  			// To avoid false errors in logs
   670  			lastBuildNumber: LastBuildNumber(),
   671  			gcsResult:       SuccessGCS(),
   672  		},
   673  		// Fail because we don't know if PR can automatically merge
   674  		{
   675  			name:   "Test6",
   676  			pr:     UndeterminedMergeablePR(),
   677  			issue:  LGTMApprovedIssue(),
   678  			reason: undeterminedMergability,
   679  			state:  "pending",
   680  			// To avoid false errors in logs
   681  			lastBuildNumber: LastBuildNumber(),
   682  			gcsResult:       SuccessGCS(),
   683  		},
   684  		// Fail because the cncfClaYesLabel or claHumanLabel label was not applied
   685  		{
   686  			name:   "Test7",
   687  			pr:     ValidPR(),
   688  			issue:  NoCLAIssue(),
   689  			reason: noCLA,
   690  			state:  "pending",
   691  			// To avoid false errors in logs
   692  			lastBuildNumber: LastBuildNumber(),
   693  			gcsResult:       SuccessGCS(),
   694  		},
   695  		// Fail because github CI tests have failed (or at least are not success)
   696  		{
   697  			name:   "Test8",
   698  			pr:     ValidPR(),
   699  			issue:  LGTMApprovedIssue(),
   700  			reason: fmt.Sprintf(ciFailureFmt, notRequiredReTestContext1),
   701  			state:  "pending",
   702  			// To avoid false errors in logs
   703  			lastBuildNumber: LastBuildNumber(),
   704  			gcsResult:       SuccessGCS(),
   705  		},
   706  		// Fail because missing LGTM label
   707  		{
   708  			name:     "Test10",
   709  			pr:       ValidPR(),
   710  			issue:    NoLGTMIssue(),
   711  			ciStatus: SuccessStatus(),
   712  			reason:   noLGTM,
   713  			state:    "pending",
   714  			// To avoid false errors in logs
   715  			lastBuildNumber: LastBuildNumber(),
   716  			gcsResult:       SuccessGCS(),
   717  		},
   718  		// Fail because we can't tell if LGTM was added before the last change
   719  		{
   720  			name:     "Test11",
   721  			pr:       ValidPR(),
   722  			issue:    LGTMApprovedIssue(),
   723  			ciStatus: SuccessStatus(),
   724  			reason:   unknown,
   725  			state:    "failure",
   726  			// To avoid false errors in logs
   727  			lastBuildNumber: LastBuildNumber(),
   728  			gcsResult:       SuccessGCS(),
   729  		},
   730  		// Fail because LGTM was added before the last change
   731  		{
   732  			name:     "Test12",
   733  			pr:       ValidPR(),
   734  			issue:    LGTMApprovedIssue(),
   735  			ciStatus: SuccessStatus(),
   736  			events:   OldLGTMEvents(),
   737  			commits:  Commits(), // Modified at time.Unix(7), 8, and 9
   738  			reason:   lgtmEarly,
   739  			state:    "pending",
   740  		},
   741  		// Fail because jenkins instances are failing (whole submit queue blocks)
   742  		{
   743  			name:            "Test13",
   744  			pr:              ValidPR(),
   745  			issue:           LGTMApprovedIssue(),
   746  			ciStatus:        SuccessStatus(),
   747  			events:          NewLGTMEvents(),
   748  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   749  			lastBuildNumber: LastBuildNumber(),
   750  			gcsResult:       FailGCS(),
   751  			reason:          ghE2EQueued,
   752  			state:           "success",
   753  		},
   754  		// Fail because the second run of github e2e tests failed
   755  		{
   756  			name:            "Test14",
   757  			pr:              ValidPR(),
   758  			issue:           LGTMApprovedIssue(),
   759  			ciStatus:        SuccessStatus(),
   760  			events:          NewLGTMEvents(),
   761  			commits:         Commits(),
   762  			lastBuildNumber: LastBuildNumber(),
   763  			gcsResult:       SuccessGCS(),
   764  			reason:          ghE2EFailed,
   765  			state:           "pending",
   766  		},
   767  		// When we check the reason it may be queued or it may already have failed.
   768  		{
   769  			name:            "Test15",
   770  			pr:              ValidPR(),
   771  			issue:           LGTMApprovedIssue(),
   772  			ciStatus:        SuccessStatus(),
   773  			events:          NewLGTMEvents(),
   774  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   775  			lastBuildNumber: LastBuildNumber(),
   776  			gcsResult:       SuccessGCS(),
   777  			reason:          ghE2EQueued,
   778  			// The state is unpredictable. When it goes on the queue it is success.
   779  			// When it fails the build it is pending. So state depends on how far along
   780  			// this were when we checked. Thus just don't check it...
   781  			state: "",
   782  		},
   783  		// Fail because the second run of github e2e tests failed
   784  		{
   785  			name:            "Test16",
   786  			pr:              ValidPR(),
   787  			issue:           LGTMApprovedIssue(),
   788  			ciStatus:        SuccessStatus(),
   789  			events:          NewLGTMEvents(),
   790  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   791  			lastBuildNumber: LastBuildNumber(),
   792  			gcsResult:       SuccessGCS(),
   793  			reason:          ghE2EFailed,
   794  			state:           "pending",
   795  		},
   796  		{
   797  			name:            "Fail because E2E pass, but unit test fail",
   798  			pr:              ValidPR(),
   799  			issue:           LGTMApprovedIssue(),
   800  			events:          NewLGTMEvents(),
   801  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   802  			ciStatus:        SuccessStatus(),
   803  			lastBuildNumber: LastBuildNumber(),
   804  			gcsResult:       SuccessGCS(),
   805  			retest1Pass:     true,
   806  			retest2Pass:     false,
   807  			reason:          ghE2EFailed,
   808  			state:           "pending",
   809  		},
   810  		{
   811  			name:            "Fail because E2E fail, but unit test pass",
   812  			pr:              ValidPR(),
   813  			issue:           LGTMApprovedIssue(),
   814  			events:          NewLGTMEvents(),
   815  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   816  			ciStatus:        SuccessStatus(),
   817  			lastBuildNumber: LastBuildNumber(),
   818  			gcsResult:       SuccessGCS(),
   819  			retest1Pass:     false,
   820  			retest2Pass:     true,
   821  			reason:          ghE2EFailed,
   822  			state:           "pending",
   823  		},
   824  		{
   825  			name:            "Fail because missing release note label is present",
   826  			pr:              ValidPR(),
   827  			issue:           MissingReleaseNoteIssue(),
   828  			events:          NewLGTMEvents(),
   829  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   830  			ciStatus:        SuccessStatus(),
   831  			lastBuildNumber: LastBuildNumber(),
   832  			gcsResult:       SuccessGCS(),
   833  			retest1Pass:     true,
   834  			retest2Pass:     true,
   835  			reason:          noMergeMessage(releaseNoteLabelNeeded),
   836  			state:           "pending",
   837  		},
   838  		{
   839  			name:            "Fail because hold label is present",
   840  			pr:              ValidPR(),
   841  			issue:           HoldLabelIssue(),
   842  			events:          NewLGTMEvents(),
   843  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   844  			ciStatus:        SuccessStatus(),
   845  			lastBuildNumber: LastBuildNumber(),
   846  			gcsResult:       SuccessGCS(),
   847  			retest1Pass:     true,
   848  			retest2Pass:     true,
   849  			reason:          noMergeMessage(holdLabel),
   850  			state:           "pending",
   851  		},
   852  		{
   853  			name:             "Fail because kind/blocker label is required but missing",
   854  			pr:               ValidPR(),
   855  			issue:            LGTMApprovedIssue(),
   856  			additionalLabels: []string{"kind/blocker"},
   857  			events:           NewLGTMEvents(),
   858  			commits:          Commits(), // Modified at time.Unix(7), 8, and 9
   859  			ciStatus:         SuccessStatus(),
   860  			lastBuildNumber:  LastBuildNumber(),
   861  			gcsResult:        SuccessGCS(),
   862  			retest1Pass:      true,
   863  			retest2Pass:      true,
   864  			reason:           noAdditionalLabelMessage("kind/blocker"),
   865  			state:            "pending",
   866  		},
   867  		{
   868  			name:             "Merge kind/blocker PR",
   869  			pr:               ValidPR(),
   870  			issue:            AdditionalLabelIssue("kind/blocker"),
   871  			additionalLabels: []string{"kind/blocker"},
   872  			events:           NewLGTMEvents(),
   873  			commits:          Commits(), // Modified at time.Unix(7), 8, and 9
   874  			ciStatus:         SuccessStatus(),
   875  			lastBuildNumber:  LastBuildNumber(),
   876  			gcsResult:        SuccessGCS(),
   877  			retest1Pass:      true,
   878  			retest2Pass:      true,
   879  			reason:           merged,
   880  			state:            "success",
   881  			isMerged:         true,
   882  		},
   883  		{
   884  			name:            "Fail because deprecated missing release note label is present",
   885  			pr:              ValidPR(),
   886  			issue:           DeprecatedMissingReleaseNoteIssue(),
   887  			events:          NewLGTMEvents(),
   888  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   889  			ciStatus:        SuccessStatus(),
   890  			lastBuildNumber: LastBuildNumber(),
   891  			gcsResult:       SuccessGCS(),
   892  			retest1Pass:     true,
   893  			retest2Pass:     true,
   894  			reason:          noMergeMessage(deprecatedReleaseNoteLabelNeeded),
   895  			state:           "pending",
   896  		},
   897  		{
   898  			name:            "Fail because do not merge label is present",
   899  			pr:              ValidPR(),
   900  			issue:           DoNotMergeIssue(),
   901  			events:          NewLGTMEvents(),
   902  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   903  			ciStatus:        SuccessStatus(),
   904  			lastBuildNumber: LastBuildNumber(),
   905  			gcsResult:       SuccessGCS(),
   906  			retest1Pass:     true,
   907  			retest2Pass:     true,
   908  			reason:          noMergeMessage(doNotMergeLabel),
   909  			state:           "pending",
   910  		},
   911  		{
   912  			name:            "Fail because cherrypick unapproved label is present",
   913  			pr:              ValidPR(),
   914  			issue:           CherrypickUnapprovedIssue(),
   915  			events:          NewLGTMEvents(),
   916  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   917  			ciStatus:        SuccessStatus(),
   918  			lastBuildNumber: LastBuildNumber(),
   919  			gcsResult:       SuccessGCS(),
   920  			retest1Pass:     true,
   921  			retest2Pass:     true,
   922  			reason:          noMergeMessage(cherrypickUnapprovedLabel),
   923  			state:           "pending",
   924  		},
   925  		{
   926  			name:            "Fail because blocked paths label is present",
   927  			pr:              ValidPR(),
   928  			issue:           BlockedPathsIssue(),
   929  			events:          NewLGTMEvents(),
   930  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   931  			ciStatus:        SuccessStatus(),
   932  			lastBuildNumber: LastBuildNumber(),
   933  			gcsResult:       SuccessGCS(),
   934  			retest1Pass:     true,
   935  			retest2Pass:     true,
   936  			reason:          noMergeMessage(blockedPathsLabel),
   937  			state:           "pending",
   938  		},
   939  		{
   940  			name:            "Fail because work-in-progress label is present",
   941  			pr:              ValidPR(),
   942  			issue:           WorkInProgressIssue(),
   943  			events:          NewLGTMEvents(),
   944  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   945  			ciStatus:        SuccessStatus(),
   946  			lastBuildNumber: LastBuildNumber(),
   947  			gcsResult:       SuccessGCS(),
   948  			retest1Pass:     true,
   949  			retest2Pass:     true,
   950  			reason:          noMergeMessage(wipLabel),
   951  			state:           "pending",
   952  		},
   953  		// Should fail because the 'do-not-merge-milestone' is set.
   954  		{
   955  			name:            "Do Not Merge Milestone Set",
   956  			pr:              ValidPR(),
   957  			issue:           DoNotMergeMilestoneIssue(),
   958  			events:          NewLGTMEvents(),
   959  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   960  			ciStatus:        SuccessStatus(),
   961  			lastBuildNumber: LastBuildNumber(),
   962  			gcsResult:       SuccessGCS(),
   963  			retest1Pass:     true,
   964  			retest2Pass:     true,
   965  			reason:          unmergeableMilestone,
   966  			state:           "pending",
   967  		},
   968  		{
   969  			name:            "Fail because retest status fail",
   970  			pr:              ValidPR(),
   971  			issue:           LGTMApprovedIssue(),
   972  			events:          NewLGTMEvents(),
   973  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   974  			ciStatus:        RetestFailStatus(),
   975  			lastBuildNumber: LastBuildNumber(),
   976  			gcsResult:       SuccessGCS(),
   977  			retest1Pass:     true,
   978  			retest2Pass:     true,
   979  			reason:          fmt.Sprintf(ciFailureFmt, requiredReTestContext2),
   980  			state:           "pending",
   981  		},
   982  		{
   983  			name:            "Fail because noretest status fail",
   984  			pr:              ValidPR(),
   985  			issue:           LGTMApprovedIssue(),
   986  			events:          NewLGTMEvents(),
   987  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
   988  			ciStatus:        NoRetestFailStatus(),
   989  			lastBuildNumber: LastBuildNumber(),
   990  			gcsResult:       SuccessGCS(),
   991  			retest1Pass:     true,
   992  			retest2Pass:     true,
   993  			reason:          fmt.Sprintf(ciFailureFmt, notRequiredReTestContext2),
   994  			state:           "pending",
   995  		},
   996  		{
   997  			name:            "Approval Can Happen Before Code Changes",
   998  			pr:              ValidPR(),
   999  			issue:           LGTMApprovedIssue(),
  1000  			events:          OldApprovedEvents(),
  1001  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
  1002  			ciStatus:        SuccessStatus(),
  1003  			lastBuildNumber: LastBuildNumber(),
  1004  			gcsResult:       SuccessGCS(),
  1005  			retest1Pass:     true,
  1006  			retest2Pass:     true,
  1007  			reason:          merged,
  1008  			state:           "success",
  1009  			isMerged:        true,
  1010  			retestsAvoided:  1,
  1011  			imHeadSHA:       "mysha", // Set by ValidPR
  1012  			imBaseSHA:       "mastersha",
  1013  			masterCommit:    MasterCommit(),
  1014  		},
  1015  		{
  1016  			name:            "criticalFixLabel should merge even though jenkins GCS fail",
  1017  			pr:              ValidPR(),
  1018  			issue:           CriticalFixLGTMApprovedIssue(),
  1019  			events:          NewLGTMEvents(),
  1020  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
  1021  			ciStatus:        SuccessStatus(),
  1022  			lastBuildNumber: LastBuildNumber(),
  1023  			gcsResult:       FailGCS(),
  1024  			retest1Pass:     true,
  1025  			retest2Pass:     true,
  1026  			reason:          merged,
  1027  			state:           "success",
  1028  			isMerged:        true,
  1029  		},
  1030  		{
  1031  			name:            "criticalFixLabel but should fail if e2e's fail",
  1032  			pr:              ValidPR(),
  1033  			issue:           CriticalFixLGTMApprovedIssue(),
  1034  			events:          NewLGTMEvents(),
  1035  			commits:         Commits(), // Modified at time.Unix(7), 8, and 9
  1036  			ciStatus:        SuccessStatus(),
  1037  			lastBuildNumber: LastBuildNumber(),
  1038  			gcsResult:       FailGCS(),
  1039  			retest1Pass:     false,
  1040  			retest2Pass:     true,
  1041  			reason:          ghE2EFailed,
  1042  			state:           "pending",
  1043  		},
  1044  	}
  1045  	for testNum := range tests {
  1046  		test := &tests[testNum]
  1047  		t.Logf("---------Starting test %v (%v)---------------------", testNum, test.name)
  1048  		issueNum := testNum + 1
  1049  		issueNumStr := strconv.Itoa(issueNum)
  1050  
  1051  		test.issue.Number = &issueNum
  1052  		client, server, mux := github_test.InitServer(t, test.issue, test.pr, test.events, test.commits, test.ciStatus, test.masterCommit, nil)
  1053  
  1054  		config := &github_util.Config{Org: "o", Project: "r"}
  1055  		config.SetClient(client)
  1056  		// Don't wait so long for retries (pending, mergeability)
  1057  		config.BaseWaitTime = time.Millisecond
  1058  
  1059  		stateSet := ""
  1060  		wasMerged := false
  1061  
  1062  		for _, job := range someJobNames {
  1063  			numTestChecks := 0
  1064  			var testChecksLock sync.Mutex
  1065  			path := fmt.Sprintf("/bucket/logs/%s/latest-build.txt", job)
  1066  			mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
  1067  				if r.Method != "GET" {
  1068  					t.Errorf("Unexpected method: %s", r.Method)
  1069  				}
  1070  				w.WriteHeader(http.StatusOK)
  1071  				w.Write([]byte(strconv.Itoa(test.lastBuildNumber)))
  1072  
  1073  				// There is no good spot for this, but this gets called
  1074  				// before we queue the PR. So mark the PR as "merged".
  1075  				// When the sq initializes, it will check the Jenkins status,
  1076  				// so we don't want to modify the PR there. Instead we need
  1077  				// to wait until the second time we check Jenkins, which happens
  1078  				// we did the IsMerged() check.
  1079  				testChecksLock.Lock()
  1080  				defer testChecksLock.Unlock()
  1081  				numTestChecks = numTestChecks + 1
  1082  				if numTestChecks == 2 && test.mergeAfterQueued {
  1083  					test.pr.Merged = boolPtr(true)
  1084  					test.pr.Mergeable = nil
  1085  				}
  1086  			})
  1087  			path = fmt.Sprintf("/bucket/logs/%s/%v/finished.json", job, test.lastBuildNumber)
  1088  			mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
  1089  				if r.Method != "GET" {
  1090  					t.Errorf("Unexpected method: %s", r.Method)
  1091  				}
  1092  				w.WriteHeader(http.StatusOK)
  1093  				data, err := json.Marshal(test.gcsResult)
  1094  				if err != nil {
  1095  					t.Errorf("Unexpected error: %v", err)
  1096  				}
  1097  				w.Write(data)
  1098  			})
  1099  		}
  1100  
  1101  		path := fmt.Sprintf("/repos/o/r/issues/%d/comments", issueNum)
  1102  		mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
  1103  			if r.Method == "POST" {
  1104  				c := new(github.IssueComment)
  1105  				json.NewDecoder(r.Body).Decode(c)
  1106  				msg := *c.Body
  1107  				if strings.HasPrefix(msg, "/test all") {
  1108  					go fakeRunGithubE2ESuccess(test.ciStatus, test.retest1Pass, test.retest2Pass)
  1109  				}
  1110  				w.WriteHeader(http.StatusOK)
  1111  				data, err := json.Marshal(github.IssueComment{})
  1112  				if err != nil {
  1113  					t.Errorf("Unexpected error: %v", err)
  1114  				}
  1115  				w.Write(data)
  1116  				return
  1117  			}
  1118  			if r.Method == "GET" {
  1119  				w.WriteHeader(http.StatusOK)
  1120  				data, err := json.Marshal([]github.IssueComment{})
  1121  				if err != nil {
  1122  					t.Errorf("Unexpected error: %v", err)
  1123  				}
  1124  				w.Write(data)
  1125  				return
  1126  			}
  1127  			t.Errorf("Unexpected method: %s", r.Method)
  1128  		})
  1129  		path = fmt.Sprintf("/repos/o/r/pulls/%d/merge", issueNum)
  1130  		mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
  1131  			if r.Method != "PUT" {
  1132  				t.Errorf("Unexpected method: %s", r.Method)
  1133  			}
  1134  			w.WriteHeader(http.StatusOK)
  1135  			data, err := json.Marshal(github.PullRequestMergeResult{})
  1136  			if err != nil {
  1137  				t.Errorf("Unexpected error: %v", err)
  1138  			}
  1139  			w.Write(data)
  1140  			test.pr.Merged = boolPtr(true)
  1141  			wasMerged = true
  1142  		})
  1143  		path = fmt.Sprintf("/repos/o/r/statuses/%s", *test.pr.Head.SHA)
  1144  		mux.HandleFunc(path, func(w http.ResponseWriter, r *http.Request) {
  1145  			if r.Method != "POST" {
  1146  				t.Errorf("Unexpected method: %s", r.Method)
  1147  			}
  1148  			decoder := json.NewDecoder(r.Body)
  1149  			var status github.RepoStatus
  1150  			err := decoder.Decode(&status)
  1151  			if err != nil {
  1152  				t.Errorf("Unable to decode status: %v", err)
  1153  			}
  1154  
  1155  			stateSet = *status.State
  1156  
  1157  			data, err := json.Marshal(status)
  1158  			if err != nil {
  1159  				t.Errorf("Unexpected error: %v", err)
  1160  			}
  1161  			w.WriteHeader(http.StatusOK)
  1162  			w.Write(data)
  1163  		})
  1164  
  1165  		sq := getTestSQ(true, config, server)
  1166  		sq.setEmergencyMergeStop(test.emergencyMergeStop)
  1167  		sq.AdditionalRequiredLabels = test.additionalLabels
  1168  
  1169  		obj := github_util.NewTestObject(config, test.issue, test.pr, test.commits, test.events)
  1170  		if test.imBaseSHA != "" && test.imHeadSHA != "" {
  1171  			sq.interruptedObj = &submitQueueInterruptedObject{obj, test.imHeadSHA, test.imBaseSHA}
  1172  		}
  1173  		sq.Munge(obj)
  1174  		done := make(chan bool, 1)
  1175  		go func(done chan bool) {
  1176  			for {
  1177  				defer func() {
  1178  					if r := recover(); r != nil {
  1179  						t.Errorf("%d:%q panic'd likely writing to 'done' channel", testNum, test.name)
  1180  					}
  1181  				}()
  1182  
  1183  				reason := func() string {
  1184  					sq.Mutex.Lock()
  1185  					defer sq.Mutex.Unlock()
  1186  					return sq.prStatus[issueNumStr].Reason
  1187  				}
  1188  
  1189  				if reason() == test.reason {
  1190  					done <- true
  1191  					return
  1192  				}
  1193  				found := false
  1194  				sq.Lock()
  1195  				for _, status := range sq.statusHistory {
  1196  					if status.Reason == test.reason {
  1197  						found = true
  1198  						break
  1199  					}
  1200  				}
  1201  				sq.Unlock()
  1202  				if found {
  1203  					done <- true
  1204  					return
  1205  				}
  1206  				time.Sleep(1 * time.Millisecond)
  1207  			}
  1208  		}(done)
  1209  		select {
  1210  		case <-done:
  1211  		case <-time.After(2 * time.Second):
  1212  			t.Errorf("%d:%q timed out waiting expected reason=%q but got prStatus:%q history:%v", testNum, test.name, test.reason, sq.prStatus[issueNumStr].Reason, sq.statusHistory)
  1213  		}
  1214  		close(done)
  1215  		server.Close()
  1216  
  1217  		if test.state != "" && test.state != stateSet {
  1218  			t.Errorf("%d:%q state set to %q but expected %q", testNum, test.name, stateSet, test.state)
  1219  		}
  1220  		if test.isMerged != wasMerged {
  1221  			t.Errorf("%d:%q PR merged = %v but wanted %v", testNum, test.name, wasMerged, test.isMerged)
  1222  		}
  1223  		if e, a := test.retestsAvoided, int(sq.retestsAvoided); e != a {
  1224  			t.Errorf("%d:%q expected %v tests avoided but got %v", testNum, test.name, e, a)
  1225  		}
  1226  	}
  1227  }
  1228  
  1229  func TestCalcMergeRate(t *testing.T) {
  1230  	runtime.GOMAXPROCS(runtime.NumCPU())
  1231  
  1232  	tests := []struct {
  1233  		name     string // because when it fails, counting is hard
  1234  		preRate  float64
  1235  		interval time.Duration
  1236  		expected func(float64) bool
  1237  	}{
  1238  		{
  1239  			name:     "0One",
  1240  			preRate:  0,
  1241  			interval: time.Duration(time.Hour),
  1242  			expected: func(rate float64) bool {
  1243  				return rate > 10 && rate < 11
  1244  			},
  1245  		},
  1246  		{
  1247  			name:     "24One",
  1248  			preRate:  24,
  1249  			interval: time.Duration(time.Hour),
  1250  			expected: func(rate float64) bool {
  1251  				return rate == float64(24)
  1252  			},
  1253  		},
  1254  		{
  1255  			name:     "24Two",
  1256  			preRate:  24,
  1257  			interval: time.Duration(2 * time.Hour),
  1258  			expected: func(rate float64) bool {
  1259  				return rate > 17 && rate < 18
  1260  			},
  1261  		},
  1262  		{
  1263  			name:     "24HalfHour",
  1264  			preRate:  24,
  1265  			interval: time.Duration(time.Hour) / 2,
  1266  			expected: func(rate float64) bool {
  1267  				return rate > 31 && rate < 32
  1268  			},
  1269  		},
  1270  		{
  1271  			name:     "24Three",
  1272  			preRate:  24,
  1273  			interval: time.Duration(3 * time.Hour),
  1274  			expected: func(rate float64) bool {
  1275  				return rate > 14 && rate < 15
  1276  			},
  1277  		},
  1278  		{
  1279  			name:     "24Then24",
  1280  			preRate:  24,
  1281  			interval: time.Duration(24 * time.Hour),
  1282  			expected: func(rate float64) bool {
  1283  				return rate > 2 && rate < 3
  1284  			},
  1285  		},
  1286  		{
  1287  			name:     "24fast",
  1288  			preRate:  24,
  1289  			interval: time.Duration(4 * time.Minute),
  1290  			expected: func(rate float64) bool {
  1291  				// Should be no change
  1292  				return rate == 24
  1293  			},
  1294  		},
  1295  	}
  1296  	for testNum, test := range tests {
  1297  		sq := getTestSQ(false, nil, nil)
  1298  		clock := sq.clock.(*utilclock.FakeClock)
  1299  		sq.mergeRate = test.preRate
  1300  		clock.Step(test.interval)
  1301  		sq.updateMergeRate()
  1302  		if !test.expected(sq.mergeRate) {
  1303  			t.Errorf("%d:%s: expected() failed: rate:%v", testNum, test.name, sq.mergeRate)
  1304  		}
  1305  	}
  1306  }
  1307  
  1308  func TestCalcMergeRateWithTail(t *testing.T) {
  1309  	runtime.GOMAXPROCS(runtime.NumCPU())
  1310  
  1311  	tests := []struct {
  1312  		name     string // because when it fails, counting is hard
  1313  		preRate  float64
  1314  		interval time.Duration
  1315  		expected func(float64) bool
  1316  	}{
  1317  		{
  1318  			name:     "ZeroPlusZero",
  1319  			preRate:  0,
  1320  			interval: time.Duration(0),
  1321  			expected: func(rate float64) bool {
  1322  				return rate == float64(0)
  1323  			},
  1324  		},
  1325  		{
  1326  			name:     "0OneHour",
  1327  			preRate:  0,
  1328  			interval: time.Duration(time.Hour),
  1329  			expected: func(rate float64) bool {
  1330  				return rate == 0
  1331  			},
  1332  		},
  1333  		{
  1334  			name:     "TinyOneHour",
  1335  			preRate:  .001,
  1336  			interval: time.Duration(time.Hour),
  1337  			expected: func(rate float64) bool {
  1338  				return rate == .001
  1339  			},
  1340  		},
  1341  		{
  1342  			name:     "TwentyFourPlusHalfHour",
  1343  			preRate:  24,
  1344  			interval: time.Duration(time.Hour) / 2,
  1345  			expected: func(rate float64) bool {
  1346  				return rate == 24
  1347  			},
  1348  		},
  1349  		{
  1350  			name:     "TwentyFourPlusOneHour",
  1351  			preRate:  24,
  1352  			interval: time.Duration(time.Hour),
  1353  			expected: func(rate float64) bool {
  1354  				return rate == 24
  1355  			},
  1356  		},
  1357  		{
  1358  			name:     "TwentyFourPlusTwoHour",
  1359  			preRate:  24,
  1360  			interval: time.Duration(2 * time.Hour),
  1361  			expected: func(rate float64) bool {
  1362  				return rate > 17 && rate < 18
  1363  			},
  1364  		},
  1365  		{
  1366  			name:     "TwentyFourPlusFourHour",
  1367  			preRate:  24,
  1368  			interval: time.Duration(4 * time.Hour),
  1369  			expected: func(rate float64) bool {
  1370  				return rate > 12 && rate < 13
  1371  			},
  1372  		},
  1373  		{
  1374  			name:     "TwentyFourPlusTwentyFourHour",
  1375  			preRate:  24,
  1376  			interval: time.Duration(24 * time.Hour),
  1377  			expected: func(rate float64) bool {
  1378  				return rate > 2 && rate < 3
  1379  			},
  1380  		},
  1381  		{
  1382  			name:     "TwentyFourPlusTiny",
  1383  			preRate:  24,
  1384  			interval: time.Duration(time.Nanosecond),
  1385  			expected: func(rate float64) bool {
  1386  				return rate == 24
  1387  			},
  1388  		},
  1389  		{
  1390  			name:     "TwentyFourPlusHuge",
  1391  			preRate:  24,
  1392  			interval: time.Duration(1024 * time.Hour),
  1393  			expected: func(rate float64) bool {
  1394  				return rate > 0 && rate < 1
  1395  			},
  1396  		},
  1397  	}
  1398  	for testNum, test := range tests {
  1399  		sq := getTestSQ(false, nil, nil)
  1400  		sq.mergeRate = test.preRate
  1401  		clock := sq.clock.(*utilclock.FakeClock)
  1402  		clock.Step(test.interval)
  1403  		rate := sq.calcMergeRateWithTail()
  1404  		if !test.expected(rate) {
  1405  			t.Errorf("%d:%s: %v", testNum, test.name, rate)
  1406  		}
  1407  	}
  1408  }
  1409  
  1410  func TestHealth(t *testing.T) {
  1411  	sq := getTestSQ(false, nil, nil)
  1412  	sq.updateHealth()
  1413  	sq.updateHealth()
  1414  	if len(sq.healthHistory) != 2 {
  1415  		t.Errorf("Wrong length healthHistory after calling updateHealth: %v", sq.healthHistory)
  1416  	}
  1417  	if sq.health.TotalLoops != 2 || sq.health.NumStable != 2 || len(sq.health.NumStablePerJob) != 2 {
  1418  		t.Errorf("Wrong number of stable loops after calling updateHealth: %v", sq.health)
  1419  	}
  1420  	for _, stable := range sq.health.NumStablePerJob {
  1421  		if stable != 2 {
  1422  			t.Errorf("Wrong number of stable loops for a job: %v", sq.health.NumStablePerJob)
  1423  		}
  1424  	}
  1425  	sq.healthHistory[0].Time = time.Now().AddDate(0, 0, -3)
  1426  	sq.healthHistory[1].Time = time.Now().AddDate(0, 0, -2)
  1427  	sq.updateHealth()
  1428  	if len(sq.healthHistory) != 1 {
  1429  		t.Errorf("updateHealth didn't truncate old entries: %v", sq.healthHistory)
  1430  	}
  1431  }
  1432  
  1433  func TestHealthSVG(t *testing.T) {
  1434  	sq := getTestSQ(false, nil, nil)
  1435  	e2e := sq.e2e.(*fake_e2e.FakeE2ETester)
  1436  
  1437  	for _, state := range []struct {
  1438  		mergePossible bool
  1439  		expected      string
  1440  		notStable     []string
  1441  	}{
  1442  		{true, "running", nil},
  1443  		{false, "blocked</text>", nil},
  1444  		{false, "blocked by kubemark-500", []string{"kubernetes-kubemark-500"}},
  1445  		{false, "blocked by a, b, c, ...", []string{"a", "b", "c", "d"}},
  1446  	} {
  1447  		sq.health.MergePossibleNow = state.mergePossible
  1448  		e2e.NotStableJobNames = state.notStable
  1449  		res := string(sq.getHealthSVG())
  1450  		if !strings.Contains(res, state.expected) {
  1451  			t.Errorf("SVG doesn't contain `%s`: %v", state.expected, res)
  1452  		}
  1453  	}
  1454  }