go.chromium.org/luci@v0.0.0-20240309015107-7cdc2e660f33/analysis/internal/changepoints/bayesian/heuristic.go (about)

     1  // Copyright 2023 The LUCI Authors.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //      http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package bayesian
    16  
    17  // changePointHeuristic identifies points in a test variant history
    18  // which have the potential of being the most likely position for a
    19  // change point.
    20  //
    21  // This is used to avoid expensive statistical calculations evaluating
    22  // non-credible change point positions.
    23  //
    24  // This heuristic is only valid for the bayesian likelihood model
    25  // used in this file.
    26  //
    27  // This heuristic is consistent with the approach in the sense that
    28  // it will never eliminate positions which are the most likely position,
    29  // assuming that:
    30  //   - the most likely position is also more likely than the null hypothesis,
    31  //   - each change point position (other than the 'no change point') is
    32  //     equally likely.
    33  //
    34  // It may however identify positions that (upon detailed statistical
    35  // evaluation) are not a likely change point location.
    36  //
    37  // The heuristic works by identifying points in the history where
    38  // there is an expected-to-unexpected transition (or vice-versa).
    39  //
    40  // Usage (where positionCounts contains statistics for each commit
    41  // position, in order of commit position):
    42  //
    43  //	var heuristic changePointHeuristic
    44  //	heuristic.AddToHistory(positionCounts[0])
    45  //
    46  //	for i := 1; i < len(positionCounts); i++ {
    47  //		if heuristic.isPossibleChangepoint(positionCounts[i]) {
    48  //			// Consider a change point between i-1 and i in more detail.
    49  //		}
    50  //		heuristic.addToHistory(positionCounts[i])
    51  //	}
    52  type changePointHeuristic struct {
    53  	addedExpectedRuns         bool
    54  	addedUnexpectedRuns       bool
    55  	addedExpectedAfterRetry   bool
    56  	addedUnexpectedAfterRetry bool
    57  }
    58  
    59  // addToHistory updates the heuristic state to reflect the given
    60  // additional history. History should be input one commit position
    61  // at a time.
    62  func (h *changePointHeuristic) addToHistory(positionCounts counts) {
    63  	if positionCounts.Runs > 0 {
    64  		h.addedUnexpectedRuns = positionCounts.HasUnexpected > 0
    65  		h.addedExpectedRuns = (positionCounts.Runs - positionCounts.HasUnexpected) > 0
    66  	}
    67  	if positionCounts.Retried > 0 {
    68  		h.addedUnexpectedAfterRetry = positionCounts.UnexpectedAfterRetry > 0
    69  		h.addedExpectedAfterRetry = (positionCounts.Retried - positionCounts.UnexpectedAfterRetry) > 0
    70  	}
    71  }
    72  
    73  // isChangepointPossibleWithNext returns whether there could be a
    74  // change point between the history consumed so far on the left,
    75  // and the remainder of the history on the right.
    76  // nextPositionCounts is the counts of the next commit position
    77  // on the right.
    78  func (h changePointHeuristic) isChangepointPossibleWithNext(nextPositionCounts counts) bool {
    79  	consider := false
    80  	if nextPositionCounts.Runs > 0 {
    81  		// Detect expected-to-unexpected transition (or vica-versa).
    82  		addingUnexpectedRuns := nextPositionCounts.HasUnexpected > 0
    83  		addingExpectedRuns := (nextPositionCounts.Runs - nextPositionCounts.HasUnexpected) > 0
    84  		consider = consider || addingUnexpectedRuns && h.addedExpectedRuns
    85  		consider = consider || addingExpectedRuns && h.addedUnexpectedRuns
    86  	}
    87  	if nextPositionCounts.Retried > 0 {
    88  		// Detect expected-to-unexpected transition in the result
    89  		// after retries (or vica-versa).
    90  		addingUnexpectedAfterRetry := nextPositionCounts.UnexpectedAfterRetry > 0
    91  		addingExpectedAfterRetry := (nextPositionCounts.Retried - nextPositionCounts.UnexpectedAfterRetry) > 0
    92  		consider = consider || addingUnexpectedAfterRetry && h.addedExpectedAfterRetry
    93  		consider = consider || addingExpectedAfterRetry && h.addedUnexpectedAfterRetry
    94  	}
    95  	return consider
    96  }