k8s.io/test-infra@v0.0.0-20240520184403-27c6b4c223d8/metrics/configs/pr-consistency-config.yaml (about)

     1  metric: pr-consistency
     2  description: Calculate PR flakiness for each day.
     3  query: |
     4    select /* Calculate the prob a commit flaked on at least one PR job at some point. + prob that a `/test all` command flakes. */
     5      timestamp_to_sec(day) day,
     6      round(sum(consistent)/count(consistent),4) commit_consistency,
     7      sum(consistent) consistent_commits,
     8      count(consistent) commits,
     9      round(sum(consistent_runs)/sum(runs), 4) run_consistency, /* should be approx. the nth root of consistency if there are n PR jobs and assumptions hold (job flake independence over the same commit). */
    10      sum(consistent_runs) consistent_runs,
    11      sum(runs) runs,
    12      sum(builds) builds
    13    from ( /* For each day, count whether a (commit) flaked */
    14      select
    15        day,
    16        commit,
    17        min(if(passed == runs or passed == 0, 1, 0)) consistent,
    18        max(runs) builds,
    19        sum(runs) runs,
    20        sum(if(passed==0, runs, passed)) consistent_runs
    21      from (
    22        select /* For each day, count runs and passes for a (job, commit) */
    23          cast(day as timestamp) day,
    24          commit,
    25          sum(if(result=='SUCCESS',1,0)) passed,
    26          INTEGER(count(result)) runs
    27        from (
    28          SELECT /* Find jobs, noting its commit, day it ran, and whether it passed */
    29            job,
    30            regexp_extract(metadata.value, r'^[^,]+,\d+:([^,"]+)') commit, /* git hash of pr */
    31            result,
    32            cast(date(started) as date) day
    33          FROM [kubernetes-public:k8s_infra_kettle.all]
    34          where
    35            /* Select results that have occurred on or after the day of <LAST_DATA_TIME> (those results were
    36               for the day before), but don't include results from today, only return results for complete days. */
    37            cast(date(started) as date) >= cast(date(sec_to_timestamp(<LAST_DATA_TIME>)) as date)
    38            and cast(date(started) as date) < cast(current_date() as date) /* Drop results from partial day */
    39            and metadata.key = 'repos'
    40            and length(regexp_replace(replace(metadata.value, ", ", ""), r'[^,]+', '')) = 1
    41            and job in ( /* only consider merge-blocking jobs */
    42              'pr:pull-kubernetes-bazel-build',
    43              'pr:pull-kubernetes-bazel-test',
    44              'pr:pull-kubernetes-e2e-gce',
    45              'pr:pull-kubernetes-e2e-gce-100-performance',
    46              'pr:pull-kubernetes-integration',
    47              'pr:pull-kubernetes-kubemark-e2e-gce-big',
    48              'pr:pull-kubernetes-node-e2e',
    49              'pr:pull-kubernetes-typecheck',
    50              'pr:pull-kubernetes-verify',
    51            )
    52        )
    53        group by day, job, commit
    54      )
    55      group by day, commit
    56    )
    57    group by day
    58    order by day
    59  
    60  jqfilter: |
    61    ([(.[] | .day|tonumber)] | max) as $newestday |
    62    [(.[] | select((.day|tonumber)==$newestday) | {
    63      day: .day,
    64      commit_consistency: (.commit_consistency|tonumber),
    65      commits: (.commits|tonumber),
    66      consistent_commits: (.consistent_commits|tonumber),
    67      run_consistency: (.run_consistency|tonumber),
    68      runs: (.runs|tonumber),
    69      consistent_runs: (.consistent_runs|tonumber),
    70      builds: (.builds|tonumber)
    71    })]