github.com/yrj2011/jx-test-infra@v0.0.0-20190529031832-7a2065ee98eb/experiment/flakedetector.py (about)

     1  #!/usr/bin/env python3
     2  
     3  # Copyright 2017 The Kubernetes Authors.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  
    18  """Counts the number of flakes in PRs using data from prow.
    19  
    20  A flake is counted if a job passes and fails for the same pull commit. This
    21  isn't a perfect signal, since something might have happened on master that
    22  makes it flake, but I think it's good enough. There will also be false
    23  negatives when flakes don't show up because the PR author changed the PR.
    24  Still, this is a good signal.
    25  
    26  Only serial jobs are considered for the flake calculation, batch jobs are
    27  ignored.
    28  
    29  """
    30  
    31  from __future__ import print_function
    32  
    33  import operator
    34  
    35  import requests
    36  
    37  def main(): # pylint: disable=too-many-branches
    38      """Run flake detector."""
    39      res = requests.get('https://prow.k8s.io/data.js')
    40      job_results = res.json()
    41  
    42      jobs = {} # job -> {sha -> [results...]}
    43      commits = {} # sha -> {job -> [results...]}
    44      for res in job_results:
    45          if res['type'] != 'presubmit':
    46              continue
    47          if res['repo'] != 'kubernetes/kubernetes':
    48              continue
    49          if res['state'] != 'success' and res['state'] != 'failure':
    50              continue
    51          # populate jobs
    52          if res['job'] not in jobs:
    53              jobs[res['job']] = {}
    54          if res['pull_sha'] not in jobs[res['job']]:
    55              jobs[res['job']][res['pull_sha']] = []
    56          jobs[res['job']][res['pull_sha']].append(res['state'])
    57          # populate commits
    58          if res['pull_sha'] not in commits:
    59              commits[res['pull_sha']] = {}
    60          if res['job'] not in commits[res['pull_sha']]:
    61              commits[res['pull_sha']][res['job']] = []
    62          commits[res['pull_sha']][res['job']].append(res['state'])
    63  
    64      job_commits = {}
    65      job_flakes = {}
    66      for job, shas in jobs.items():
    67          job_commits[job] = len(shas)
    68          job_flakes[job] = 0
    69          for results in shas.values():
    70              if 'success' in results and 'failure' in results:
    71                  job_flakes[job] += 1
    72  
    73      print('Certain flakes:')
    74      for job, flakes in sorted(job_flakes.items(), key=operator.itemgetter(1), reverse=True):
    75          if job_commits[job] < 10:
    76              continue
    77          fail_chance = flakes / job_commits[job]
    78          print('{}/{}\t({:.0f}%)\t{}'.format(flakes, job_commits[job], 100*fail_chance, job))
    79  
    80      # for each commit, flaked iff exists job that flaked
    81      flaked = 0
    82      for _, job_results in commits.items():
    83          for job, results in job_results.items():
    84              if 'success' in results and 'failure' in results:
    85                  flaked += 1
    86                  break
    87      print('Commits that flaked (passed and failed some job): %d/%d  %.2f%%' %
    88            (flaked, len(commits), (flaked*100.0)/len(commits)))
    89  
    90  
    91  if __name__ == '__main__':
    92      main()