github.com/status-im/status-go@v1.1.0/_assets/scripts/test_stats.py (about)

     1  #!/usr/bin/env python
     2  
     3  import glob
     4  import xml.etree.ElementTree as ET
     5  from collections import defaultdict
     6  import re
     7  
     8  test_stats = defaultdict(lambda: defaultdict(int))
     9  skipped_tests = {}  # Use a dictionary to store test names and their skip reasons
    10  
    11  file_path = "**/report_*.xml"
    12  
    13  for file in glob.glob(file_path, recursive=True):
    14      tree = ET.parse(file)
    15      root = tree.getroot()
    16      for testcase in root.iter("testcase"):
    17          test_name = testcase.attrib["name"]
    18  
    19          test_stats[test_name]["total_runs"] += 1
    20  
    21          if testcase.find("failure") is not None:
    22              test_stats[test_name]["failed_runs"] += 1
    23          elif testcase.find("error") is not None:
    24              test_stats[test_name]["failed_runs"] += 1
    25  
    26          # Check for skipped tests
    27          skipped_element = testcase.find("skipped")
    28          if skipped_element is not None:
    29              message = skipped_element.attrib.get("message", "")
    30              # Extract the real reason from the message
    31              match = re.search(r': (.*?)\s*--- SKIP', message)
    32              skip_reason = match.group(1).strip() if match else "unknown reason"
    33              skipped_tests[test_name] = skip_reason  # Store test name and skip reason
    34  
    35  # Filter out root test cases if they have subtests
    36  filtered_test_stats = {
    37      name: stats for name, stats in test_stats.items()
    38      if not any(subtest.startswith(name + "/") for subtest in test_stats)
    39  }
    40  
    41  failing_test_stats = [
    42      {
    43          "name": name,
    44          "failure_rate": stats["failed_runs"] / stats["total_runs"],
    45          "failed_runs": stats["failed_runs"],
    46          "total_runs": stats["total_runs"]
    47      }
    48      for name, stats in filtered_test_stats.items() if stats["failed_runs"] != 0
    49  ]
    50  
    51  sorted_failing_test_stats = sorted(failing_test_stats,
    52                                     key=lambda x: x["failure_rate"],
    53                                     reverse=True)
    54  
    55  flaky_skipped_count = sum(1 for reason in skipped_tests.values() if reason == "flaky test")
    56  
    57  print("---")
    58  print(f"Failing tests stats (total: {len(failing_test_stats)})")
    59  print("---")
    60  for test_stat in sorted_failing_test_stats:
    61      print("{}: {:.1f}% ({} of {} failed)".format(
    62          test_stat['name'],
    63          test_stat['failure_rate'] * 100,
    64          test_stat['failed_runs'],
    65          test_stat['total_runs']
    66      ))
    67  
    68  print("---")
    69  print(f"Skipped tests (total: {len(skipped_tests)}, skipped as flaky: {flaky_skipped_count})")
    70  print("---")
    71  for test_name, skip_reason in skipped_tests.items():
    72      print(f"{test_name}: {skip_reason}")