github.com/replit/upm@v0.0.0-20240423230255-9ce4fc3ea24c/internal/backends/python/regression_tests/test.py (about)

     1  #!/usr/bin/env python3
     2  
     3  import json
     4  import subprocess
     5  import sys
     6  import os
     7  
     8  TEST_DIR = "test"
     9  
    10  def normalize_name(name):
    11      return name.replace('.', '-').replace('_', '-').lower()
    12  
    13  def load_json_file(filepath):
    14      file = open(filepath)
    15      data = json.load(file)
    16      file.close()
    17      return data
    18  
    19  def reverse_mapping(dct):
    20      retval = {}
    21      for key, value in dct.items():
    22          value = value.lower()
    23          if value in retval:
    24              retval[value].append(key)
    25          else:
    26              retval[value] = [key]
    27  
    28      return retval
    29  
    30  os.makedirs(TEST_DIR, exist_ok=True)
    31  module_to_pypi = load_json_file("module_to_pypi.legacy.json")
    32  pypi_to_module = reverse_mapping(module_to_pypi)
    33  
    34  gen_go_file = open('../pypi_map.gen.go')
    35  gen_go = gen_go_file.read()
    36  gen_go_file.close()
    37  
    38  pkgs_file = open('../pkgs.json')
    39  pkgs = {}
    40  for line in pkgs_file:
    41      info = json.loads(line)
    42      norm_name = normalize_name(info['name'])
    43      pkgs[norm_name] = {
    44          'name': info['name'],
    45          'error': info.get('error')
    46      }
    47  pkgs_file.close()
    48  
    49  skip_manual_checked = {
    50      'python-louvain': True,
    51      'pymilvus': True,
    52      'transformers': True,
    53      'configparser': True,
    54      'spark-nlp': True,
    55      'aws-sam-cli': True,
    56      'suds-jurko': True,
    57      'opsgenie-sdk': True,
    58      'dbutils': True,
    59      'biopython': True,
    60      'cmake-format': True,
    61      'mbed-tools': True,
    62      'fake-factory': True,
    63      'pysqlite3': True,
    64      'paddle2onnx': True,
    65      'django-extra-fields': True,
    66      'pyicu': True,
    67      'censys': True,
    68      'sip': True,
    69      'giturlparse': True,
    70      'filesplit': True,
    71      'pycausalimpact': True,
    72      'pip-autoremove': True,
    73      'demisto-py': True,
    74      'enos-mqtt-sdk-python': True,
    75      'cyclonedx-bom': True,
    76      'weather-api': True,
    77      'bio': True,
    78      'express': True,
    79      'django-meta': True,
    80      'frappe-bench': True,
    81      'python-chess': True,
    82      'optimuspyspark': True,
    83      'poetry': True,
    84      'pymatgen': True,
    85      'seeq': True,
    86      'pystan': True,
    87  }
    88  
    89  # override the module choice from the module_to_pypi.json file
    90  override = {
    91      'tableau-api-lib': 'tableau_api_lib',
    92      'wmd': 'wmd',
    93      "django-tinymce": "tinymce",
    94      "cmake-format": "cmakelang",
    95      "pytest-xprocess": "xprocess",
    96      "notifications-python-client": "notifications_python_client",
    97      "imdb-cli-tool": "imdb_cli_tool",
    98  }
    99  
   100  def test_package(pkg):
   101      if pkg in skip_manual_checked:
   102          print("skip %s" % pkg)
   103          return
   104      main_file = open(TEST_DIR + "/main.py", "w")
   105      if pkg not in pypi_to_module:
   106          if pkg in pkgs:
   107              info = pkgs[pkg]
   108              if info['error']:
   109                  print("%s test-errored" % pkg)
   110              else:
   111                  str_to_look = '"' + info['name'] + '",'
   112                  if str_to_look in gen_go:
   113                      print("%s added" % pkg)
   114                  else:
   115                      print("%s no-guess" % pkg)
   116          else:
   117              print("%s missing" % pkg)
   118  
   119          return
   120      if pkg in override:
   121          mod = override[pkg]
   122      else:
   123          mod = choose_module(pkg, pypi_to_module[pkg])
   124      main_file.write("import %s" % mod)
   125      main_file.close()
   126  
   127      proc1 = subprocess.run(["upm-old", "guess", "-f"], cwd=TEST_DIR, capture_output=True)
   128      proc2 = subprocess.run(["upm", "guess", "-f"], cwd=TEST_DIR, capture_output=True)
   129      if proc1.stdout == proc2.stdout:
   130          print("%s ok" % pkg)
   131      else:
   132          print("%s failed" % pkg)
   133          print("  Expected: %s" % proc1.stdout)
   134          print("  Actual: %s" % proc2.stdout)
   135  
   136  def choose_module(pkg, modules):
   137      best_score = 0
   138      choosen = None
   139      for mod in modules:
   140          if mod == pkg:
   141              return mod
   142          if mod.startswith("_"):
   143              score = 1
   144          else:
   145              score = 2
   146          if score > best_score:
   147              best_score = score
   148              choosen = mod
   149      return choosen
   150  
   151  if len(sys.argv) > 1:
   152      pkg = sys.argv[1]
   153      test_package(pkg)
   154  else:
   155      downloads_file = open('../download_stats.json')
   156      downloads = json.load(downloads_file)
   157      downloads = list(downloads.items())
   158      downloads.sort(key=lambda item: item[1], reverse=True)
   159      for pkg, _ in downloads[:10000]:
   160          test_package(pkg)
   161  
   162  
   163  
   164