github.com/grumpyhome/grumpy@v0.3.1-0.20201208125205-7b775405bdf1/grumpy-tools-src/grumpy_tools/pep_support/pep3147pycache.py (about)

     1  # coding: utf-8
     2  from __future__ import unicode_literals
     3  
     4  import os
     5  import sys
     6  import logging
     7  import hashlib
     8  import tempfile
     9  import warnings
    10  from backports.functools_lru_cache import lru_cache
    11  from backports.tempfile import TemporaryDirectory
    12  
    13  import importlib2
    14  import grumpy_tools
    15  
    16  from ..compiler import imputil
    17  
    18  logger = logging.getLogger(__name__)
    19  
    20  GOPATH_FOLDER = 'gopath'
    21  TRANSPILED_MODULES_FOLDER = 'src/__python__/'
    22  GRUMPY_MAGIC_TAG = 'grumpy-' + grumpy_tools.__version__.replace('.', '')  # alike cpython-27
    23  ORIGINAL_MAGIC_TAG = sys.implementation.cache_tag  # On Py27, only because importlib2
    24  
    25  # See: https://golang.org/ref/spec#Keywords
    26  _GO_RESERVED_KEYWORDS = {
    27    'break',        'default',      'func',         'interface',    'select',
    28    'case',         'defer',        'go',           'map',          'struct',
    29    'chan',         'else',         'goto',         'package',      'switch',
    30    'const',        'fallthrough',  'if',           'range',        'type',
    31    'continue',     'for',          'import',       'return',       'var',
    32  }.union(['main'])  # Found to be troublesome as module names
    33  
    34  _temporary_directories = []  # Will be cleaned up on main Python exit.
    35  
    36  
    37  class SilentTemporaryDirectory(TemporaryDirectory):
    38      '''TemporaryDirectory that does not warn on implicit cleanup'''
    39      @classmethod
    40      def _cleanup(cls, name, warn_message):
    41          logger.debug(warn_message)
    42          with warnings.catch_warnings():
    43              warnings.simplefilter("ignore")
    44              result = TemporaryDirectory._cleanup(name, warn_message)
    45          return result
    46  
    47  
    48  def get_depends_path(script_path, module_name):
    49      pycache_folder = get_pycache_folder(script_path, module_name)
    50      return os.path.join(pycache_folder, 'dependencies-%s.pkl' % module_name)
    51  
    52  
    53  def get_checksum_path(script_path, module_name):
    54      pycache_folder = get_pycache_folder(script_path, module_name)
    55      return os.path.join(pycache_folder, 'checksum-%s.sha1' % module_name)
    56  
    57  
    58  def get_checksum(stream):
    59      stream.seek(0)
    60      return hashlib.sha1(stream.read()).hexdigest()
    61  
    62  
    63  def set_checksum(stream, script_path, module_name):
    64      with open(get_checksum_path(script_path, module_name), 'w') as chk_file:
    65          chk_file.write(get_checksum(stream))
    66  
    67  
    68  def should_refresh(stream, script_path, modname):
    69      checksum_filename = get_checksum_path(script_path, modname)
    70      if not os.path.exists(checksum_filename):
    71          logger.debug("Should transpile '%s'", modname)
    72          return True
    73  
    74      with open(checksum_filename, 'r+') as checksum_file:
    75          existing_checksum = checksum_file.read()
    76  
    77      new_checksum = get_checksum(stream)
    78      if new_checksum != existing_checksum:
    79          logger.debug("Should refresh '%s' (%s)", modname, existing_checksum[:8])
    80          return True
    81  
    82      logger.debug("No need to refresh '%s' (%s)", modname, existing_checksum[:8])
    83      return False
    84  
    85  
    86  @lru_cache()
    87  def get_pycache_folder(script_path, module_name):
    88      """
    89      Gets the __pycache__ folder or PEP-3147
    90  
    91      Returns cache_folder path. Can be temporary.
    92      If so, will be cleaned automatically unless it is for __main__ module.
    93      """
    94      assert script_path.endswith('.py')
    95  
    96      if module_name == '__main__':
    97          cache_folder = tempfile.mkdtemp(suffix='__pycache__')  # Will be cleaned by grumprun
    98          logger.info("__main__ pycache folder: %s", cache_folder)
    99          return cache_folder
   100  
   101      ### TODO: Fix race conditions
   102      sys.implementation.cache_tag = GRUMPY_MAGIC_TAG
   103      cache_folder = os.path.abspath(os.path.normpath(
   104          importlib2._bootstrap.cache_from_source(script_path)
   105      ))
   106      sys.implementation.cache_tag = ORIGINAL_MAGIC_TAG
   107      ###
   108  
   109      first_existing = _get_first_existing_parent(cache_folder)
   110  
   111      if not os.access(first_existing, os.W_OK):
   112          cache_folder = SilentTemporaryDirectory(suffix='__pycache__')
   113          _temporary_directories.append(cache_folder)  # Hold GC until Python exits
   114          logger.info("Natural __pycache__ folder not available. Using %s", cache_folder.name)
   115          return cache_folder.name
   116  
   117      return cache_folder
   118  
   119  
   120  def _get_first_existing_parent(cache_folder):
   121      path_parts = cache_folder.split(os.path.sep)
   122      if path_parts[0] == '':  # From root.
   123          path_parts[0] = os.path.sep
   124  
   125      for i, _ in enumerate(path_parts):
   126          subpath = os.path.join(*path_parts[:(-i or None)])
   127          if os.path.exists(subpath):
   128              return subpath
   129  
   130  
   131  def get_gopath_folder(script_path, module_name):
   132      cache_folder = get_pycache_folder(script_path, module_name)
   133      return os.path.join(cache_folder, GOPATH_FOLDER)
   134  
   135  
   136  def get_transpiled_base_folder(script_path, module_name):
   137      gopath_folder = get_gopath_folder(script_path, module_name)
   138      return os.path.join(gopath_folder, TRANSPILED_MODULES_FOLDER)
   139  
   140  
   141  def get_transpiled_module_folder(script_path, module_name):
   142      module_name = fixed_keyword(module_name)
   143      transpiled_base_folder = get_transpiled_base_folder(script_path, module_name)
   144      parts = module_name.split('.')
   145      return os.path.join(transpiled_base_folder, *parts)
   146  
   147  
   148  def link_parent_modules(script_path, module_name):
   149      package_parts = module_name.split('.')[:-1]
   150      if not package_parts:
   151          return  # No parent packages to be linked
   152  
   153      script_parts = script_path.split(os.sep)
   154      if script_parts[-1] == '__init__.py':
   155          script_parts = script_parts[:-1]
   156      if script_parts[0] == '':
   157          script_parts[0] = '/'
   158      script_parts = script_parts[:-1]
   159  
   160      for i, part in enumerate(reversed(package_parts)):
   161          parent_script = os.path.join(*script_parts[:(-i or None)])
   162          parent_package = '.'.join(package_parts[:(-i or None)])
   163          parent_package_script = imputil.find_script(parent_package, parent_script)
   164          if not parent_package_script:
   165              continue
   166          parent_module_folder = get_transpiled_module_folder(parent_package_script, parent_package)
   167          local_parent_module_folder = get_transpiled_module_folder(script_path, parent_package)
   168  
   169          logger.debug("Checking link of package '%s' on %s",
   170                       parent_package, local_parent_module_folder)
   171          _maybe_link_paths(os.path.join(parent_module_folder, 'module.go'),
   172                            os.path.join(local_parent_module_folder, 'module.go'))
   173  
   174  
   175  def make_transpiled_module_folders(script_path, module_name):
   176      """
   177      Make the folder to store all the tree needed by the 'script_path' script
   178  
   179      Recursively "stomp" the files found in places that a folder is needed.
   180      """
   181      needed_folders = {
   182          'cache_folder': get_pycache_folder(script_path, module_name),
   183          'gopath_folder': get_gopath_folder(script_path, module_name),
   184          'transpiled_base_folder': get_transpiled_base_folder(script_path, module_name),
   185          'transpiled_module_folder': get_transpiled_module_folder(script_path, module_name),
   186      }
   187      for role, folder in needed_folders.items():
   188          if os.path.isfile(folder):  # 1. Need a folder. Remove the file
   189              os.unlink(folder)
   190          if not os.path.exists(folder):  # 2. Create the needed folder
   191              os.makedirs(folder)
   192  
   193      link_parent_modules(script_path, module_name)
   194  
   195      result = needed_folders.copy()
   196      result.update({
   197          'checksum_file': get_checksum_path(script_path, module_name),
   198          'dependencies_file': get_depends_path(script_path, module_name),
   199      })
   200      return result
   201  
   202  
   203  def _maybe_link_paths(orig, dest):
   204      relpath = os.path.relpath(orig, os.path.dirname(dest))
   205      if os.path.exists(dest) and not os.path.islink(dest):
   206          os.unlink(dest)
   207  
   208      if not os.path.exists(dest):
   209          try:
   210              os.symlink(relpath, dest)
   211          except OSError as err:  # Got created on an OS race condition?
   212              if 'exists' not in str(err):
   213                  raise
   214          else:
   215              logger.debug('Linked %s to %s', orig, dest)
   216              return True
   217      return False
   218  
   219  
   220  def fixed_keyword(keyword, split='.'):
   221      """
   222      Go have some reserved words that could be Python module names. This modules
   223      needs to be renamed at least on "naked" Go code, e.g. `package` definition
   224      """
   225      if split:
   226          keys = keyword.split(split)
   227      else:
   228          keys = [keyword]
   229  
   230      for i, kw in enumerate(keys):
   231          if kw in _GO_RESERVED_KEYWORDS:
   232              keys[i] += '_goreservedkeyword'
   233      return split.join(keys)