github.com/grumpyhome/grumpy@v0.3.1-0.20201208125205-7b775405bdf1/grumpy-tools-src/grumpy_tools/compiler/imputil.py (about)

     1  # coding=utf-8
     2  
     3  # Copyright 2016 Google Inc. All Rights Reserved.
     4  #
     5  # Licensed under the Apache License, Version 2.0 (the "License");
     6  # you may not use this file except in compliance with the License.
     7  # You may obtain a copy of the License at
     8  #
     9  #     http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  
    17  """Functionality for importing modules in Grumpy."""
    18  
    19  
    20  from __future__ import unicode_literals
    21  
    22  import collections
    23  import functools
    24  import logging
    25  import os
    26  import os.path
    27  import sys
    28  import sysconfig
    29  from pkg_resources import resource_filename, Requirement, DistributionNotFound
    30  
    31  try:
    32      from functools import lru_cache
    33  except ImportError:
    34      from backports.functools_lru_cache import lru_cache
    35  
    36  from grumpy_tools.compiler import util, parser
    37  import pythonparser
    38  from pythonparser import algorithm
    39  from pythonparser import ast
    40  
    41  try:
    42    xrange          # Python 2
    43  except NameError:
    44    xrange = range  # Python 3
    45  
    46  logger = logging.getLogger(__name__)
    47  
    48  
    49  def _get_grumpy_stdlib():
    50      try:
    51          runtime_gopath = resource_filename(
    52              Requirement.parse('grumpy-runtime'),
    53              'grumpy_runtime/data/gopath',
    54          )
    55      except DistributionNotFound:
    56          return None
    57      return os.path.join(os.path.sep, runtime_gopath, 'src/__python__')
    58  
    59  _GRUMPY_STDLIB_PATH = _get_grumpy_stdlib()
    60  
    61  _CPYTHON_STDLIB_PATHS = (
    62    [sysconfig.get_path('platstdlib') + sysconfig.get_path('stdlib')]
    63    + sysconfig.get_config_vars('LIBDEST', 'DESTLIB', 'BINLIBDEST')
    64  )
    65  
    66  _NATIVE_MODULE_PREFIX = '__go__/'
    67  
    68  
    69  class Import(object):
    70    """Represents a single module import and all its associated bindings.
    71  
    72    Each import pertains to a single module that is imported. Thus one import
    73    statement may produce multiple Import objects. E.g. "import foo, bar" makes
    74    an Import object for module foo and another one for module bar.
    75    """
    76  
    77    Binding = collections.namedtuple('Binding', ('bind_type', 'alias', 'value'))
    78  
    79    MODULE = "<BindType 'module'>"
    80    MEMBER = "<BindType 'member'>"
    81    STAR = "<BindType 'star'>"
    82  
    83    def __init__(self, name, script=None, is_native=False):
    84      self.name = name
    85      self.script = script
    86      self.is_native = is_native
    87      self.bindings = []
    88  
    89    def __repr__(self):
    90      if self.bindings:
    91        bind_type = self.bindings[0].bind_type
    92        if bind_type == Import.MODULE:
    93          bind_type = 'MODULE'
    94        elif bind_type == Import.MEMBER:
    95          bind_type = 'MEMBER'
    96        elif bind_type == Import.STAR:
    97          bind_type = 'STAR'
    98      else:
    99        bind_type = 'NONE'
   100      repr_ = '<Import %s:%s>' % (self.name, bind_type.lower())
   101      return repr_
   102  
   103    def add_binding(self, bind_type, alias, value):
   104      self.bindings.append(Import.Binding(bind_type, alias, value))
   105  
   106  
   107  class Importer(algorithm.Visitor):
   108    """Visits import nodes and produces corresponding Import objects."""
   109  
   110    # pylint: disable=invalid-name,missing-docstring,no-init
   111  
   112    def __init__(self, gopath, modname, script, absolute_import, package_dir=''):
   113      self.script = script
   114      self.pathdirs = []
   115      if gopath:
   116        self.pathdirs.extend(os.path.join(d, 'src', '__python__')
   117                             for d in gopath.split(os.pathsep))
   118      self.pathdirs.extend(sys.path)
   119      dirname, basename = os.path.split(script)
   120      self.package_dir = package_dir or dirname
   121  
   122      if basename == '__init__.py':
   123        self.package_name = modname
   124      elif (modname.find('.') != -1 and
   125            os.path.isfile(os.path.join(dirname, '__init__.py'))):
   126        self.package_name = modname[:modname.rfind('.')]
   127      else:
   128        self.package_name = ''
   129      self.absolute_import = absolute_import
   130  
   131    def generic_visit(self, node):
   132      raise ValueError('Import cannot visit {} node'.format(type(node).__name__))
   133  
   134    def visit_Import(self, node):
   135      imports = []
   136      for alias in node.names:
   137        if alias.name.startswith(_NATIVE_MODULE_PREFIX):
   138          imp = Import(alias.name, is_native=True)
   139          asname = alias.asname if alias.asname else alias.name.split('/')[-1]
   140          imp.add_binding(Import.MODULE, asname, 0)
   141        else:
   142          imp = self._resolve_import(node, alias.name, allow_error=True)
   143  
   144          if alias.asname:
   145            imp.add_binding(Import.MODULE, alias.asname, imp.name.count('.'))
   146          else:
   147            parts = alias.name.split('.')
   148            imp.add_binding(Import.MODULE, parts[0],
   149                            imp.name.count('.') - len(parts) + 1)
   150        imports.append(imp)
   151      return imports
   152  
   153    def visit_ImportFrom(self, node):
   154      if any(a.name == '*' for a in node.names):
   155        if len(node.names) != 1:
   156          # TODO: Change to SyntaxError, as CPython does on "from foo import *, bar"
   157          raise util.ImportError(node, 'invalid syntax on wildcard import')
   158  
   159        # Imported name is * (star). Will bind __all__ the module contents.
   160        imp = self._resolve_import(node, node.module, allow_error=True)
   161        imp.add_binding(Import.STAR, '*', imp.name.count('.'))
   162        return [imp]
   163  
   164      if not node.level and node.module == '__future__':
   165        return []
   166  
   167      if not node.level and node.module.startswith(_NATIVE_MODULE_PREFIX):
   168        imp = Import(node.module, is_native=True)
   169        for alias in node.names:
   170          asname = alias.asname or alias.name
   171          imp.add_binding(Import.MEMBER, asname, alias.name)
   172        return [imp]
   173  
   174      imports = []
   175      member_imp = None
   176      for alias in node.names:
   177        asname = alias.asname or alias.name
   178        if node.level:
   179          resolver = functools.partial(self._resolve_relative_import, node.level)
   180        else:
   181          resolver = self._resolve_import
   182        try:
   183          if not node.module:
   184            modname = alias.name
   185          else:
   186            modname = '{}.{}'.format(node.module, alias.name)
   187          imp = resolver(node, modname)
   188        except (util.ImportError, AttributeError):
   189          # A member (not a submodule) is being imported, so bind it.
   190          if not member_imp:
   191            member_imp = resolver(node, node.module, allow_error=True)
   192            imports.append(member_imp)
   193          member_imp.add_binding(Import.MEMBER, asname, alias.name)
   194        else:
   195          # Imported name is a submodule within a package, so bind that module.
   196          imp.add_binding(Import.MODULE, asname, imp.name.count('.'))
   197          imports.append(imp)
   198      return imports
   199  
   200    def _resolve_import(self, node, modname, allow_error=False):
   201      if not self.absolute_import and self.package_dir:
   202        script = find_script(self.package_dir, modname)
   203        if script:
   204          return Import('.'.join((self.package_name, modname)).lstrip('.'), script)
   205      for dirname in self.pathdirs:
   206        script = find_script(dirname, modname)
   207        if script:
   208          return Import(modname, script)
   209      if allow_error:
   210        return Import(modname, '')
   211      raise util.ImportError(node, 'no such module: {} (script: {})'.format(modname, self.script))
   212  
   213    def _resolve_relative_import(self, level, node, modname, allow_error=False):
   214      if not self.package_dir:
   215        raise util.ImportError(node, 'attempted relative import in non-package')
   216      uplevel = level - 1
   217      if uplevel > self.package_name.count('.'):
   218        raise util.ImportError(
   219            node, 'attempted relative import beyond toplevel package')
   220      dirname = os.path.normpath(os.path.join(
   221          self.package_dir, *(['..'] * uplevel)))
   222      script = find_script(dirname, modname or '__init__')
   223      if not script and not allow_error:
   224        raise util.ImportError(node, 'no such module: {} (script: {})'.format(modname, self.script))
   225      parts = self.package_name.split('.')
   226      return Import('.'.join(parts[:len(parts)-uplevel] + ([modname] if modname else [])), script)
   227  
   228  
   229  class _ImportCollector(algorithm.Visitor):
   230  
   231    # pylint: disable=invalid-name
   232  
   233    def __init__(self, importer, future_node):
   234      self.importer = importer
   235      self.future_node = future_node
   236      self.imports = []
   237  
   238    def visit_Import(self, node):
   239      self.imports.extend(self.importer.visit(node))
   240  
   241    def visit_ImportFrom(self, node):
   242      if node.module == '__future__':
   243        if node != self.future_node:
   244          raise util.LateFutureError(node)
   245        return
   246      self.imports.extend(self.importer.visit(node))
   247  
   248  
   249  def collect_imports(modname, script, gopath, package_dir=''):
   250    parser.patch_pythonparser()
   251    with open(script) as py_file:
   252      py_contents = py_file.read()
   253    mod = pythonparser.parse(py_contents)
   254    future_node, future_features = parse_future_features(mod)
   255    importer = Importer(gopath, modname, script,
   256                        future_features.absolute_import, package_dir=package_dir)
   257    collector = _ImportCollector(importer, future_node)
   258    collector.visit(mod)
   259    return collector.imports
   260  
   261  
   262  def calculate_transitive_deps(modname, script, gopath):
   263    """Determines all modules that script transitively depends upon."""
   264    deps = set()
   265    def calc(modname, script):
   266      if modname in deps:
   267        return
   268      deps.add(modname)
   269      for imp in collect_imports(modname, script, gopath):
   270        if imp.is_native:
   271          deps.add(imp.name)
   272          continue
   273        parts = imp.name.split('.')
   274        calc(imp.name, imp.script)
   275        if len(parts) == 1:
   276          continue
   277        # For submodules, the parent packages are also deps.
   278        package_dir, filename = os.path.split(imp.script)
   279        if filename == '__init__.py':
   280          package_dir = os.path.dirname(package_dir)
   281        for i in xrange(len(parts) - 1, 0, -1):
   282          modname = '.'.join(parts[:i])
   283          script = os.path.join(package_dir, '__init__.py')
   284          calc(modname, script)
   285          package_dir = os.path.dirname(package_dir)
   286    calc(modname, script)
   287    deps.remove(modname)
   288    return deps
   289  
   290  
   291  @lru_cache()
   292  def find_script(dirname, name, main=False, use_grumpy_stdlib=True):
   293    if use_grumpy_stdlib and _GRUMPY_STDLIB_PATH and dirname in _CPYTHON_STDLIB_PATHS:
   294      # Grumpy stdlib have preference over CPython stdlib
   295      result = find_script(_GRUMPY_STDLIB_PATH, name, main=main, use_grumpy_stdlib=False)
   296      if result:
   297        logger.debug("Package '%s' is from Grumpy stdlib", name)
   298        return result
   299  
   300    prefix = os.path.join(dirname, name.replace('.', os.sep))
   301    script = prefix + '.py'
   302    if os.path.isfile(script):
   303      return script
   304    if main:
   305      script = os.path.join(prefix, '__main__.py')
   306      if os.path.isfile(script):
   307        return script
   308    script = os.path.join(prefix, '__init__.py')
   309    if os.path.isfile(script):
   310      return script
   311    return None
   312  
   313  
   314  _FUTURE_FEATURES = (
   315      'absolute_import',
   316      'division',
   317      'print_function',
   318      'unicode_literals',
   319  )
   320  
   321  _IMPLEMENTED_FUTURE_FEATURES = (
   322      'absolute_import',
   323      'print_function',
   324      'unicode_literals'
   325  )
   326  
   327  # These future features are already in the language proper as of 2.6, so
   328  # importing them via __future__ has no effect.
   329  _REDUNDANT_FUTURE_FEATURES = ('generators', 'with_statement', 'nested_scopes')
   330  
   331  
   332  class FutureFeatures(object):
   333    """Spec for future feature flags imported by a module."""
   334  
   335    def __init__(self, absolute_import=False, division=False,
   336                 print_function=False, unicode_literals=False):
   337      self.absolute_import = absolute_import
   338      self.division = division
   339      self.print_function = print_function
   340      self.unicode_literals = unicode_literals
   341  
   342  
   343  def _make_future_features(node):
   344    """Processes a future import statement, returning set of flags it defines."""
   345    assert isinstance(node, ast.ImportFrom)
   346    assert node.module == '__future__'
   347    features = FutureFeatures()
   348    for alias in node.names:
   349      name = alias.name
   350      if name in _FUTURE_FEATURES:
   351        if name not in _IMPLEMENTED_FUTURE_FEATURES:
   352          msg = 'future feature {} not yet implemented by grumpy'.format(name)
   353          raise util.ParseError(node, msg)
   354        setattr(features, name, True)
   355      elif name == 'braces':
   356        raise util.ParseError(node, 'not a chance')
   357      elif name not in _REDUNDANT_FUTURE_FEATURES:
   358        msg = 'future feature {} is not defined'.format(name)
   359        raise util.ParseError(node, msg)
   360    return features
   361  
   362  
   363  def parse_future_features(mod):
   364    """Accumulates a set of flags for the compiler __future__ imports."""
   365    assert isinstance(mod, ast.Module)
   366    found_docstring = False
   367    for node in mod.body:
   368      if isinstance(node, ast.ImportFrom):
   369        if node.module == '__future__':
   370          return node, _make_future_features(node)
   371        break
   372      elif isinstance(node, ast.Expr) and not found_docstring:
   373        if not isinstance(node.value, ast.Str):
   374          break
   375        found_docstring = True
   376      else:
   377        break
   378    return None, FutureFeatures()