github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/utils/python_callable.py (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  """Python Callable utilities.
    19  
    20  For internal use only; no backwards-compatibility guarantees.
    21  """
    22  
    23  import importlib
    24  
    25  
    26  class PythonCallableWithSource(object):
    27    """Represents a Python callable object with source codes before evaluated.
    28  
    29    Proxy object to Store a callable object with its string form (source code).
    30    The string form is used when the object is encoded and transferred to foreign
    31    SDKs (non-Python SDKs).
    32  
    33    Supported formats include fully-qualified names such as `math.sin`,
    34    expressions such as `lambda x: x * x` or `str.upper`, and multi-line function
    35    definitions such as `def foo(x): ...` or class definitions like
    36    `class Foo(...): ...`. If the source string contains multiple lines then lines
    37    prior to the last will be evaluated to provide the context in which to
    38    evaluate the expression, for example::
    39  
    40        import math
    41  
    42        lambda x: x - math.sin(x)
    43  
    44    is a valid chunk of source code.
    45    """
    46    def __init__(self, source):
    47      # type: (str) -> None
    48      self._source = source
    49      self._callable = self.load_from_source(source)
    50  
    51    @classmethod
    52    def load_from_source(cls, source):
    53      if source in __builtins__:
    54        return cls.load_from_expression(source)
    55      elif all(s.isidentifier() for s in source.split('.')):
    56        if source.split('.')[0] in __builtins__:
    57          return cls.load_from_expression(source)
    58        else:
    59          return cls.load_from_fully_qualified_name(source)
    60      else:
    61        return cls.load_from_script(source)
    62  
    63    @staticmethod
    64    def load_from_expression(source):
    65      return eval(source)  # pylint: disable=eval-used
    66  
    67    @staticmethod
    68    def load_from_fully_qualified_name(fully_qualified_name):
    69      o = None
    70      path = ''
    71      for segment in fully_qualified_name.split('.'):
    72        path = '.'.join([path, segment]) if path else segment
    73        if o is not None and hasattr(o, segment):
    74          o = getattr(o, segment)
    75        else:
    76          o = importlib.import_module(path)
    77      return o
    78  
    79    @staticmethod
    80    def load_from_script(source):
    81      lines = [
    82          line for line in source.split('\n')
    83          if line.strip() and line.strip()[0] != '#'
    84      ]
    85      common_indent = min(len(line) - len(line.lstrip()) for line in lines)
    86      lines = [line[common_indent:] for line in lines]
    87  
    88      for ix, line in reversed(list(enumerate(lines))):
    89        if line[0] != ' ':
    90          if line.startswith('def '):
    91            name = line[4:line.index('(')].strip()
    92          elif line.startswith('class '):
    93            name = line[5:line.index('(') if '(' in
    94                        line else line.index(':')].strip()
    95          else:
    96            name = '__python_callable__'
    97            lines[ix] = name + ' = ' + line
    98          break
    99      else:
   100        raise ValueError("Unable to identify callable from %r" % source)
   101  
   102      # pylint: disable=exec-used
   103      # pylint: disable=ungrouped-imports
   104      import apache_beam as beam
   105      exec_globals = {'beam': beam}
   106      exec('\n'.join(lines), exec_globals)
   107      return exec_globals[name]
   108  
   109    def default_label(self):
   110      src = self._source.strip()
   111      last_line = src.split('\n')[-1]
   112      if last_line[0] != ' ' and len(last_line) < 72:
   113        return last_line
   114      # Avoid circular import.
   115      from apache_beam.transforms.ptransform import label_from_callable
   116      return label_from_callable(self._callable)
   117  
   118    @property
   119    def _argspec_fn(self):
   120      return self._callable
   121  
   122    def get_source(self):
   123      # type: () -> str
   124      return self._source
   125  
   126    def __call__(self, *args, **kwargs):
   127      return self._callable(*args, **kwargs)