github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/coders/coder_impl.pxd (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  # cython: profile=True
    19  
    20  cimport cython
    21  
    22  cimport cpython.ref
    23  cimport cpython.tuple
    24  cimport libc.stdint
    25  cimport libc.stdlib
    26  cimport libc.string
    27  
    28  cdef extern from "math.h":
    29    libc.stdint.int64_t abs "llabs"(libc.stdint.int64_t)
    30  
    31  ctypedef char* char_ptr
    32  
    33  from .stream cimport InputStream, OutputStream
    34  from apache_beam.utils cimport windowed_value
    35  
    36  
    37  cdef object loads, dumps, create_InputStream, create_OutputStream, ByteCountingOutputStream, get_varint_size, past_unicode
    38  # Temporarily untyped to allow monkeypatching on failed import.
    39  #cdef type WindowedValue
    40  
    41  cdef bint is_compiled
    42  
    43  cdef class CoderImpl(object):
    44    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
    45    cpdef decode_from_stream(self, InputStream stream, bint nested)
    46    cpdef bytes encode(self, value)
    47    cpdef decode(self, bytes encoded)
    48    cpdef bytes encode_nested(self, value)
    49    cpdef decode_nested(self, bytes encoded)
    50    cpdef estimate_size(self, value, bint nested=?)
    51    @cython.locals(varint_size=int, bits=libc.stdint.uint64_t)
    52    @cython.overflowcheck(False)
    53    cpdef int _get_nested_size(self, int inner_size, bint nested)
    54    cpdef get_estimated_size_and_observables(self, value, bint nested=?)
    55  
    56  
    57  cdef class SimpleCoderImpl(CoderImpl):
    58    pass
    59  
    60  
    61  cdef class StreamCoderImpl(CoderImpl):
    62    pass
    63  
    64  
    65  cdef class CallbackCoderImpl(CoderImpl):
    66    cdef object _encoder
    67    cdef object _decoder
    68    cdef object _size_estimator
    69  
    70  
    71  cdef object NoneType
    72  cdef unsigned char UNKNOWN_TYPE, NONE_TYPE, INT_TYPE, FLOAT_TYPE, BOOL_TYPE
    73  cdef unsigned char BYTES_TYPE, UNICODE_TYPE, LIST_TYPE, TUPLE_TYPE, DICT_TYPE
    74  cdef unsigned char SET_TYPE, ITERABLE_LIKE_TYPE
    75  cdef unsigned char PROTO_TYPE, DATACLASS_TYPE, NAMED_TUPLE_TYPE
    76  
    77  
    78  cdef set _ITERABLE_LIKE_TYPES
    79  
    80  cdef class FastPrimitivesCoderImpl(StreamCoderImpl):
    81    cdef CoderImpl fallback_coder_impl
    82    cdef CoderImpl iterable_coder_impl
    83    cdef object requires_deterministic_step_label
    84    cdef bint warn_deterministic_fallback
    85  
    86    @cython.locals(dict_value=dict, int_value=libc.stdint.int64_t,
    87                   unicode_value=unicode)
    88    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
    89    @cython.locals(t=int)
    90    cpdef decode_from_stream(self, InputStream stream, bint nested)
    91    cdef encode_special_deterministic(self, value, OutputStream stream)
    92    cdef encode_type(self, t, OutputStream stream)
    93    cdef decode_type(self, InputStream stream)
    94  
    95  cdef dict _unpickled_types
    96  
    97  
    98  cdef class MapCoderImpl(StreamCoderImpl):
    99    cdef CoderImpl _key_coder
   100    cdef CoderImpl _value_coder
   101    cdef bint _is_deterministic
   102  
   103  
   104  cdef class BytesCoderImpl(CoderImpl):
   105    pass
   106  
   107  
   108  cdef class BooleanCoderImpl(CoderImpl):
   109    pass
   110  
   111  
   112  cdef class BigEndianShortCoderImpl(StreamCoderImpl):
   113    pass
   114  
   115  
   116  cdef class SinglePrecisionFloatCoderImpl(StreamCoderImpl):
   117    pass
   118  
   119  
   120  cdef class FloatCoderImpl(StreamCoderImpl):
   121    pass
   122  
   123  
   124  cdef class TimestampCoderImpl(StreamCoderImpl):
   125    cdef object timestamp_class
   126  
   127  
   128  cdef list small_ints
   129  cdef class VarIntCoderImpl(StreamCoderImpl):
   130    @cython.locals(ivalue=libc.stdint.int64_t)
   131    cpdef bytes encode(self, value)
   132  
   133  
   134  cdef class SingletonCoderImpl(CoderImpl):
   135    cdef object _value
   136  
   137  
   138  cdef class AbstractComponentCoderImpl(StreamCoderImpl):
   139    cdef tuple _coder_impls
   140  
   141    cpdef _extract_components(self, value)
   142    cpdef _construct_from_components(self, components)
   143  
   144    @cython.locals(c=CoderImpl)
   145    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
   146    @cython.locals(c=CoderImpl)
   147    cpdef decode_from_stream(self, InputStream stream, bint nested)
   148  
   149    @cython.locals(c=CoderImpl)
   150    cpdef get_estimated_size_and_observables(self, value, bint nested=?)
   151  
   152  
   153  cdef class TupleCoderImpl(AbstractComponentCoderImpl):
   154    pass
   155  
   156  
   157  cdef class SequenceCoderImpl(StreamCoderImpl):
   158    cdef CoderImpl _elem_coder
   159    cdef object _read_state
   160    cdef object _write_state
   161    cdef int _write_state_threshold
   162  
   163    cpdef _construct_from_sequence(self, values)
   164  
   165    @cython.locals(buffer=OutputStream, target_buffer_size=libc.stdint.int64_t,
   166                   index=libc.stdint.int64_t, prev_index=libc.stdint.int64_t)
   167    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
   168  
   169  
   170  cdef class TupleSequenceCoderImpl(SequenceCoderImpl):
   171    pass
   172  
   173  
   174  cdef class _AbstractIterable:
   175    cdef object _contents
   176  
   177  
   178  cdef class IterableCoderImpl(SequenceCoderImpl):
   179    cdef bint _use_abstract_iterable
   180  
   181  
   182  cdef class ListCoderImpl(SequenceCoderImpl):
   183    pass
   184  
   185  
   186  cdef object IntervalWindow
   187  
   188  cdef class IntervalWindowCoderImpl(StreamCoderImpl):
   189    cdef libc.stdint.uint64_t _to_normal_time(self, libc.stdint.int64_t value)
   190    cdef libc.stdint.int64_t _from_normal_time(self, libc.stdint.uint64_t value)
   191  
   192    @cython.locals(typed_value=windowed_value._IntervalWindowBase,
   193                   span_millis=libc.stdint.int64_t)
   194    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
   195  
   196    @cython.locals(typed_value=windowed_value._IntervalWindowBase)
   197    cpdef decode_from_stream(self, InputStream stream, bint nested)
   198  
   199    @cython.locals(typed_value=windowed_value._IntervalWindowBase,
   200                   span_millis=libc.stdint.int64_t)
   201    cpdef estimate_size(self, value, bint nested=?)
   202  
   203  
   204  cdef int PaneInfoTiming_UNKNOWN
   205  cdef int PaneInfoEncoding_FIRST
   206  
   207  
   208  cdef class PaneInfoCoderImpl(StreamCoderImpl):
   209    cdef int _choose_encoding(self, windowed_value.PaneInfo value)
   210  
   211    @cython.locals(pane_info=windowed_value.PaneInfo, encoding_type=int)
   212    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
   213  
   214    @cython.locals(encoded_first_byte=int, encoding_type=int)
   215    cpdef decode_from_stream(self, InputStream stream, bint nested)
   216  
   217  
   218  cdef libc.stdint.uint64_t _TIME_SHIFT
   219  cdef libc.stdint.int64_t MIN_TIMESTAMP_micros
   220  cdef libc.stdint.int64_t MAX_TIMESTAMP_micros
   221  
   222  
   223  cdef class WindowedValueCoderImpl(StreamCoderImpl):
   224    """A coder for windowed values."""
   225    cdef CoderImpl _value_coder
   226    cdef CoderImpl _timestamp_coder
   227    cdef CoderImpl _windows_coder
   228    cdef CoderImpl _pane_info_coder
   229  
   230    cdef libc.stdint.uint64_t _to_normal_time(self, libc.stdint.int64_t value)
   231    cdef libc.stdint.int64_t _from_normal_time(self, libc.stdint.uint64_t value)
   232  
   233    @cython.locals(c=CoderImpl)
   234    cpdef get_estimated_size_and_observables(self, value, bint nested=?)
   235  
   236    @cython.locals(timestamp=libc.stdint.int64_t)
   237    cpdef decode_from_stream(self, InputStream stream, bint nested)
   238  
   239    @cython.locals(wv=windowed_value.WindowedValue, restore_sign=int)
   240    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
   241  
   242  
   243  cdef class ParamWindowedValueCoderImpl(WindowedValueCoderImpl):
   244    """A coder for windowed values with constant timestamp, windows and pane info."""
   245    cdef readonly libc.stdint.int64_t _timestamp
   246    cdef readonly object _windows
   247    cdef readonly windowed_value.PaneInfo _pane_info
   248  
   249  
   250  cdef class LengthPrefixCoderImpl(StreamCoderImpl):
   251    cdef CoderImpl _value_coder
   252  
   253  
   254  cdef class RowColumnEncoder:
   255    cdef bint encode_to_stream(self, size_t index, OutputStream stream) except -1
   256    cdef bint decode_from_stream(self, size_t index, InputStream stream) except -1
   257  
   258  
   259  cdef class GenericRowColumnEncoder(RowColumnEncoder):
   260    cdef object column
   261    cdef CoderImpl coder_impl
   262  
   263  
   264  cdef class RowCoderImpl(StreamCoderImpl):
   265    cdef object schema
   266    cdef int num_fields
   267    cdef list field_names
   268    cdef list field_nullable
   269    cdef object constructor
   270    cdef list encoding_positions
   271    cdef list encoding_positions_argsort
   272    cdef bint encoding_positions_are_trivial
   273    cdef list components
   274    cdef bint has_nullable_fields
   275  
   276    @cython.locals(i=int, nvals=libc.stdint.int64_t, running=int, component_coder=CoderImpl,
   277                   null_mask=bytes, null_mask_c=char_ptr)
   278    cpdef decode_from_stream(self, InputStream stream, bint nested)
   279  
   280    @cython.locals(i=int, nvals=libc.stdint.int64_t, k=size_t, n=size_t,
   281                   null_mask=bytes, null_mask_c=char_ptr)
   282    cpdef decode_batch_from_stream(self, dict dest, InputStream stream)
   283  
   284    @cython.locals(i=int, running=int, component_coder=CoderImpl)
   285    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
   286  
   287    @cython.locals(i=int, k=size_t, n=size_t,
   288                   null_flags=libc.stdint.uint8_t[:,::1],
   289                   null_bits=libc.stdint.uint8_t[:,::1],
   290                   has_null_bits=libc.stdint.uint8_t[::1])
   291    cpdef encode_batch_to_stream(self, dict values, OutputStream stream)
   292  
   293  
   294  cdef class LogicalTypeCoderImpl(StreamCoderImpl):
   295    cdef object logical_type
   296    cdef CoderImpl representation_coder
   297  
   298    cpdef decode_from_stream(self, InputStream stream, bint nested)
   299    cpdef encode_to_stream(self, value, OutputStream stream, bint nested)
   300  
   301  
   302  cdef class BigIntegerCoderImpl(StreamCoderImpl):
   303    pass
   304  
   305  
   306  cdef class DecimalCoderImpl(StreamCoderImpl):
   307    pass