github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/runners/common.pxd (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  cimport cython
    19  
    20  from apache_beam.utils.windowed_value cimport WindowedValue
    21  from apache_beam.utils.windowed_value cimport WindowedBatch
    22  from apache_beam.transforms.cy_dataflow_distribution_counter cimport DataflowDistributionCounter
    23  
    24  from libc.stdint cimport int64_t
    25  
    26  
    27  cdef type TaggedOutput, TimestampedValue
    28  
    29  
    30  cdef class Receiver(object):
    31    cpdef receive(self, WindowedValue windowed_value)
    32    cpdef receive_batch(self, WindowedBatch windowed_batch)
    33    cpdef flush(self)
    34  
    35  
    36  cdef class MethodWrapper(object):
    37    cdef public object args
    38    cdef public object defaults
    39    cdef public object method_value
    40    cdef str method_name
    41    cdef bint has_userstate_arguments
    42    cdef object state_args_to_replace
    43    cdef object timer_args_to_replace
    44    cdef object timestamp_arg_name
    45    cdef object window_arg_name
    46    cdef object key_arg_name
    47    cdef object restriction_provider
    48    cdef object restriction_provider_arg_name
    49    cdef object watermark_estimator_provider
    50    cdef object watermark_estimator_provider_arg_name
    51    cdef object dynamic_timer_tag_arg_name
    52    cdef bint unbounded_per_element
    53  
    54  
    55  cdef class DoFnSignature(object):
    56    cdef public MethodWrapper process_method
    57    cdef public MethodWrapper process_batch_method
    58    cdef public MethodWrapper start_bundle_method
    59    cdef public MethodWrapper finish_bundle_method
    60    cdef public MethodWrapper setup_lifecycle_method
    61    cdef public MethodWrapper teardown_lifecycle_method
    62    cdef public MethodWrapper create_watermark_estimator_method
    63    cdef public MethodWrapper initial_restriction_method
    64    cdef public MethodWrapper create_tracker_method
    65    cdef public MethodWrapper split_method
    66    cdef public object batching_configuration
    67    cdef public object do_fn
    68    cdef public object timer_methods
    69    cdef bint _is_stateful_dofn
    70  
    71  
    72  cdef class DoFnInvoker(object):
    73    cdef public DoFnSignature signature
    74    cdef OutputHandler output_handler
    75    cdef object user_state_context
    76    cdef public object bundle_finalizer_param
    77  
    78    cpdef invoke_process(self, WindowedValue windowed_value,
    79                         restriction=*, watermark_estimator_state=*,
    80                         additional_args=*, additional_kwargs=*)
    81    cpdef invoke_start_bundle(self)
    82    cpdef invoke_finish_bundle(self)
    83    cpdef invoke_split(self, element, restriction)
    84    cpdef invoke_initial_restriction(self, element)
    85    cpdef invoke_create_tracker(self, restriction)
    86  
    87  
    88  cdef class SimpleInvoker(DoFnInvoker):
    89    cdef object process_method
    90    cdef object process_batch_method
    91  
    92  
    93  cdef class PerWindowInvoker(DoFnInvoker):
    94    cdef list side_inputs
    95    cdef DoFnContext context
    96    cdef list args_for_process
    97    cdef dict kwargs_for_process
    98    cdef list placeholders_for_process
    99    cdef list args_for_process_batch
   100    cdef dict kwargs_for_process_batch
   101    cdef list placeholders_for_process_batch
   102    cdef bint has_windowed_inputs
   103    cdef bint cache_globally_windowed_args
   104    cdef object process_method
   105    cdef object process_batch_method
   106    cdef bint is_splittable
   107    cdef object threadsafe_restriction_tracker
   108    cdef object threadsafe_watermark_estimator
   109    cdef WindowedValue current_windowed_value
   110    cdef object restriction
   111    cdef object watermark_estimator_state
   112    cdef object current_window_index
   113    cdef object stop_window_index
   114    cdef bint is_key_param_required
   115    cdef object splitting_lock
   116  
   117  
   118  cdef class DoFnRunner:
   119    cdef DoFnContext context
   120    cdef object step_name
   121    cdef list side_inputs
   122    cdef DoFnInvoker do_fn_invoker
   123    cdef public object bundle_finalizer_param
   124    cpdef process(self, WindowedValue windowed_value)
   125  
   126  
   127  cdef class OutputHandler(object):
   128    @cython.locals(windowed_value=WindowedValue,
   129                   output_element_count=int64_t)
   130    cpdef handle_process_outputs(self, WindowedValue element, results,
   131                                 watermark_estimator=*)
   132  
   133    @cython.locals(windowed_batch=WindowedBatch,
   134                   output_element_count=int64_t)
   135    cpdef handle_process_batch_outputs(self, WindowedBatch input_batch, results,
   136                                       watermark_estimator=*)
   137  
   138  
   139  cdef class _OutputHandler(OutputHandler):
   140    cdef object window_fn
   141    cdef Receiver main_receivers
   142    cdef object tagged_receivers
   143    cdef DataflowDistributionCounter per_element_output_counter
   144    cdef object output_batch_converter
   145    cdef bint _process_batch_yields_elements
   146    cdef bint _process_yields_batches
   147  
   148    @cython.locals(windowed_value=WindowedValue,
   149                   windowed_batch=WindowedBatch,
   150                   output_element_count=int64_t)
   151    cpdef handle_process_outputs(self, WindowedValue element, results,
   152                                 watermark_estimator=*)
   153  
   154    @cython.locals(windowed_value=WindowedValue,
   155                   windowed_batch=WindowedBatch,
   156                   output_element_count=int64_t)
   157    cpdef handle_process_batch_outputs(self, WindowedBatch input_batch, results,
   158                                       watermark_estimator=*)
   159  
   160    @cython.locals(windowed_value=WindowedValue)
   161    cdef inline WindowedValue _maybe_propagate_windowing_info(self, WindowedValue input_element, result)
   162    cdef inline tuple _handle_tagged_output(self, result)
   163    cdef inline _write_value_to_tag(self, tag, WindowedValue windowed_value,
   164                                    watermark_estimator)
   165    cdef inline _write_batch_to_tag(self, tag, WindowedBatch windowed_batch,
   166                                    watermark_estimator)
   167    cdef inline _verify_batch_output(self, result)
   168  
   169  cdef class DoFnContext(object):
   170    cdef object label
   171    cdef object state
   172    cdef WindowedValue windowed_value
   173    cpdef set_element(self, WindowedValue windowed_value)
   174  
   175  
   176  cdef class _ReceiverAdapter(Receiver):
   177    cdef object underlying