github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/runners/worker/operations.pxd (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  cimport cython
    19  
    20  from apache_beam.runners.common cimport DoFnRunner
    21  from apache_beam.runners.common cimport Receiver
    22  from apache_beam.runners.worker cimport opcounters
    23  from apache_beam.utils.windowed_value cimport WindowedValue
    24  from apache_beam.utils.windowed_value cimport WindowedBatch
    25  #from libcpp.string cimport string
    26  
    27  cdef WindowedValue _globally_windowed_value
    28  cdef type _global_window_type
    29  
    30  
    31  cdef class ConsumerSet(Receiver):
    32    cdef list consumers
    33    cdef readonly opcounters.OperationCounters opcounter
    34    cdef public step_name
    35    cdef public output_index
    36    cdef public coder
    37  
    38    cpdef update_counters_start(self, WindowedValue windowed_value)
    39    cpdef update_counters_finish(self)
    40    cpdef update_counters_batch(self, WindowedBatch windowed_batch)
    41  
    42  cdef class SingletonElementConsumerSet(ConsumerSet):
    43    cdef Operation consumer
    44  
    45    cpdef receive(self, WindowedValue windowed_value)
    46    cpdef receive_batch(self, WindowedBatch windowed_batch)
    47    cpdef flush(self)
    48  
    49  cdef class GeneralPurposeConsumerSet(ConsumerSet):
    50    cdef list element_consumers
    51    cdef list passthrough_batch_consumers
    52    cdef dict other_batch_consumers
    53    cdef bint has_batch_consumers
    54    cdef list _batched_elements
    55    cdef object producer_batch_converter
    56  
    57    cpdef receive(self, WindowedValue windowed_value)
    58    cpdef receive_batch(self, WindowedBatch windowed_batch)
    59    cpdef flush(self)
    60  
    61  cdef class Operation(object):
    62    cdef readonly name_context
    63    cdef readonly operation_name
    64    cdef readonly spec
    65    cdef object consumers
    66    cdef readonly counter_factory
    67    cdef public metrics_container
    68    cdef public execution_context
    69    # Public for access by Fn harness operations.
    70    # TODO(robertwb): Cythonize FnHarness.
    71    cdef public list receivers
    72    cdef readonly bint debug_logging_enabled
    73    # For legacy workers.
    74    cdef bint setup_done
    75  
    76    cdef public step_name  # initialized lazily
    77  
    78    cdef readonly object state_sampler
    79  
    80    cdef readonly object scoped_start_state
    81    cdef readonly object scoped_process_state
    82    cdef readonly object scoped_finish_state
    83  
    84    cpdef start(self)
    85    cpdef process(self, WindowedValue windowed_value)
    86    cpdef finish(self)
    87    cpdef teardown(self)
    88    cpdef output(self, WindowedValue windowed_value, int output_index=*)
    89    cpdef execution_time_monitoring_infos(self, transform_id)
    90    cpdef user_monitoring_infos(self, transform_id)
    91    cpdef pcollection_count_monitoring_infos(self, tag_to_pcollection_id)
    92    cpdef monitoring_infos(self, transform_id, tag_to_pcollection_id)
    93  
    94  
    95  cdef class ReadOperation(Operation):
    96    @cython.locals(windowed_value=WindowedValue)
    97    cpdef start(self)
    98  
    99  
   100  cdef class ImpulseReadOperation(Operation):
   101    cdef object source
   102    @cython.locals(windowed_value=WindowedValue)
   103    cpdef process(self, WindowedValue impulse)
   104  
   105  
   106  cdef class DoOperation(Operation):
   107    cdef DoFnRunner dofn_runner
   108    cdef object tagged_receivers
   109    cdef object side_input_maps
   110    cdef object user_state_context
   111    cdef public dict timer_inputs
   112    cdef dict timer_specs
   113    cdef public object input_info
   114    cdef object fn
   115  
   116  
   117  cdef class SdfProcessSizedElements(DoOperation):
   118    cdef object lock
   119    cdef object element_start_output_bytes
   120  
   121  cdef class SdfTruncateSizedRestrictions(DoOperation):
   122    pass
   123  
   124  cdef class CombineOperation(Operation):
   125    cdef object phased_combine_fn
   126  
   127  
   128  cdef class PGBKCVOperation(Operation):
   129    cdef public object combine_fn
   130    cdef public object combine_fn_add_input
   131    cdef public object combine_fn_compact
   132    cdef public bint is_default_windowing
   133    cdef public object timestamp_combiner
   134    cdef dict table
   135    cdef long max_keys
   136    cdef long key_count
   137  
   138    cpdef output_key(self, wkey, value, timestamp)
   139  
   140  
   141  cdef class FlattenOperation(Operation):
   142    pass