github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/io/gcp/pubsublite/external.py (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  """Google Pub/Sub Lite sources and sinks.
    19  
    20  This API is currently under development and is subject to change.
    21  """
    22  
    23  # pytype: skip-file
    24  
    25  import typing
    26  
    27  from apache_beam.transforms.external import BeamJarExpansionService
    28  from apache_beam.transforms.external import ExternalTransform
    29  from apache_beam.transforms.external import NamedTupleBasedPayloadBuilder
    30  
    31  _ReadSchema = typing.NamedTuple(
    32      '_ReadSchema', [('subscription_path', str), ('deduplicate', bool)])
    33  
    34  
    35  def _default_io_expansion_service():
    36    return BeamJarExpansionService(
    37        'sdks:java:io:google-cloud-platform:expansion-service:shadowJar')
    38  
    39  
    40  class _ReadExternal(ExternalTransform):
    41    """
    42      An external PTransform which reads from Pub/Sub Lite and returns a
    43      SequencedMessage as serialized bytes.
    44  
    45      This transform is not part of the public API.
    46  
    47      Experimental; no backwards-compatibility guarantees.
    48    """
    49    def __init__(
    50        self,
    51        subscription_path,
    52        deduplicate=None,
    53        expansion_service=None,
    54    ):
    55      """
    56      Initializes a read operation from Pub/Sub Lite, returning the serialized
    57      bytes of SequencedMessage protos.
    58  
    59      Args:
    60        subscription_path: A Pub/Sub Lite Subscription path.
    61        deduplicate: Whether to deduplicate messages based on the value of
    62            the 'x-goog-pubsublite-dataflow-uuid' attribute.
    63      """
    64      if deduplicate is None:
    65        deduplicate = False
    66      if expansion_service is None:
    67        expansion_service = _default_io_expansion_service()
    68      super().__init__(
    69          'beam:transform:org.apache.beam:pubsublite_read:v1',
    70          NamedTupleBasedPayloadBuilder(
    71              _ReadSchema(
    72                  subscription_path=subscription_path, deduplicate=deduplicate)),
    73          expansion_service)
    74  
    75  
    76  _WriteSchema = typing.NamedTuple(
    77      '_WriteSchema', [('topic_path', str), ('add_uuids', bool)])
    78  
    79  
    80  class _WriteExternal(ExternalTransform):
    81    """
    82      An external PTransform which writes serialized PubSubMessage protos to
    83      Pub/Sub Lite.
    84  
    85      This transform is not part of the public API.
    86  
    87      Experimental; no backwards-compatibility guarantees.
    88    """
    89    def __init__(
    90        self,
    91        topic_path,
    92        add_uuids=None,
    93        expansion_service=None,
    94    ):
    95      """
    96      Initializes a write operation to Pub/Sub Lite, writing the serialized bytes
    97      of PubSubMessage protos.
    98  
    99      Args:
   100        topic_path: A Pub/Sub Lite Topic path.
   101        add_uuids: Whether to add uuids to the 'x-goog-pubsublite-dataflow-uuid'
   102            uuid attribute.
   103      """
   104      if add_uuids is None:
   105        add_uuids = False
   106      if expansion_service is None:
   107        expansion_service = _default_io_expansion_service()
   108      super().__init__(
   109          'beam:transform:org.apache.beam:pubsublite_write:v1',
   110          NamedTupleBasedPayloadBuilder(
   111              _WriteSchema(
   112                  topic_path=topic_path,
   113                  add_uuids=add_uuids,
   114              )),
   115          expansion_service)