github.com/apache/beam/sdks/v2@v2.48.2/python/apache_beam/testing/benchmarks/nexmark/monitor.py (about)

     1  #
     2  # Licensed to the Apache Software Foundation (ASF) under one or more
     3  # contributor license agreements.  See the NOTICE file distributed with
     4  # this work for additional information regarding copyright ownership.
     5  # The ASF licenses this file to You under the Apache License, Version 2.0
     6  # (the "License"); you may not use this file except in compliance with
     7  # the License.  You may obtain a copy of the License at
     8  #
     9  #    http://www.apache.org/licenses/LICENSE-2.0
    10  #
    11  # Unless required by applicable law or agreed to in writing, software
    12  # distributed under the License is distributed on an "AS IS" BASIS,
    13  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14  # See the License for the specific language governing permissions and
    15  # limitations under the License.
    16  #
    17  
    18  from time import time
    19  
    20  import apache_beam as beam
    21  from apache_beam.metrics import Metrics
    22  
    23  
    24  class Monitor(object):
    25    """
    26    A monitor of elements with support for later retrieving their metrics
    27  
    28    monitor objects contains a doFn to record metrics
    29  
    30    Args:
    31      namespace: the namespace all metrics within this Monitor uses
    32      name_prefix: a prefix for this Monitor's metrics' names, intended to
    33        be unique in per-monitor basis in pipeline
    34    """
    35    def __init__(self, namespace, name_prefix):
    36      # type: (str, str) -> None
    37      self.namespace = namespace
    38      self.name_prefix = name_prefix
    39      self.doFn = MonitorDoFn(namespace, name_prefix)
    40  
    41  
    42  class MonitorDoFn(beam.DoFn):
    43    def __init__(self, namespace, prefix):
    44      self.element_count = Metrics.counter(
    45          namespace, prefix + MonitorSuffix.ELEMENT_COUNTER)
    46      self.event_time = Metrics.distribution(
    47          namespace, prefix + MonitorSuffix.EVENT_TIME)
    48      self.event_timestamp = Metrics.distribution(
    49          namespace, prefix + MonitorSuffix.EVENT_TIMESTAMP)
    50  
    51    def start_bundle(self):
    52      self.event_time.update(int(time() * 1000))
    53  
    54    def process(self, element, timestamp=beam.DoFn.TimestampParam):
    55      self.element_count.inc()
    56      self.event_timestamp.update(timestamp)
    57      yield element
    58  
    59    def finish_bundle(self):
    60      self.event_time.update(int(time() * 1000))
    61  
    62  
    63  class MonitorSuffix:
    64    ELEMENT_COUNTER = '.elements'
    65    EVENT_TIMESTAMP = '.event_timestamp'
    66    EVENT_TIME = '.event_time'