github.com/netdata/go.d.plugin@v0.58.1/modules/pulsar/metrics.go (about)

     1  // SPDX-License-Identifier: GPL-3.0-or-later
     2  
     3  package pulsar
     4  
     5  /*
     6  Architecture:
     7   - https://pulsar.apache.org/docs/en/concepts-overview/
     8  
     9  Terminology:
    10   - https://pulsar.apache.org/docs/en/reference-terminology/
    11  
    12  Deploy Monitoring:
    13   - http://pulsar.apache.org/docs/en/deploy-monitoring/
    14  
    15  Metrics Reference:
    16   - https://github.com/apache/pulsar/blob/master/site2/docs/reference-metrics.md
    17  
    18  REST API
    19   - http://pulsar.apache.org/admin-rest-api/?version=master
    20  
    21  Grafana Dashboards:
    22   - https://github.com/apache/pulsar/tree/master/docker/grafana/dashboards
    23  
    24  Stats in the source code:
    25   - https://github.com/apache/pulsar/blob/master/pulsar-common/src/main/java/org/apache/pulsar/common/policies/data/
    26   - https://github.com/apache/pulsar/tree/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus
    27  
    28  If !'exposeTopicLevelMetricsInPrometheus:
    29   - https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
    30  else:
    31   - https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/TopicStats.java
    32  
    33  Metrics updates parameters:
    34   - statsUpdateFrequencyInSecs=60
    35   - statsUpdateInitialDelayInSecs=60
    36  
    37  Metrics Exposing:
    38   - Namespace   : 'exposeTopicLevelMetricsInPrometheus' is set to false.
    39   - Replication : 'replicationMetricsEnabled' is enabled.
    40   - Topic       : 'exposeTopicLevelMetricsInPrometheus' is set to true.
    41   - Subscription: 'exposeTopicLevelMetricsInPrometheus' is set to true
    42   - Consumer    : 'exposeTopicLevelMetricsInPrometheus' and 'exposeConsumerLevelMetricsInPrometheus' are set to true.
    43   - Publisher   : 'exposePublisherStats' is set to true. RESP API option. (/admin/v2/broker-stats/topics)
    44  */
    45  
    46  /*
    47  TODO:
    48  Unused broker metrics:
    49   - "pulsar_storage_backlog_size"           : ?? is estimated total unconsumed or backlog size in bytes for the managed ledger, without accounting for replicas.
    50   - "pulsar_storage_offloaded_size"         : ?? is the size of all ledgers offloaded to 2nd tier storage.
    51   - "pulsar_storage_backlog_quota_limit"    : ?? is the total amount of the data in this topic that limit the backlog quota.
    52   - "pulsar_in_bytes_total"                 : use "pulsar_throughput_in" for the same data.
    53   - "pulsar_in_messages_total"              : use "pulsar_rate_in" for the same data.
    54   - "pulsar_subscription_unacked_messages"  : negative values (https://github.com/apache/pulsar/issues/6510)
    55   - "pulsar_subscription_back_log"          : to detailed, we have summary per topic. Part of "pulsar_msg_backlog" (msgBacklog).
    56   - "pulsar_subscription_msg_rate_out"      : to detailed, we have summary per topic. Part of "pulsar_rate_out".
    57   - "pulsar_subscription_msg_throughput_out": to detailed, we have summary per topic. Part of "pulsar_throughput_out".
    58  
    59   + All Consumer metrics (for each namespace, topic, subscription).
    60   + JVM metrics.
    61   + Zookeeper metrics.
    62   + Bookkeeper metrics.
    63  
    64  Hardcoded update interval? (60)
    65   - pulsar_storage_write_latency_le_*
    66   - pulsar_entry_size_le_*
    67  */
    68  
    69  /*
    70  https://github.com/apache/pulsar/blob/master/pulsar-broker/src/main/java/org/apache/pulsar/broker/stats/prometheus/NamespaceStatsAggregator.java
    71  Zero metrics which always present (labels: cluster):
    72   - "pulsar_topics_count"
    73   - "pulsar_subscriptions_count"
    74   - "pulsar_producers_count"
    75   - "pulsar_consumers_count"
    76   - "pulsar_rate_in"
    77   - "pulsar_rate_out"
    78   - "pulsar_throughput_in"
    79   - "pulsar_throughput_out"
    80   - "pulsar_storage_size"
    81   - "pulsar_storage_write_rate"
    82   - "pulsar_storage_read_rate"
    83   - "pulsar_msg_backlog"
    84  */
    85  
    86  const (
    87  	// Namespace metrics (labels: namespace)
    88  	metricPulsarTopicsCount = "pulsar_topics_count"
    89  	// Namespace, Topic metrics (labels: namespace || namespace, topic)
    90  	metricPulsarSubscriptionsCount = "pulsar_subscriptions_count"
    91  	metricPulsarProducersCount     = "pulsar_producers_count"
    92  	metricPulsarConsumersCount     = "pulsar_consumers_count"
    93  	metricPulsarRateIn             = "pulsar_rate_in"
    94  	metricPulsarRateOut            = "pulsar_rate_out"
    95  	metricPulsarThroughputIn       = "pulsar_throughput_in"
    96  	metricPulsarThroughputOut      = "pulsar_throughput_out"
    97  	metricPulsarStorageSize        = "pulsar_storage_size"
    98  	metricPulsarStorageWriteRate   = "pulsar_storage_write_rate" // exposed with labels only if there is Bookie
    99  	metricPulsarStorageReadRate    = "pulsar_storage_read_rate"  // exposed with labels only if there is Bookie
   100  	metricPulsarMsgBacklog         = "pulsar_msg_backlog"        // has 'remote_cluster' label if no topic stats
   101  	// pulsar_storage_write_latency_le_*
   102  	// pulsar_entry_size_le_*
   103  
   104  	// Subscriptions metrics (labels: namespace, topic, subscription)
   105  	metricPulsarSubscriptionDelayed                  = "pulsar_subscription_delayed" // Number of delayed messages currently being tracked
   106  	metricPulsarSubscriptionMsgRateRedeliver         = "pulsar_subscription_msg_rate_redeliver"
   107  	metricPulsarSubscriptionBlockedOnUnackedMessages = "pulsar_subscription_blocked_on_unacked_messages"
   108  
   109  	// Replication metrics (labels: namespace, remote_cluster || namespace, topic, remote_cluster)
   110  	// Exposed only when replication is enabled.
   111  	metricPulsarReplicationRateIn        = "pulsar_replication_rate_in"
   112  	metricPulsarReplicationRateOut       = "pulsar_replication_rate_out"
   113  	metricPulsarReplicationThroughputIn  = "pulsar_replication_throughput_in"
   114  	metricPulsarReplicationThroughputOut = "pulsar_replication_throughput_out"
   115  	metricPulsarReplicationBacklog       = "pulsar_replication_backlog"
   116  )