github.com/pingcap/tiflow@v0.0.0-20240520035814-5bf52d54e205/pkg/sink/kafka/metrics.go (about)

     1  // Copyright 2022 PingCAP, Inc.
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // See the License for the specific language governing permissions and
    12  // limitations under the License.
    13  
    14  package kafka
    15  
    16  import "github.com/prometheus/client_golang/prometheus"
    17  
    18  var (
    19  	// Counter inc by 1 once a request send, dec by 1 for a response received.
    20  	requestsInFlightGauge = prometheus.NewGaugeVec(
    21  		prometheus.GaugeOpts{
    22  			Namespace: "ticdc",
    23  			Subsystem: "sink",
    24  			Name:      "kafka_producer_in_flight_requests",
    25  			Help: "The current number of in-flight requests" +
    26  				" awaiting a response for all brokers.",
    27  		}, []string{"namespace", "changefeed", "broker"})
    28  	// OutgoingByteRateGauge for outgoing events.
    29  	// Meter mark for each request's size in bytes.
    30  	OutgoingByteRateGauge = prometheus.NewGaugeVec(
    31  		prometheus.GaugeOpts{
    32  			Namespace: "ticdc",
    33  			Subsystem: "sink",
    34  			Name:      "kafka_producer_outgoing_byte_rate",
    35  			Help:      "Bytes/second written off all brokers.",
    36  		}, []string{"namespace", "changefeed", "broker"})
    37  	// RequestRateGauge Meter mark by 1 for each request.
    38  	RequestRateGauge = prometheus.NewGaugeVec(
    39  		prometheus.GaugeOpts{
    40  			Namespace: "ticdc",
    41  			Subsystem: "sink",
    42  			Name:      "kafka_producer_request_rate",
    43  			Help:      "Requests/second sent to all brokers.",
    44  		}, []string{"namespace", "changefeed", "broker"})
    45  	// RequestLatencyGauge Histogram update by `requestLatency`.
    46  	RequestLatencyGauge = prometheus.NewGaugeVec(
    47  		prometheus.GaugeOpts{
    48  			Namespace: "ticdc",
    49  			Subsystem: "sink",
    50  			Name:      "kafka_producer_request_latency",
    51  			Help:      "The request latency for all brokers.",
    52  		}, []string{"namespace", "changefeed", "broker"})
    53  	// Histogram update by `compression-ratio`.
    54  	compressionRatioGauge = prometheus.NewGaugeVec(
    55  		prometheus.GaugeOpts{
    56  			Namespace: "ticdc",
    57  			Subsystem: "sink",
    58  			Name:      "kafka_producer_compression_ratio",
    59  			Help:      "The compression ratio times 100 of record batches for all topics.",
    60  		}, []string{"namespace", "changefeed"})
    61  
    62  	// updated by `records-per-request`.
    63  	recordsPerRequestGauge = prometheus.NewGaugeVec(
    64  		prometheus.GaugeOpts{
    65  			Namespace: "ticdc",
    66  			Subsystem: "sink",
    67  			Name:      "kafka_producer_records_per_request",
    68  			Help:      "The P99 number of records per request for all topics.",
    69  		}, []string{"namespace", "changefeed"})
    70  
    71  	// Meter mark by 1 once a response received.
    72  	responseRateGauge = prometheus.NewGaugeVec(
    73  		prometheus.GaugeOpts{
    74  			Namespace: "ticdc",
    75  			Subsystem: "sink",
    76  			Name:      "kafka_producer_response_rate",
    77  			Help:      "Responses/second received from all brokers.",
    78  		}, []string{"namespace", "changefeed", "broker"})
    79  
    80  	// ClientRetryGauge only for kafka-go client to track internal retry count.
    81  	ClientRetryGauge = prometheus.NewGaugeVec(
    82  		prometheus.GaugeOpts{
    83  			Namespace: "ticdc",
    84  			Subsystem: "sink",
    85  			Name:      "kafka_producer_retry_count",
    86  			Help:      "Kafka client send request retry count",
    87  		}, []string{"namespace", "changefeed"})
    88  
    89  	// ClientErrorGauge only for kafka-go client to track internal error count.
    90  	ClientErrorGauge = prometheus.NewGaugeVec(
    91  		prometheus.GaugeOpts{
    92  			Namespace: "ticdc",
    93  			Subsystem: "sink",
    94  			Name:      "kafka_producer_err_count",
    95  			Help:      "Kafka client send request retry count",
    96  		}, []string{"namespace", "changefeed"})
    97  
    98  	// BatchDurationGauge only for kafka-go client to track internal batch duration.
    99  	BatchDurationGauge = prometheus.NewGaugeVec(
   100  		prometheus.GaugeOpts{
   101  			Namespace: "ticdc",
   102  			Subsystem: "sink",
   103  			Name:      "kafka_producer_batch_duration",
   104  			Help:      "Kafka client internal average batch message time cost in milliseconds",
   105  		}, []string{"namespace", "changefeed"})
   106  
   107  	// BatchMessageCountGauge only for kafka-go client to track each batch's messages count.
   108  	BatchMessageCountGauge = prometheus.NewGaugeVec(
   109  		prometheus.GaugeOpts{
   110  			Namespace: "ticdc",
   111  			Subsystem: "sink",
   112  			Name:      "kafka_producer_batch_message_count",
   113  			Help:      "Kafka client internal average batch message count",
   114  		}, []string{"namespace", "changefeed"})
   115  
   116  	// BatchSizeGauge only for kafka-go client to track each batch's size in bytes.
   117  	BatchSizeGauge = prometheus.NewGaugeVec(
   118  		prometheus.GaugeOpts{
   119  			Namespace: "ticdc",
   120  			Subsystem: "sink",
   121  			Name:      "kafka_producer_batch_size",
   122  			Help:      "Kafka client internal average batch size in bytes",
   123  		}, []string{"namespace", "changefeed"})
   124  )
   125  
   126  // InitMetrics registers all metrics in this file.
   127  func InitMetrics(registry *prometheus.Registry) {
   128  	registry.MustRegister(compressionRatioGauge)
   129  	registry.MustRegister(recordsPerRequestGauge)
   130  	registry.MustRegister(OutgoingByteRateGauge)
   131  	registry.MustRegister(RequestRateGauge)
   132  	registry.MustRegister(RequestLatencyGauge)
   133  	registry.MustRegister(requestsInFlightGauge)
   134  	registry.MustRegister(responseRateGauge)
   135  
   136  	// only used by kafka sink v2.
   137  	registry.MustRegister(BatchDurationGauge)
   138  	registry.MustRegister(BatchMessageCountGauge)
   139  	registry.MustRegister(BatchSizeGauge)
   140  	registry.MustRegister(ClientRetryGauge)
   141  	registry.MustRegister(ClientErrorGauge)
   142  }