github.com/badrootd/celestia-core@v0.0.0-20240305091328-aa4207a4b25d/scripts/qa/reporting/prometheus_plotter.py (about) 1 # pip install numpy pandas matplotlib requests 2 3 import sys 4 import os 5 6 import matplotlib as mpl 7 import matplotlib.pyplot as plt 8 9 import numpy as np 10 import pandas as pd 11 12 import requests 13 from urllib.parse import urljoin 14 15 from prometheus_pandas import query 16 17 release = 'v0.34.x-baseline' 18 path = os.path.join('imgs') 19 prometheus = query.Prometheus('http://localhost:9090') 20 21 # Time window 22 window_size = dict(seconds=150) 23 #window_size = dict(seconds=130) #homogeneous 24 #window_size = dict(seconds=1270) #baseline 25 ext_window_size = dict(seconds=180) 26 27 #right_end = '2023-02-08T13:14:50Z' #cmt2 tm1 28 #right_end = '2023-02-08T10:33:50Z' #cmt1 tm2 29 right_end = '2023-02-14T15:20:30.000Z' #cmt1 tm1 30 #right_end = '2023-02-07T18:09:10Z' #homogeneous 31 #right_end = '2022-10-13T19:43:30Z' #baseline 32 left_end = pd.to_datetime(right_end) - pd.Timedelta(**window_size) 33 time_window = (left_end.strftime('%Y-%m-%dT%H:%M:%SZ'), right_end) 34 35 ext_left_end = pd.to_datetime(right_end) - pd.Timedelta(**ext_window_size) 36 ext_time_window = (ext_left_end.strftime('%Y-%m-%dT%H:%M:%SZ'), right_end) 37 38 fork='cometbft' 39 #fork='tendermint' 40 41 # Do prometheus queries 42 queries = [ 43 (( fork + '_mempool_size', time_window[0], time_window[1], '1s'), 'mempool_size', dict(ylabel='TXs', xlabel='time (s)', title='Mempool Size', legend=False, figsize=(10,6), grid=True, kind='area',stacked=True), False), 44 (( fork + '_p2p_peers', time_window[0], time_window[1], '1s'), 'peers', dict(ylabel='# Peers', xlabel='time (s)', title='Peers', legend=False, figsize=(10,6), grid=True), True), 45 (( 'avg(' + fork + '_mempool_size)', time_window[0], time_window[1], '1s'), 'avg_mempool_size', dict(ylabel='TXs', xlabel='time (s)', title='Average Mempool Size', legend=False, figsize=(10,6), grid=True), False), 46 #(( 'cometbft_consensus_height', time_window[0], time_window[1], '1s'), 'blocks_regular', dict(ylabel='# Blocks', xlabel='time (s)', title='Blocks in time', legend=False, figsize=(10,6), grid=True), False), 47 (( fork + '_consensus_rounds', time_window[0], time_window[1], '1s'), 'rounds', dict(ylabel='# Rounds', xlabel='time (s)', title='Rounds per block', legend=False, figsize=(10,6), grid=True), False), 48 (( 'rate(' + fork + '_consensus_height[20s])*60', time_window[0], time_window[1], '1s'), 'block_rate_regular', dict(ylabel='Blocks/min', xlabel='time (s)', title='Rate of block creation', legend=False, figsize=(10,6), grid=True), True), 49 #(( 'avg(rate(cometbft_consensus_height[20s])*60)', time_window[0], time_window[1], '1s'), 'block_rate_avg_reg', dict(ylabel='Blocks/min', xlabel='time (s)', title='Rate of block creation', legend=False, figsize=(10,6), grid=True), False), 50 #(( 'cometbft_consensus_total_txs', time_window[0], time_window[1], '1s'), 'total_txs_regular', dict(ylabel='# TXs', xlabel='time (s)', title='Transactions in time', legend=False, figsize=(10,6), grid=True), False), 51 (( 'rate(' + fork + '_consensus_total_txs[20s])*60', time_window[0], time_window[1], '1s'), 'total_txs_rate_regular', dict(ylabel='TXs/min', xlabel='time (s)', title='Rate of transaction processing', legend=False, figsize=(10,6), grid=True), True), 52 #(( 'avg(rate(cometbft_consensus_total_txs[20s])*60)', time_window[0], time_window[1], '1s'), 'total_txs_rate_avg_reg', dict(ylabel='TXs/min', xlabel='time (s)', title='Rate of transaction processing', legend=False, figsize=(10,6), grid=True), False), 53 (( 'process_resident_memory_bytes', time_window[0], time_window[1], '1s'), 'memory', dict(ylabel='Memory (bytes)', xlabel='time (s)', title='Memory usage', legend=False, figsize=(10,6), grid=True), False), 54 (( 'avg(process_resident_memory_bytes)', time_window[0], time_window[1], '1s'), 'avg_memory', dict(ylabel='Memory (bytes)', xlabel='time (s)', title='Average Memory usage', legend=False, figsize=(10,6), grid=True), False), 55 (( 'node_load1', time_window[0], time_window[1], '1s'), 'cpu', dict(ylabel='Load', xlabel='time (s)', title='Node load', legend=False, figsize=(10,6), grid=True), False), 56 (( 'avg(node_load1)', time_window[0], time_window[1], '1s'), 'avg_cpu', dict(ylabel='Load', xlabel='time (s)', title='Average Node load', legend=False, figsize=(10,6), grid=True), False), 57 #extended window metrics 58 (( fork + '_consensus_height', ext_time_window[0], ext_time_window[1], '1s'), 'blocks', dict(ylabel='# Blocks', xlabel='time (s)', title='Blocks in time', legend=False, figsize=(10,6), grid=True), False), 59 (( 'rate(' + fork + '_consensus_height[20s])*60', ext_time_window[0], ext_time_window[1], '1s'), 'block_rate', dict(ylabel='Blocks/min', xlabel='time (s)', title='Rate of block creation', legend=False, figsize=(10,6), grid=True), True), 60 (( fork + '_consensus_total_txs', ext_time_window[0], ext_time_window[1], '1s'), 'total_txs', dict(ylabel='# TXs', xlabel='time (s)', title='Transactions in time', legend=False, figsize=(10,6), grid=True), False), 61 (( 'rate(' + fork + '_consensus_total_txs[20s])*60', ext_time_window[0], ext_time_window[1], '1s'), 'total_txs_rate', dict(ylabel='TXs/min', xlabel='time (s)', title='Rate of transaction processing', legend=False, figsize=(10,6), grid=True), True), 62 ] 63 64 for (query, file_name, pandas_params, plot_average) in queries: 65 print(query) 66 67 data_frame = prometheus.query_range(*query) 68 #Tweak the x ticks 69 delta_index = pd.to_timedelta(data_frame.index.strftime('%H:%M:%S')) 70 data_frame = data_frame.set_index(delta_index) 71 72 data_frame.plot(**pandas_params) 73 if plot_average: 74 average = data_frame.mean(axis=1) 75 data_frame['__average__'] = average 76 pandas_params['lw'] = 8 77 pandas_params['style'] = ['--'] 78 pandas_params['color'] = ['red'] 79 data_frame['__average__'].plot(**pandas_params) 80 81 plt.savefig(os.path.join(path, file_name + '.png')) 82 plt.plot() 83 84 plt.show()