github.com/yankunsam/loki/v2@v2.6.3-0.20220817130409-389df5235c27/production/loki-mixin/dashboards/loki-chunks.libsonnet (about) 1 local grafana = import 'grafonnet/grafana.libsonnet'; 2 local utils = import 'mixin-utils/utils.libsonnet'; 3 4 (import 'dashboard-utils.libsonnet') { 5 grafanaDashboards+: { 6 local dashboards = self, 7 'loki-chunks.json': { 8 local cfg = self, 9 labelsSelector:: $._config.per_cluster_label + '="$cluster", job=~"$namespace/%s"' % (if $._config.ssd.enabled then '%s-write' % $._config.ssd.pod_prefix_matcher else 'ingester'), 10 } + 11 $.dashboard('Loki / Chunks', uid='chunks') 12 .addCluster() 13 .addNamespace() 14 .addTag() 15 .addRow( 16 $.row('Active Series / Chunks') 17 .addPanel( 18 $.panel('Series') + 19 $.queryPanel('sum(loki_ingester_memory_chunks{%s})' % dashboards['loki-chunks.json'].labelsSelector, 'series'), 20 ) 21 .addPanel( 22 $.panel('Chunks per series') + 23 $.queryPanel( 24 'sum(loki_ingester_memory_chunks{%s}) / sum(loki_ingester_memory_streams{%s})' % [ 25 dashboards['loki-chunks.json'].labelsSelector, 26 dashboards['loki-chunks.json'].labelsSelector, 27 ], 28 'chunks' 29 ), 30 ) 31 ) 32 .addRow( 33 $.row('Flush Stats') 34 .addPanel( 35 $.panel('Utilization') + 36 $.latencyPanel('loki_ingester_chunk_utilization', '{%s}' % dashboards['loki-chunks.json'].labelsSelector, multiplier='1') + 37 { yaxes: $.yaxes('percentunit') }, 38 ) 39 .addPanel( 40 $.panel('Age') + 41 $.latencyPanel('loki_ingester_chunk_age_seconds', '{%s}' % dashboards['loki-chunks.json'].labelsSelector), 42 ), 43 ) 44 .addRow( 45 $.row('Flush Stats') 46 .addPanel( 47 $.panel('Log Entries Per Chunk') + 48 $.latencyPanel('loki_ingester_chunk_entries', '{%s}' % dashboards['loki-chunks.json'].labelsSelector, multiplier='1') + 49 { yaxes: $.yaxes('short') }, 50 ) 51 .addPanel( 52 $.panel('Index Entries Per Chunk') + 53 $.queryPanel( 54 'sum(rate(loki_chunk_store_index_entries_per_chunk_sum{%s}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{%s}[5m]))' % [ 55 dashboards['loki-chunks.json'].labelsSelector, 56 dashboards['loki-chunks.json'].labelsSelector, 57 ], 58 'Index Entries' 59 ), 60 ), 61 ) 62 .addRow( 63 $.row('Flush Stats') 64 .addPanel( 65 $.panel('Queue Length') + 66 $.queryPanel('cortex_ingester_flush_queue_length{%s}' % dashboards['loki-chunks.json'].labelsSelector, '{{pod}}'), 67 ) 68 .addPanel( 69 $.panel('Flush Rate') + 70 $.qpsPanel('loki_ingester_chunk_age_seconds_count{%s}' % dashboards['loki-chunks.json'].labelsSelector,), 71 ), 72 ) 73 .addRow( 74 $.row('Flush Stats') 75 .addPanel( 76 $.panel('Chunks Flushed/Second') + 77 $.queryPanel('sum(rate(loki_ingester_chunks_flushed_total{%s}[$__rate_interval]))' % dashboards['loki-chunks.json'].labelsSelector, '{{pod}}'), 78 ) 79 .addPanel( 80 $.panel('Chunk Flush Reason') + 81 $.queryPanel('sum by (reason) (rate(loki_ingester_chunks_flushed_total{%s}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{%s}[$__rate_interval]))' % [dashboards['loki-chunks.json'].labelsSelector, dashboards['loki-chunks.json'].labelsSelector], '{{reason}}') + { 82 stack: true, 83 yaxes: [ 84 { format: 'short', label: null, logBase: 1, max: 1, min: 0, show: true }, 85 { format: 'short', label: null, logBase: 1, max: 1, min: null, show: false }, 86 ], 87 }, 88 ), 89 ) 90 .addRow( 91 $.row('Utilization') 92 .addPanel( 93 grafana.heatmapPanel.new( 94 'Chunk Utilization', 95 datasource='$datasource', 96 yAxis_format='percentunit', 97 tooltip_showHistogram=true, 98 color_colorScheme='interpolateSpectral', 99 dataFormat='tsbuckets', 100 yAxis_decimals=0, 101 legend_show=true, 102 ).addTargets( 103 [ 104 grafana.prometheus.target( 105 'sum by (le) (rate(loki_ingester_chunk_utilization_bucket{%s}[$__rate_interval]))' % dashboards['loki-chunks.json'].labelsSelector, 106 legendFormat='{{le}}', 107 format='heatmap', 108 ), 109 ], 110 ) 111 ) 112 ) 113 .addRow( 114 $.row('Utilization') 115 .addPanel( 116 grafana.heatmapPanel.new( 117 'Chunk Size Bytes', 118 datasource='$datasource', 119 yAxis_format='bytes', 120 tooltip_showHistogram=true, 121 color_colorScheme='interpolateSpectral', 122 dataFormat='tsbuckets', 123 yAxis_decimals=0, 124 // tooltipDecimals=3, 125 // span=3, 126 legend_show=true, 127 ).addTargets( 128 [ 129 grafana.prometheus.target( 130 'sum(rate(loki_ingester_chunk_size_bytes_bucket{%s}[$__rate_interval])) by (le)' % dashboards['loki-chunks.json'].labelsSelector, 131 legendFormat='{{le}}', 132 format='heatmap', 133 ), 134 ], 135 ) 136 ) 137 ) 138 .addRow( 139 $.row('Utilization') 140 .addPanel( 141 $.panel('Chunk Size Quantiles') + 142 $.queryPanel( 143 [ 144 'histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{%s}[1m])) by (le))' % dashboards['loki-chunks.json'].labelsSelector, 145 'histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{%s}[1m])) by (le))' % dashboards['loki-chunks.json'].labelsSelector, 146 'histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{%s}[1m])) by (le))' % dashboards['loki-chunks.json'].labelsSelector, 147 ], 148 [ 149 'p99', 150 'p90', 151 'p50', 152 ], 153 ) + { 154 yaxes: $.yaxes('bytes'), 155 }, 156 ) 157 ) 158 .addRow( 159 $.row('Duration') 160 .addPanel( 161 $.panel('Chunk Duration hours (end-start)') + 162 $.queryPanel( 163 [ 164 'histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{%s}[5m])) by (le))' % dashboards['loki-chunks.json'].labelsSelector, 165 'histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{%s}[5m])) by (le))' % dashboards['loki-chunks.json'].labelsSelector, 166 'sum(rate(loki_ingester_chunk_bounds_hours_sum{%s}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{%s}[5m]))' % [ 167 dashboards['loki-chunks.json'].labelsSelector, 168 dashboards['loki-chunks.json'].labelsSelector, 169 ], 170 ], 171 [ 172 'p50', 173 'p99', 174 'avg', 175 ], 176 ), 177 ) 178 ), 179 }, 180 }