github.com/verrazzano/verrazzano@v1.7.0/platform-operator/thirdparty/charts/prometheus-community/kube-prometheus-stack/hack/sync_prometheus_rules.py (about) 1 #!/usr/bin/env python3 2 """Fetch alerting and aggregation rules from provided urls into this chart.""" 3 import json 4 import re 5 import textwrap 6 from os import makedirs 7 8 import _jsonnet 9 import requests 10 import yaml 11 from yaml.representer import SafeRepresenter 12 13 14 # https://stackoverflow.com/a/20863889/961092 15 class LiteralStr(str): 16 pass 17 18 19 def change_style(style, representer): 20 def new_representer(dumper, data): 21 scalar = representer(dumper, data) 22 scalar.style = style 23 return scalar 24 25 return new_representer 26 27 28 # Source files list 29 charts = [ 30 { 31 'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/alertmanager-prometheusRule.yaml', 32 'destination': '../templates/prometheus/rules-1.14', 33 'min_kubernetes': '1.14.0-0' 34 }, 35 { 36 'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubePrometheus-prometheusRule.yaml', 37 'destination': '../templates/prometheus/rules-1.14', 38 'min_kubernetes': '1.14.0-0' 39 }, 40 { 41 'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml', 42 'destination': '../templates/prometheus/rules-1.14', 43 'min_kubernetes': '1.14.0-0' 44 }, 45 { 46 'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubeStateMetrics-prometheusRule.yaml', 47 'destination': '../templates/prometheus/rules-1.14', 48 'min_kubernetes': '1.14.0-0' 49 }, 50 { 51 'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/nodeExporter-prometheusRule.yaml', 52 'destination': '../templates/prometheus/rules-1.14', 53 'min_kubernetes': '1.14.0-0' 54 }, 55 { 56 'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheus-prometheusRule.yaml', 57 'destination': '../templates/prometheus/rules-1.14', 58 'min_kubernetes': '1.14.0-0' 59 }, 60 { 61 'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheusOperator-prometheusRule.yaml', 62 'destination': '../templates/prometheus/rules-1.14', 63 'min_kubernetes': '1.14.0-0' 64 }, 65 { 66 'source': 'https://raw.githubusercontent.com/etcd-io/etcd/main/contrib/mixin/mixin.libsonnet', 67 'destination': '../templates/prometheus/rules-1.14', 68 'min_kubernetes': '1.14.0-0', 69 'is_mixin': True 70 }, 71 ] 72 73 # Additional conditions map 74 condition_map = { 75 'alertmanager.rules': ' .Values.defaultRules.rules.alertmanager', 76 'config-reloaders': ' .Values.defaultRules.rules.configReloaders', 77 'etcd': ' .Values.kubeEtcd.enabled .Values.defaultRules.rules.etcd', 78 'general.rules': ' .Values.defaultRules.rules.general', 79 'k8s.rules': ' .Values.defaultRules.rules.k8s', 80 'kube-apiserver-availability.rules': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability', 81 'kube-apiserver-burnrate.rules': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate', 82 'kube-apiserver-histogram.rules': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram', 83 'kube-apiserver-slos': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos', 84 'kube-prometheus-general.rules': ' .Values.defaultRules.rules.kubePrometheusGeneral', 85 'kube-prometheus-node-recording.rules': ' .Values.defaultRules.rules.kubePrometheusNodeRecording', 86 'kube-scheduler.rules': ' .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording', 87 'kube-state-metrics': ' .Values.defaultRules.rules.kubeStateMetrics', 88 'kubelet.rules': ' .Values.kubelet.enabled .Values.defaultRules.rules.kubelet', 89 'kubernetes-apps': ' .Values.defaultRules.rules.kubernetesApps', 90 'kubernetes-resources': ' .Values.defaultRules.rules.kubernetesResources', 91 'kubernetes-storage': ' .Values.defaultRules.rules.kubernetesStorage', 92 'kubernetes-system': ' .Values.defaultRules.rules.kubernetesSystem', 93 'kubernetes-system-kube-proxy': ' .Values.kubeProxy.enabled .Values.defaultRules.rules.kubeProxy', 94 'kubernetes-system-apiserver': ' .Values.defaultRules.rules.kubernetesSystem', # kubernetes-system was split into more groups in 1.14, one of them is kubernetes-system-apiserver 95 'kubernetes-system-kubelet': ' .Values.defaultRules.rules.kubernetesSystem', # kubernetes-system was split into more groups in 1.14, one of them is kubernetes-system-kubelet 96 'kubernetes-system-controller-manager': ' .Values.kubeControllerManager.enabled .Values.defaultRules.rules.kubeControllerManager', 97 'kubernetes-system-scheduler': ' .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting', 98 'node-exporter.rules': ' .Values.defaultRules.rules.nodeExporterRecording', 99 'node-exporter': ' .Values.defaultRules.rules.nodeExporterAlerting', 100 'node.rules': ' .Values.defaultRules.rules.node', 101 'node-network': ' .Values.defaultRules.rules.network', 102 'prometheus-operator': ' .Values.defaultRules.rules.prometheusOperator', 103 'prometheus': ' .Values.defaultRules.rules.prometheus', # kube-prometheus >= 1.14 uses prometheus as group instead of prometheus.rules 104 } 105 106 alert_condition_map = { 107 'AggregatedAPIDown': 'semverCompare ">=1.18.0-0" $kubeTargetVersion', 108 'AlertmanagerDown': '.Values.alertmanager.enabled', 109 'CoreDNSDown': '.Values.kubeDns.enabled', 110 'KubeAPIDown': '.Values.kubeApiServer.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics 111 'KubeControllerManagerDown': '.Values.kubeControllerManager.enabled', 112 'KubeletDown': '.Values.prometheusOperator.kubeletService.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics 113 'KubeSchedulerDown': '.Values.kubeScheduler.enabled', 114 'KubeStateMetricsDown': '.Values.kubeStateMetrics.enabled', # there are more alerts which are left enabled, because they'll never fire without metrics 115 'NodeExporterDown': '.Values.nodeExporter.enabled', 116 'PrometheusOperatorDown': '.Values.prometheusOperator.enabled', 117 } 118 119 replacement_map = { 120 'job="prometheus-operator"': { 121 'replacement': 'job="{{ $operatorJob }}"', 122 'init': '{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }}'}, 123 'job="prometheus-k8s"': { 124 'replacement': 'job="{{ $prometheusJob }}"', 125 'init': '{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }}'}, 126 'job="alertmanager-main"': { 127 'replacement': 'job="{{ $alertmanagerJob }}"', 128 'init': '{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }}'}, 129 'namespace="monitoring"': { 130 'replacement': 'namespace="{{ $namespace }}"', 131 'init': '{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }}'}, 132 'alertmanager-$1': { 133 'replacement': '$1', 134 'init': ''}, 135 'job="kube-state-metrics"': { 136 'replacement': 'job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"', 137 'limitGroup': ['kubernetes-apps'], 138 'init': '{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }}'}, 139 'job="kubelet"': { 140 'replacement': 'job="kubelet", namespace=~"{{ $targetNamespace }}"', 141 'limitGroup': ['kubernetes-storage'], 142 'init': '{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }}'}, 143 'runbook_url: https://runbooks.prometheus-operator.dev/runbooks/': { 144 'replacement': 'runbook_url: {{ .Values.defaultRules.runbookUrl }}/', 145 'init': ''}, 146 '(controller,namespace)': { 147 'replacement': '(controller,namespace,cluster)', 148 'init': ''} 149 } 150 151 # standard header 152 header = '''{{- /* 153 Generated from '%(name)s' group from %(url)s 154 Do not change in-place! In order to change this file first read following link: 155 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack 156 */ -}} 157 {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }} 158 {{- if and (semverCompare ">=%(min_kubernetes)s" $kubeTargetVersion) (semverCompare "<%(max_kubernetes)s" $kubeTargetVersion) .Values.defaultRules.create%(condition)s }}%(init_line)s 159 apiVersion: monitoring.coreos.com/v1 160 kind: PrometheusRule 161 metadata: 162 name: {{ printf "%%s-%%s" (include "kube-prometheus-stack.fullname" .) "%(name)s" | trunc 63 | trimSuffix "-" }} 163 namespace: {{ template "kube-prometheus-stack.namespace" . }} 164 labels: 165 app: {{ template "kube-prometheus-stack.name" . }} 166 {{ include "kube-prometheus-stack.labels" . | indent 4 }} 167 {{- if .Values.defaultRules.labels }} 168 {{ toYaml .Values.defaultRules.labels | indent 4 }} 169 {{- end }} 170 {{- if .Values.defaultRules.annotations }} 171 annotations: 172 {{ toYaml .Values.defaultRules.annotations | indent 4 }} 173 {{- end }} 174 spec: 175 groups: 176 -''' 177 178 179 def init_yaml_styles(): 180 represent_literal_str = change_style('|', SafeRepresenter.represent_str) 181 yaml.add_representer(LiteralStr, represent_literal_str) 182 183 184 def escape(s): 185 return s.replace("{{", "{{`{{").replace("}}", "}}`}}").replace("{{`{{", "{{`{{`}}").replace("}}`}}", "{{`}}`}}") 186 187 188 def fix_expr(rules): 189 """Remove trailing whitespaces and line breaks, which happen to creep in 190 due to yaml import specifics; 191 convert multiline expressions to literal style, |-""" 192 for rule in rules: 193 rule['expr'] = rule['expr'].rstrip() 194 if '\n' in rule['expr']: 195 rule['expr'] = LiteralStr(rule['expr']) 196 197 198 def yaml_str_repr(struct, indent=4): 199 """represent yaml as a string""" 200 text = yaml.dump( 201 struct, 202 width=1000, # to disable line wrapping 203 default_flow_style=False # to disable multiple items on single line 204 ) 205 text = escape(text) # escape {{ and }} for helm 206 text = textwrap.indent(text, ' ' * indent)[indent - 1:] # indent everything, and remove very first line extra indentation 207 return text 208 209 210 def add_rules_conditions(rules, rules_map, indent=4): 211 """Add if wrapper for rules, listed in rules_map""" 212 rule_condition = '{{- if %s }}\n' 213 for alert_name in rules_map: 214 line_start = ' ' * indent + '- alert: ' 215 if line_start + alert_name in rules: 216 rule_text = rule_condition % rules_map[alert_name] 217 start = 0 218 # to modify all alerts with same name 219 while True: 220 try: 221 # add if condition 222 index = rules.index(line_start + alert_name, start) 223 start = index + len(rule_text) + 1 224 rules = rules[:index] + rule_text + rules[index:] 225 # add end of if 226 try: 227 next_index = rules.index(line_start, index + len(rule_text) + 1) 228 except ValueError: 229 # we found the last alert in file if there are no alerts after it 230 next_index = len(rules) 231 232 # depending on the rule ordering in rules_map it's possible that an if statement from another rule is present at the end of this block. 233 found_block_end = False 234 last_line_index = next_index 235 while not found_block_end: 236 last_line_index = rules.rindex('\n', index, last_line_index - 1) # find the starting position of the last line 237 last_line = rules[last_line_index + 1:next_index] 238 239 if last_line.startswith('{{- if'): 240 next_index = last_line_index + 1 # move next_index back if the current block ends in an if statement 241 continue 242 243 found_block_end = True 244 rules = rules[:next_index] + '{{- end }}\n' + rules[next_index:] 245 except ValueError: 246 break 247 return rules 248 249 250 def add_rules_conditions_from_condition_map(rules, indent=4): 251 """Add if wrapper for rules, listed in alert_condition_map""" 252 rules = add_rules_conditions(rules, alert_condition_map, indent) 253 return rules 254 255 256 def add_rules_per_rule_conditions(rules, group, indent=4): 257 """Add if wrapper for rules, listed in alert_condition_map""" 258 rules_condition_map = {} 259 for rule in group['rules']: 260 if 'alert' in rule: 261 rules_condition_map[rule['alert']] = f"not (.Values.defaultRules.disabled.{rule['alert']} | default false)" 262 263 rules = add_rules_conditions(rules, rules_condition_map, indent) 264 return rules 265 266 267 def add_custom_labels(rules, indent=4): 268 """Add if wrapper for additional rules labels""" 269 rule_condition = '{{- if .Values.defaultRules.additionalRuleLabels }}\n{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}\n{{- end }}' 270 rule_condition_len = len(rule_condition) + 1 271 272 separator = " " * indent + "- alert:.*" 273 alerts_positions = re.finditer(separator,rules) 274 alert=-1 275 for alert_position in alerts_positions: 276 # add rule_condition at the end of the alert block 277 if alert >= 0 : 278 index = alert_position.start() + rule_condition_len * alert - 1 279 rules = rules[:index] + "\n" + rule_condition + rules[index:] 280 alert += 1 281 282 # add rule_condition at the end of the last alert 283 if alert >= 0: 284 index = len(rules) - 1 285 rules = rules[:index] + "\n" + rule_condition + rules[index:] 286 return rules 287 288 289 def add_custom_annotations(rules, indent=4): 290 """Add if wrapper for additional rules annotations""" 291 rule_condition = '{{- if .Values.defaultRules.additionalRuleAnnotations }}\n{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}\n{{- end }}' 292 annotations = " annotations:" 293 annotations_len = len(annotations) + 1 294 rule_condition_len = len(rule_condition) + 1 295 296 separator = " " * indent + "- alert:.*" 297 alerts_positions = re.finditer(separator,rules) 298 alert = 0 299 300 for alert_position in alerts_positions: 301 # Add rule_condition after 'annotations:' statement 302 index = alert_position.end() + annotations_len + rule_condition_len * alert 303 rules = rules[:index] + "\n" + rule_condition + rules[index:] 304 alert += 1 305 306 return rules 307 308 309 def write_group_to_file(group, url, destination, min_kubernetes, max_kubernetes): 310 fix_expr(group['rules']) 311 group_name = group['name'] 312 313 # prepare rules string representation 314 rules = yaml_str_repr(group) 315 # add replacements of custom variables and include their initialisation in case it's needed 316 init_line = '' 317 for line in replacement_map: 318 if group_name in replacement_map[line].get('limitGroup', [group_name]) and line in rules: 319 rules = rules.replace(line, replacement_map[line]['replacement']) 320 if replacement_map[line]['init']: 321 init_line += '\n' + replacement_map[line]['init'] 322 # append per-alert rules 323 rules = add_custom_labels(rules) 324 rules = add_custom_annotations(rules) 325 rules = add_rules_conditions_from_condition_map(rules) 326 rules = add_rules_per_rule_conditions(rules, group) 327 # initialize header 328 lines = header % { 329 'name': group['name'], 330 'url': url, 331 'condition': condition_map.get(group['name'], ''), 332 'init_line': init_line, 333 'min_kubernetes': min_kubernetes, 334 'max_kubernetes': max_kubernetes 335 } 336 337 # rules themselves 338 lines += rules 339 340 # footer 341 lines += '{{- end }}' 342 343 filename = group['name'] + '.yaml' 344 new_filename = "%s/%s" % (destination, filename) 345 346 # make sure directories to store the file exist 347 makedirs(destination, exist_ok=True) 348 349 # recreate the file 350 with open(new_filename, 'w') as f: 351 f.write(lines) 352 353 print("Generated %s" % new_filename) 354 355 def write_rules_names_template(): 356 with open('../templates/prometheus/_rules.tpl', 'w') as f: 357 f.write('''{{- /* 358 Generated file. Do not change in-place! In order to change this file first read following link: 359 https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack 360 */ -}}\n''') 361 f.write('{{- define "rules.names" }}\n') 362 f.write('rules:\n') 363 for rule in condition_map: 364 f.write(' - "%s"\n' % rule) 365 f.write('{{- end }}') 366 367 def main(): 368 init_yaml_styles() 369 # read the rules, create a new template file per group 370 for chart in charts: 371 print("Generating rules from %s" % chart['source']) 372 response = requests.get(chart['source']) 373 if response.status_code != 200: 374 print('Skipping the file, response code %s not equals 200' % response.status_code) 375 continue 376 raw_text = response.text 377 if chart.get('is_mixin'): 378 alerts = json.loads(_jsonnet.evaluate_snippet(chart['source'], raw_text + '.prometheusAlerts')) 379 else: 380 alerts = yaml.full_load(raw_text) 381 382 if ('max_kubernetes' not in chart): 383 chart['max_kubernetes']="9.9.9-9" 384 385 # etcd workaround, their file don't have spec level 386 groups = alerts['spec']['groups'] if alerts.get('spec') else alerts['groups'] 387 for group in groups: 388 write_group_to_file(group, chart['source'], chart['destination'], chart['min_kubernetes'], chart['max_kubernetes']) 389 390 # write rules.names named template 391 write_rules_names_template() 392 393 print("Finished") 394 395 396 if __name__ == '__main__': 397 main()