github.com/verrazzano/verrazzano@v1.7.0/platform-operator/thirdparty/charts/prometheus-community/kube-prometheus-stack/hack/sync_prometheus_rules.py (about)

     1  #!/usr/bin/env python3
     2  """Fetch alerting and aggregation rules from provided urls into this chart."""
     3  import json
     4  import re
     5  import textwrap
     6  from os import makedirs
     7  
     8  import _jsonnet
     9  import requests
    10  import yaml
    11  from yaml.representer import SafeRepresenter
    12  
    13  
    14  # https://stackoverflow.com/a/20863889/961092
    15  class LiteralStr(str):
    16      pass
    17  
    18  
    19  def change_style(style, representer):
    20      def new_representer(dumper, data):
    21          scalar = representer(dumper, data)
    22          scalar.style = style
    23          return scalar
    24  
    25      return new_representer
    26  
    27  
    28  # Source files list
    29  charts = [
    30      {
    31          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/alertmanager-prometheusRule.yaml',
    32          'destination': '../templates/prometheus/rules-1.14',
    33          'min_kubernetes': '1.14.0-0'
    34      },
    35      {
    36          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubePrometheus-prometheusRule.yaml',
    37          'destination': '../templates/prometheus/rules-1.14',
    38          'min_kubernetes': '1.14.0-0'
    39      },
    40      {
    41          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubernetesControlPlane-prometheusRule.yaml',
    42          'destination': '../templates/prometheus/rules-1.14',
    43          'min_kubernetes': '1.14.0-0'
    44      },
    45      {
    46          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/kubeStateMetrics-prometheusRule.yaml',
    47          'destination': '../templates/prometheus/rules-1.14',
    48          'min_kubernetes': '1.14.0-0'
    49      },
    50      {
    51          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/nodeExporter-prometheusRule.yaml',
    52          'destination': '../templates/prometheus/rules-1.14',
    53          'min_kubernetes': '1.14.0-0'
    54      },
    55      {
    56          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheus-prometheusRule.yaml',
    57          'destination': '../templates/prometheus/rules-1.14',
    58          'min_kubernetes': '1.14.0-0'
    59      },
    60      {
    61          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/prometheusOperator-prometheusRule.yaml',
    62          'destination': '../templates/prometheus/rules-1.14',
    63          'min_kubernetes': '1.14.0-0'
    64      },
    65      {
    66          'source': 'https://raw.githubusercontent.com/etcd-io/etcd/main/contrib/mixin/mixin.libsonnet',
    67          'destination': '../templates/prometheus/rules-1.14',
    68          'min_kubernetes': '1.14.0-0',
    69          'is_mixin': True
    70      },
    71  ]
    72  
    73  # Additional conditions map
    74  condition_map = {
    75      'alertmanager.rules': ' .Values.defaultRules.rules.alertmanager',
    76      'config-reloaders': ' .Values.defaultRules.rules.configReloaders',
    77      'etcd': ' .Values.kubeEtcd.enabled .Values.defaultRules.rules.etcd',
    78      'general.rules': ' .Values.defaultRules.rules.general',
    79      'k8s.rules': ' .Values.defaultRules.rules.k8s',
    80      'kube-apiserver-availability.rules': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverAvailability',
    81      'kube-apiserver-burnrate.rules': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverBurnrate',
    82      'kube-apiserver-histogram.rules': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverHistogram',
    83      'kube-apiserver-slos': ' .Values.kubeApiServer.enabled .Values.defaultRules.rules.kubeApiserverSlos',
    84      'kube-prometheus-general.rules': ' .Values.defaultRules.rules.kubePrometheusGeneral',
    85      'kube-prometheus-node-recording.rules': ' .Values.defaultRules.rules.kubePrometheusNodeRecording',
    86      'kube-scheduler.rules': ' .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerRecording',
    87      'kube-state-metrics': ' .Values.defaultRules.rules.kubeStateMetrics',
    88      'kubelet.rules': ' .Values.kubelet.enabled .Values.defaultRules.rules.kubelet',
    89      'kubernetes-apps': ' .Values.defaultRules.rules.kubernetesApps',
    90      'kubernetes-resources': ' .Values.defaultRules.rules.kubernetesResources',
    91      'kubernetes-storage': ' .Values.defaultRules.rules.kubernetesStorage',
    92      'kubernetes-system': ' .Values.defaultRules.rules.kubernetesSystem',
    93      'kubernetes-system-kube-proxy': ' .Values.kubeProxy.enabled .Values.defaultRules.rules.kubeProxy',
    94      'kubernetes-system-apiserver': ' .Values.defaultRules.rules.kubernetesSystem', # kubernetes-system was split into more groups in 1.14, one of them is kubernetes-system-apiserver
    95      'kubernetes-system-kubelet': ' .Values.defaultRules.rules.kubernetesSystem', # kubernetes-system was split into more groups in 1.14, one of them is kubernetes-system-kubelet
    96      'kubernetes-system-controller-manager': ' .Values.kubeControllerManager.enabled .Values.defaultRules.rules.kubeControllerManager',
    97      'kubernetes-system-scheduler': ' .Values.kubeScheduler.enabled .Values.defaultRules.rules.kubeSchedulerAlerting',
    98      'node-exporter.rules': ' .Values.defaultRules.rules.nodeExporterRecording',
    99      'node-exporter': ' .Values.defaultRules.rules.nodeExporterAlerting',
   100      'node.rules': ' .Values.defaultRules.rules.node',
   101      'node-network': ' .Values.defaultRules.rules.network',
   102      'prometheus-operator': ' .Values.defaultRules.rules.prometheusOperator',
   103      'prometheus': ' .Values.defaultRules.rules.prometheus', # kube-prometheus >= 1.14 uses prometheus as group instead of prometheus.rules
   104  }
   105  
   106  alert_condition_map = {
   107      'AggregatedAPIDown': 'semverCompare ">=1.18.0-0" $kubeTargetVersion',
   108      'AlertmanagerDown': '.Values.alertmanager.enabled',
   109      'CoreDNSDown': '.Values.kubeDns.enabled',
   110      'KubeAPIDown': '.Values.kubeApiServer.enabled',  # there are more alerts which are left enabled, because they'll never fire without metrics
   111      'KubeControllerManagerDown': '.Values.kubeControllerManager.enabled',
   112      'KubeletDown': '.Values.prometheusOperator.kubeletService.enabled',  # there are more alerts which are left enabled, because they'll never fire without metrics
   113      'KubeSchedulerDown': '.Values.kubeScheduler.enabled',
   114      'KubeStateMetricsDown': '.Values.kubeStateMetrics.enabled',  # there are more alerts which are left enabled, because they'll never fire without metrics
   115      'NodeExporterDown': '.Values.nodeExporter.enabled',
   116      'PrometheusOperatorDown': '.Values.prometheusOperator.enabled',
   117  }
   118  
   119  replacement_map = {
   120      'job="prometheus-operator"': {
   121          'replacement': 'job="{{ $operatorJob }}"',
   122          'init': '{{- $operatorJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "operator" }}'},
   123      'job="prometheus-k8s"': {
   124          'replacement': 'job="{{ $prometheusJob }}"',
   125          'init': '{{- $prometheusJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "prometheus" }}'},
   126      'job="alertmanager-main"': {
   127          'replacement': 'job="{{ $alertmanagerJob }}"',
   128          'init': '{{- $alertmanagerJob := printf "%s-%s" (include "kube-prometheus-stack.fullname" .) "alertmanager" }}'},
   129      'namespace="monitoring"': {
   130          'replacement': 'namespace="{{ $namespace }}"',
   131          'init': '{{- $namespace := printf "%s" (include "kube-prometheus-stack.namespace" .) }}'},
   132      'alertmanager-$1': {
   133          'replacement': '$1',
   134          'init': ''},
   135      'job="kube-state-metrics"': {
   136          'replacement': 'job="kube-state-metrics", namespace=~"{{ $targetNamespace }}"',
   137          'limitGroup': ['kubernetes-apps'],
   138          'init': '{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }}'},
   139      'job="kubelet"': {
   140          'replacement': 'job="kubelet", namespace=~"{{ $targetNamespace }}"',
   141          'limitGroup': ['kubernetes-storage'],
   142          'init': '{{- $targetNamespace := .Values.defaultRules.appNamespacesTarget }}'},
   143      'runbook_url: https://runbooks.prometheus-operator.dev/runbooks/': {
   144          'replacement': 'runbook_url: {{ .Values.defaultRules.runbookUrl }}/',
   145          'init': ''},
   146      '(controller,namespace)': {
   147          'replacement': '(controller,namespace,cluster)',
   148          'init': ''}
   149  }
   150  
   151  # standard header
   152  header = '''{{- /*
   153  Generated from '%(name)s' group from %(url)s
   154  Do not change in-place! In order to change this file first read following link:
   155  https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
   156  */ -}}
   157  {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
   158  {{- if and (semverCompare ">=%(min_kubernetes)s" $kubeTargetVersion) (semverCompare "<%(max_kubernetes)s" $kubeTargetVersion) .Values.defaultRules.create%(condition)s }}%(init_line)s
   159  apiVersion: monitoring.coreos.com/v1
   160  kind: PrometheusRule
   161  metadata:
   162    name: {{ printf "%%s-%%s" (include "kube-prometheus-stack.fullname" .) "%(name)s" | trunc 63 | trimSuffix "-" }}
   163    namespace: {{ template "kube-prometheus-stack.namespace" . }}
   164    labels:
   165      app: {{ template "kube-prometheus-stack.name" . }}
   166  {{ include "kube-prometheus-stack.labels" . | indent 4 }}
   167  {{- if .Values.defaultRules.labels }}
   168  {{ toYaml .Values.defaultRules.labels | indent 4 }}
   169  {{- end }}
   170  {{- if .Values.defaultRules.annotations }}
   171    annotations:
   172  {{ toYaml .Values.defaultRules.annotations | indent 4 }}
   173  {{- end }}
   174  spec:
   175    groups:
   176    -'''
   177  
   178  
   179  def init_yaml_styles():
   180      represent_literal_str = change_style('|', SafeRepresenter.represent_str)
   181      yaml.add_representer(LiteralStr, represent_literal_str)
   182  
   183  
   184  def escape(s):
   185      return s.replace("{{", "{{`{{").replace("}}", "}}`}}").replace("{{`{{", "{{`{{`}}").replace("}}`}}", "{{`}}`}}")
   186  
   187  
   188  def fix_expr(rules):
   189      """Remove trailing whitespaces and line breaks, which happen to creep in
   190       due to yaml import specifics;
   191       convert multiline expressions to literal style, |-"""
   192      for rule in rules:
   193          rule['expr'] = rule['expr'].rstrip()
   194          if '\n' in rule['expr']:
   195              rule['expr'] = LiteralStr(rule['expr'])
   196  
   197  
   198  def yaml_str_repr(struct, indent=4):
   199      """represent yaml as a string"""
   200      text = yaml.dump(
   201          struct,
   202          width=1000,  # to disable line wrapping
   203          default_flow_style=False  # to disable multiple items on single line
   204      )
   205      text = escape(text)  # escape {{ and }} for helm
   206      text = textwrap.indent(text, ' ' * indent)[indent - 1:]  # indent everything, and remove very first line extra indentation
   207      return text
   208  
   209  
   210  def add_rules_conditions(rules, rules_map, indent=4):
   211      """Add if wrapper for rules, listed in rules_map"""
   212      rule_condition = '{{- if %s }}\n'
   213      for alert_name in rules_map:
   214          line_start = ' ' * indent + '- alert: '
   215          if line_start + alert_name in rules:
   216              rule_text = rule_condition % rules_map[alert_name]
   217              start = 0
   218              # to modify all alerts with same name
   219              while True:
   220                  try:
   221                      # add if condition
   222                      index = rules.index(line_start + alert_name, start)
   223                      start = index + len(rule_text) + 1
   224                      rules = rules[:index] + rule_text + rules[index:]
   225                      # add end of if
   226                      try:
   227                          next_index = rules.index(line_start, index + len(rule_text) + 1)
   228                      except ValueError:
   229                          # we found the last alert in file if there are no alerts after it
   230                          next_index = len(rules)
   231  
   232                      # depending on the rule ordering in rules_map it's possible that an if statement from another rule is present at the end of this block.
   233                      found_block_end = False
   234                      last_line_index = next_index
   235                      while not found_block_end:
   236                          last_line_index = rules.rindex('\n', index, last_line_index - 1)  # find the starting position of the last line
   237                          last_line = rules[last_line_index + 1:next_index]
   238  
   239                          if last_line.startswith('{{- if'):
   240                              next_index = last_line_index + 1  # move next_index back if the current block ends in an if statement
   241                              continue
   242  
   243                          found_block_end = True
   244                      rules = rules[:next_index] + '{{- end }}\n' + rules[next_index:]
   245                  except ValueError:
   246                      break
   247      return rules
   248  
   249  
   250  def add_rules_conditions_from_condition_map(rules, indent=4):
   251      """Add if wrapper for rules, listed in alert_condition_map"""
   252      rules = add_rules_conditions(rules, alert_condition_map, indent)
   253      return rules
   254  
   255  
   256  def add_rules_per_rule_conditions(rules, group, indent=4):
   257      """Add if wrapper for rules, listed in alert_condition_map"""
   258      rules_condition_map = {}
   259      for rule in group['rules']:
   260          if 'alert' in rule:
   261              rules_condition_map[rule['alert']] = f"not (.Values.defaultRules.disabled.{rule['alert']} | default false)"
   262  
   263      rules = add_rules_conditions(rules, rules_condition_map, indent)
   264      return rules
   265  
   266  
   267  def add_custom_labels(rules, indent=4):
   268      """Add if wrapper for additional rules labels"""
   269      rule_condition = '{{- if .Values.defaultRules.additionalRuleLabels }}\n{{ toYaml .Values.defaultRules.additionalRuleLabels | indent 8 }}\n{{- end }}'
   270      rule_condition_len = len(rule_condition) + 1
   271  
   272      separator = " " * indent + "- alert:.*"
   273      alerts_positions = re.finditer(separator,rules)
   274      alert=-1
   275      for alert_position in alerts_positions:
   276          # add rule_condition at the end of the alert block
   277          if alert >= 0 :
   278              index = alert_position.start() + rule_condition_len * alert - 1
   279              rules = rules[:index] + "\n" + rule_condition + rules[index:]
   280          alert += 1
   281  
   282      # add rule_condition at the end of the last alert
   283      if alert >= 0:
   284          index = len(rules) - 1
   285          rules = rules[:index] + "\n" + rule_condition + rules[index:]
   286      return rules
   287  
   288  
   289  def add_custom_annotations(rules, indent=4):
   290      """Add if wrapper for additional rules annotations"""
   291      rule_condition = '{{- if .Values.defaultRules.additionalRuleAnnotations }}\n{{ toYaml .Values.defaultRules.additionalRuleAnnotations | indent 8 }}\n{{- end }}'
   292      annotations = "      annotations:"
   293      annotations_len = len(annotations) + 1
   294      rule_condition_len = len(rule_condition) + 1
   295  
   296      separator = " " * indent + "- alert:.*"
   297      alerts_positions = re.finditer(separator,rules)
   298      alert = 0
   299  
   300      for alert_position in alerts_positions:
   301          # Add rule_condition after 'annotations:' statement
   302          index = alert_position.end() + annotations_len + rule_condition_len * alert
   303          rules = rules[:index] + "\n" + rule_condition + rules[index:]
   304          alert += 1
   305  
   306      return rules
   307  
   308  
   309  def write_group_to_file(group, url, destination, min_kubernetes, max_kubernetes):
   310      fix_expr(group['rules'])
   311      group_name = group['name']
   312  
   313      # prepare rules string representation
   314      rules = yaml_str_repr(group)
   315      # add replacements of custom variables and include their initialisation in case it's needed
   316      init_line = ''
   317      for line in replacement_map:
   318          if group_name in replacement_map[line].get('limitGroup', [group_name]) and line in rules:
   319              rules = rules.replace(line, replacement_map[line]['replacement'])
   320              if replacement_map[line]['init']:
   321                  init_line += '\n' + replacement_map[line]['init']
   322      # append per-alert rules
   323      rules = add_custom_labels(rules)
   324      rules = add_custom_annotations(rules)
   325      rules = add_rules_conditions_from_condition_map(rules)
   326      rules = add_rules_per_rule_conditions(rules, group)
   327      # initialize header
   328      lines = header % {
   329          'name': group['name'],
   330          'url': url,
   331          'condition': condition_map.get(group['name'], ''),
   332          'init_line': init_line,
   333          'min_kubernetes': min_kubernetes,
   334          'max_kubernetes': max_kubernetes
   335      }
   336  
   337      # rules themselves
   338      lines += rules
   339  
   340      # footer
   341      lines += '{{- end }}'
   342  
   343      filename = group['name'] + '.yaml'
   344      new_filename = "%s/%s" % (destination, filename)
   345  
   346      # make sure directories to store the file exist
   347      makedirs(destination, exist_ok=True)
   348  
   349      # recreate the file
   350      with open(new_filename, 'w') as f:
   351          f.write(lines)
   352  
   353      print("Generated %s" % new_filename)
   354  
   355  def write_rules_names_template():
   356      with open('../templates/prometheus/_rules.tpl', 'w') as f:
   357          f.write('''{{- /*
   358  Generated file. Do not change in-place! In order to change this file first read following link:
   359  https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
   360  */ -}}\n''')
   361          f.write('{{- define "rules.names" }}\n')
   362          f.write('rules:\n')
   363          for rule in condition_map:
   364              f.write('  - "%s"\n' % rule)
   365          f.write('{{- end }}')
   366  
   367  def main():
   368      init_yaml_styles()
   369      # read the rules, create a new template file per group
   370      for chart in charts:
   371          print("Generating rules from %s" % chart['source'])
   372          response = requests.get(chart['source'])
   373          if response.status_code != 200:
   374              print('Skipping the file, response code %s not equals 200' % response.status_code)
   375              continue
   376          raw_text = response.text
   377          if chart.get('is_mixin'):
   378              alerts = json.loads(_jsonnet.evaluate_snippet(chart['source'], raw_text + '.prometheusAlerts'))
   379          else:
   380              alerts = yaml.full_load(raw_text)
   381  
   382          if ('max_kubernetes' not in chart):
   383              chart['max_kubernetes']="9.9.9-9"
   384  
   385          # etcd workaround, their file don't have spec level
   386          groups = alerts['spec']['groups'] if alerts.get('spec') else alerts['groups']
   387          for group in groups:
   388              write_group_to_file(group, chart['source'], chart['destination'], chart['min_kubernetes'], chart['max_kubernetes'])
   389  
   390      # write rules.names named template
   391      write_rules_names_template()
   392  
   393      print("Finished")
   394  
   395  
   396  if __name__ == '__main__':
   397      main()