github.com/verrazzano/verrazzano@v1.7.0/platform-operator/thirdparty/charts/prometheus-community/kube-prometheus-stack/hack/sync_grafana_dashboards.py (about)

     1  #!/usr/bin/env python3
     2  """Fetch dashboards from provided urls into this chart."""
     3  import json
     4  import re
     5  import textwrap
     6  from os import makedirs, path
     7  
     8  import _jsonnet
     9  import requests
    10  import yaml
    11  from yaml.representer import SafeRepresenter
    12  
    13  
    14  # https://stackoverflow.com/a/20863889/961092
    15  class LiteralStr(str):
    16      pass
    17  
    18  
    19  def change_style(style, representer):
    20      def new_representer(dumper, data):
    21          scalar = representer(dumper, data)
    22          scalar.style = style
    23          return scalar
    24  
    25      return new_representer
    26  
    27  
    28  # Source files list
    29  charts = [
    30      {
    31          'source': 'https://raw.githubusercontent.com/prometheus-operator/kube-prometheus/main/manifests/grafana-dashboardDefinitions.yaml',
    32          'destination': '../templates/grafana/dashboards-1.14',
    33          'type': 'yaml',
    34          'min_kubernetes': '1.14.0-0',
    35          'multicluster_key': '.Values.grafana.sidecar.dashboards.multicluster.global.enabled',
    36      },
    37      {
    38          'source': 'https://raw.githubusercontent.com/etcd-io/etcd/main/contrib/mixin/mixin.libsonnet',
    39          'destination': '../templates/grafana/dashboards-1.14',
    40          'type': 'jsonnet_mixin',
    41          'min_kubernetes': '1.14.0-0',
    42          'multicluster_key': '(or .Values.grafana.sidecar.dashboards.multicluster.global.enabled .Values.grafana.sidecar.dashboards.multicluster.etcd.enabled)'
    43      },
    44  ]
    45  
    46  # Additional conditions map
    47  condition_map = {
    48      'grafana-coredns-k8s': ' .Values.coreDns.enabled',
    49      'etcd': ' .Values.kubeEtcd.enabled',
    50      'apiserver': ' .Values.kubeApiServer.enabled',
    51      'controller-manager': ' .Values.kubeControllerManager.enabled',
    52      'kubelet': ' .Values.kubelet.enabled',
    53      'proxy': ' .Values.kubeProxy.enabled',
    54      'scheduler': ' .Values.kubeScheduler.enabled',
    55      'node-rsrc-use': ' .Values.nodeExporter.enabled',
    56      'node-cluster-rsrc-use': ' .Values.nodeExporter.enabled',
    57      'nodes': ' .Values.nodeExporter.enabled',
    58      'nodes-darwin': ' .Values.nodeExporter.enabled',
    59      'prometheus-remote-write': ' .Values.prometheus.prometheusSpec.remoteWriteDashboards'
    60  }
    61  
    62  # standard header
    63  header = '''{{- /*
    64  Generated from '%(name)s' from %(url)s
    65  Do not change in-place! In order to change this file first read following link:
    66  https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-prometheus-stack/hack
    67  */ -}}
    68  {{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
    69  {{- if and (or .Values.grafana.enabled .Values.grafana.forceDeployDashboards) (semverCompare ">=%(min_kubernetes)s" $kubeTargetVersion) (semverCompare "<%(max_kubernetes)s" $kubeTargetVersion) .Values.grafana.defaultDashboardsEnabled%(condition)s }}
    70  apiVersion: v1
    71  kind: ConfigMap
    72  metadata:
    73    namespace: {{ template "kube-prometheus-stack-grafana.namespace" . }}
    74    name: {{ printf "%%s-%%s" (include "kube-prometheus-stack.fullname" $) "%(name)s" | trunc 63 | trimSuffix "-" }}
    75    annotations:
    76  {{ toYaml .Values.grafana.sidecar.dashboards.annotations | indent 4 }}
    77    labels:
    78      {{- if $.Values.grafana.sidecar.dashboards.label }}
    79      {{ $.Values.grafana.sidecar.dashboards.label }}: {{ ternary $.Values.grafana.sidecar.dashboards.labelValue "1" (not (empty $.Values.grafana.sidecar.dashboards.labelValue)) | quote }}
    80      {{- end }}
    81      app: {{ template "kube-prometheus-stack.name" $ }}-grafana
    82  {{ include "kube-prometheus-stack.labels" $ | indent 4 }}
    83  data:
    84  '''
    85  
    86  
    87  def init_yaml_styles():
    88      represent_literal_str = change_style('|', SafeRepresenter.represent_str)
    89      yaml.add_representer(LiteralStr, represent_literal_str)
    90  
    91  
    92  def escape(s):
    93      return s.replace("{{", "{{`{{").replace("}}", "}}`}}").replace("{{`{{", "{{`{{`}}").replace("}}`}}", "{{`}}`}}")
    94  
    95  
    96  def unescape(s):
    97      return s.replace("\{\{", "{{").replace("\}\}", "}}")
    98  
    99  
   100  def yaml_str_repr(struct, indent=2):
   101      """represent yaml as a string"""
   102      text = yaml.dump(
   103          struct,
   104          width=1000,  # to disable line wrapping
   105          default_flow_style=False  # to disable multiple items on single line
   106      )
   107      text = escape(text)  # escape {{ and }} for helm
   108      text = unescape(text)  # unescape \{\{ and \}\} for templating
   109      text = textwrap.indent(text, ' ' * indent)
   110      return text
   111  
   112  def patch_dashboards_json(content, multicluster_key):
   113      try:
   114          content_struct = json.loads(content)
   115  
   116          # multicluster
   117          overwrite_list = []
   118          for variable in content_struct['templating']['list']:
   119              if variable['name'] == 'cluster':
   120                  variable['hide'] = ':multicluster:'
   121              overwrite_list.append(variable)
   122          content_struct['templating']['list'] = overwrite_list
   123  
   124          # fix drilldown links. See https://github.com/kubernetes-monitoring/kubernetes-mixin/issues/659
   125          for row in content_struct['rows']:
   126              for panel in row['panels']:
   127                  for style in panel.get('styles', []):
   128                      if 'linkUrl' in style and style['linkUrl'].startswith('./d'):
   129                          style['linkUrl'] = style['linkUrl'].replace('./d', '/d')
   130  
   131          content_array = []
   132          original_content_lines = content.split('\n')
   133          for i, line in enumerate(json.dumps(content_struct, indent=4).split('\n')):
   134              if (' []' not in line and ' {}' not in line) or line == original_content_lines[i]:
   135                  content_array.append(line)
   136                  continue
   137  
   138              append = ''
   139              if line.endswith(','):
   140                  line = line[:-1]
   141                  append = ','
   142  
   143              if line.endswith('{}') or line.endswith('[]'):
   144                  content_array.append(line[:-1])
   145                  content_array.append('')
   146                  content_array.append(' ' * (len(line) - len(line.lstrip())) + line[-1] + append)
   147  
   148          content = '\n'.join(content_array)
   149  
   150          multicluster = content.find(':multicluster:')
   151          if multicluster != -1:
   152              content = ''.join((
   153                  content[:multicluster-1],
   154                  '\{\{ if %s \}\}0\{\{ else \}\}2\{\{ end \}\}' % multicluster_key,
   155                  content[multicluster + 15:]
   156              ))
   157      except (ValueError, KeyError):
   158          pass
   159  
   160      return content
   161  
   162  
   163  def patch_json_set_timezone_as_variable(content):
   164      # content is no more in json format, so we have to replace using regex
   165      return re.sub(r'"timezone"\s*:\s*"(?:\\.|[^\"])*"', '"timezone": "\{\{ .Values.grafana.defaultDashboardsTimezone \}\}"', content, flags=re.IGNORECASE)
   166  
   167  
   168  def write_group_to_file(resource_name, content, url, destination, min_kubernetes, max_kubernetes, multicluster_key):
   169      # initialize header
   170      lines = header % {
   171          'name': resource_name,
   172          'url': url,
   173          'condition': condition_map.get(resource_name, ''),
   174          'min_kubernetes': min_kubernetes,
   175          'max_kubernetes': max_kubernetes
   176      }
   177  
   178      content = patch_dashboards_json(content, multicluster_key)
   179      content = patch_json_set_timezone_as_variable(content)
   180  
   181      filename_struct = {resource_name + '.json': (LiteralStr(content))}
   182      # rules themselves
   183      lines += yaml_str_repr(filename_struct)
   184  
   185      # footer
   186      lines += '{{- end }}'
   187  
   188      filename = resource_name + '.yaml'
   189      new_filename = "%s/%s" % (destination, filename)
   190  
   191      # make sure directories to store the file exist
   192      makedirs(destination, exist_ok=True)
   193  
   194      # recreate the file
   195      with open(new_filename, 'w') as f:
   196          f.write(lines)
   197  
   198      print("Generated %s" % new_filename)
   199  
   200  
   201  def main():
   202      init_yaml_styles()
   203      # read the rules, create a new template file per group
   204      for chart in charts:
   205          print("Generating rules from %s" % chart['source'])
   206          response = requests.get(chart['source'])
   207          if response.status_code != 200:
   208              print('Skipping the file, response code %s not equals 200' % response.status_code)
   209              continue
   210          raw_text = response.text
   211  
   212          if ('max_kubernetes' not in chart):
   213              chart['max_kubernetes']="9.9.9-9"
   214  
   215          if chart['type'] == 'yaml':
   216              yaml_text = yaml.full_load(raw_text)
   217              groups = yaml_text['items']
   218              for group in groups:
   219                  for resource, content in group['data'].items():
   220                      write_group_to_file(resource.replace('.json', ''), content, chart['source'], chart['destination'], chart['min_kubernetes'], chart['max_kubernetes'], chart['multicluster_key'])
   221          elif chart['type'] == 'jsonnet_mixin':
   222              json_text = json.loads(_jsonnet.evaluate_snippet(chart['source'], raw_text + '.grafanaDashboards'))
   223              # is it already a dashboard structure or is it nested (etcd case)?
   224              flat_structure = bool(json_text.get('annotations'))
   225              if flat_structure:
   226                  resource = path.basename(chart['source']).replace('.json', '')
   227                  write_group_to_file(resource, json.dumps(json_text, indent=4), chart['source'], chart['destination'], chart['min_kubernetes'], chart['max_kubernetes'], chart['multicluster_key'])
   228              else:
   229                  for resource, content in json_text.items():
   230                      write_group_to_file(resource.replace('.json', ''), json.dumps(content, indent=4), chart['source'], chart['destination'], chart['min_kubernetes'], chart['max_kubernetes'], chart['multicluster_key'])
   231      print("Finished")
   232  
   233  
   234  if __name__ == '__main__':
   235      main()