github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/cloud/kubernetes/fluentd-configmap.yml (about)

     1  # This ConfigMap is used to ingest logs against new resources like "k8s_container".
     2  # The configuration instruct Fluentd deamons to correctly parse CockroachDB logs,
     3  # extracting timestamp, severity and log message into the log output format for follow on output stanzas to process.
     4  # This is tested on Google Cloud Platform using a custom deployment of Stackdriver:
     5  # https://cloud.google.com/solutions/customizing-stackdriver-logs-fluentd
     6  
     7  kind: ConfigMap
     8  apiVersion: v1
     9  metadata:
    10    name: fluentd-config
    11    namespace: kube-system
    12    labels:
    13      addonmanager.kubernetes.io/mode: Reconcile
    14  data:
    15    containers.input.conf: |-
    16      # Read and tail all log files on the node
    17      # Match timestamp with one of the patterns, and prefix the tag with "reform".
    18      <source>
    19        @type tail
    20        path /var/log/containers/*.log
    21        pos_file /var/log/containers.log.pos
    22        # Tags at this point are in the format of:
    23        # reform.var.log.containers.<POD_NAME>_<NAMESPACE_NAME>_<CONTAINER_NAME>-<CONTAINER_ID>.log
    24        tag reform.*
    25        read_from_head true
    26        format multi_format
    27        <pattern>
    28          format json
    29          time_key time
    30          time_format %Y-%m-%dT%H:%M:%S.%NZ
    31        </pattern>
    32        <pattern>
    33          format /^(?<time>.+) (?<stream>stdout|stderr) [^ ]* (?<log>.*)$/
    34          time_format %Y-%m-%dT%H:%M:%S.%N%:z
    35        </pattern>
    36      </source>
    37  
    38      # This filter tries to parse the log records. The follow on filters will override the values set by this filter.
    39      # This is for all containers that don't match the follow on filters - effectively treat this as the 'default' case.
    40      # Note: Fluentd processes the log file on every 'Filter' that matches the pattern,
    41      # however Fluentd allows only one 'Match' per pattern.
    42      <filter reform.**>
    43        @type parser
    44        format /^(?<severity>\w)(?<time>\d{4} [^\s]*)\s+(?<pid>\d+)\s+(?<source>[^ \]]+)\] (?<log>.*)/
    45        reserve_data true
    46        suppress_parse_error_log true
    47        emit_invalid_record_to_error false
    48        key_name log
    49      </filter>
    50  
    51      # Reprocess the log line, this time match on particular formats.
    52      # You can add your own custom format in this section.
    53      # For CockroachDB, we are assigning the severity, time and log as capture groups.
    54      <filter reform.**>
    55        @type parser
    56        key_name log
    57        reserve_data true
    58        suppress_parse_error_log true
    59        emit_invalid_record_to_error false
    60        format multi_format
    61        <pattern>
    62          # CockroachDB log format
    63          format /^(?<severity>\w{1})(?<time>\d{6}\s\d{2}:\d{2}:\d{2})\.\d{6}\s*(?<log>.*)/
    64          time_key time
    65          time_format %y%m%d %H:%M:%S.%N
    66        </pattern>
    67      </filter>
    68  
    69      # Match only on cockroachdb log records, so we can normalize the severity format.
    70      # This is useful for output pipelines that expect certain format (like Stackdriver)
    71      # After this we don't need to further process records, so we'll tag the log record with "parsed" to indicate as such.
    72      <match reform.var.log.containers.cockroachdb**>
    73        @type record_reformer
    74        enable_ruby true
    75        tag parsed.${tag_suffix[1]}
    76        <record>
    77          severity ${ if (record["severity"] == "E") then "error" elsif (record["severity"] == "W") then "warning" elsif (record["severity"] == "I") then "info" elsif (record["severity"] == "D") then "debug" else record["severity"] end}
    78        </record>
    79      </match>
    80  
    81      # After this we don't need to further process records, so we'll tag the log record with "parsed" to indicate as such.
    82      <match reform.**>
    83        @type record_reformer
    84        enable_ruby true
    85        tag parsed.${tag_suffix[1]}
    86      </match>
    87  
    88      # We remove the prefix "parsed.reform.var.log.containers" and replace it with the format below
    89      <match parsed.**>
    90        @type record_reformer
    91        enable_ruby true
    92        # Tags at this point are in the format of:
    93        # 'raw.kubernetes.<POD_NAME>_<NAMESPACE_NAME>_<CONTAINER_NAME>'.
    94        tag raw.kubernetes.${tag_suffix[4].split('-')[0..-2].join('-')}
    95      </match>
    96  
    97      <match raw.kubernetes.**>
    98        #########
    99        # NOTE: You must replace this with your log inject pipeline
   100        #########
   101        type stdout
   102      </match>