github.com/verrazzano/verrazzano@v1.7.1/platform-operator/helm_config/charts/verrazzano-fluentd/templates/fluentd-config-configmap.yaml (about) 1 # Copyright (c) 2022, 2023, Oracle and/or its affiliates. 2 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 apiVersion: v1 5 kind: ConfigMap 6 metadata: 7 name: {{ .Values.logging.name }}-config 8 namespace: {{ .Release.Namespace }} 9 labels: 10 app: {{ .Values.logging.name }} 11 data: 12 fluent.conf: | 13 # Use the config specified by the FLUENTD_CONFIG environment variable, or 14 # default to fluentd-standalone.conf 15 @include "#{ENV['FLUENTD_CONFIG'] || 'fluentd-standalone.conf'}" 16 17 # A config for running Fluentd as a daemon which collects, filters, parses, 18 # and sends log to storage. No extra Fluentd processes required. 19 fluentd-standalone.conf: | 20 # Common config 21 @include general.conf 22 @include prometheus.conf 23 24 # Input sources 25 @include systemd-input.conf 26 @include kubernetes-input.conf 27 28 # Parsing/Filtering 29 @include systemd-filter.conf 30 @include kubernetes-filter.conf 31 @include components-filter.conf 32 33 # Send to storage 34 @include output.conf 35 {{- if .Values.fluentd.oci }} 36 # Start namespace logging configs 37 # End namespace logging configs 38 {{- if .Values.fluentd.oci.systemLogId }} 39 @include oci-logging-system.conf 40 {{- if .Values.fluentd.oci.defaultAppLogId }} 41 @include oci-logging-default-app.conf 42 {{- end }} 43 {{- end }} 44 {{- else }} 45 @include es-output.conf 46 {{- end }} 47 48 general.conf: | 49 # Prevent Fluentd from handling records containing its own logs. Otherwise 50 # it can lead to an infinite loop, when error in sending one message generates 51 # another message which also fails to be sent and so on. 52 <label @FLUENT_LOG> 53 <match fluent.*> 54 @type null 55 </match> 56 </label> 57 58 # Used for health checking 59 <source> 60 @type http 61 @id in_http 62 port 9880 63 bind 0.0.0.0 64 </source> 65 66 # Emits internal metrics to every minute, and also exposes them on port 67 # 24220. Useful for determining if an output plugin is retrying/erroring, 68 # or determining the buffer queue length. 69 <source> 70 @type monitor_agent 71 @id in_monitor_agent 72 bind 0.0.0.0 73 port 24220 74 </source> 75 76 prometheus.conf: | 77 # Prometheus Exporter Plugin 78 # input plugin that exports metrics 79 <source> 80 @type prometheus 81 port 24231 82 metrics_path /metrics 83 </source> 84 # input plugin that collects metrics from MonitorAgent 85 <source> 86 @type prometheus_monitor 87 <labels> 88 host ${hostname} 89 </labels> 90 </source> 91 # input plugin that collects metrics for output plugin 92 <source> 93 @type prometheus_output_monitor 94 <labels> 95 host ${hostname} 96 </labels> 97 </source> 98 # input plugin that collects metrics for in_tail plugin 99 <source> 100 @type prometheus_tail_monitor 101 <labels> 102 host ${hostname} 103 </labels> 104 </source> 105 106 systemd-input.conf: | 107 <source> 108 @type systemd 109 @id in_systemd_run 110 read_from_head true 111 tag systemd 112 path /run/log/journal 113 <storage> 114 @type local 115 persistent true 116 path /tmp/run_journald_pos.json 117 </storage> 118 <entry> 119 fields_strip_underscores true 120 </entry> 121 </source> 122 123 systemd-filter.conf: | 124 <filter systemd> 125 @type record_transformer 126 @id systemd_index 127 <record> 128 tag systemd 129 </record> 130 </filter> 131 132 <filter systemd.kubelet> 133 @type parser 134 @id systemd_kubelet_parser 135 format kubernetes 136 reserve_data true 137 key_name MESSAGE 138 </filter> 139 140 <filter systemd.docker> 141 @type parser 142 @id systemd_docker_parser 143 format /^time="(?<time>[^)]*)" level=(?<severity>[^ ]*) msg="(?<message>[^"]*)"( err="(?<error>[^"]*)")?( statusCode=($<status_code>\d+))?/ 144 reserve_data true 145 key_name MESSAGE 146 </filter> 147 148 # Filter ssh logs since it's mostly bots trying to login 149 <filter systemd.**> 150 @type grep 151 @id systemd_grep 152 <exclude> 153 key SYSTEMD_UNIT 154 pattern (sshd@.*\.service) 155 </exclude> 156 </filter> 157 158 kubernetes-input.conf: | 159 # Capture Kubernetes pod logs 160 # The kubelet creates symlinks that capture the pod name, namespace, 161 # container name & Docker container ID to the docker logs for pods in the 162 # /var/log/containers directory on the host. 163 <source> 164 @type tail 165 # @id in_tail 166 path /var/log/containers/*.log 167 pos_file /var/log/vz-fluentd-containers.log.pos 168 # Exclude the log of the Fluentd daemonset itself 169 exclude_path ["/var/log/containers/fluentd*_verrazzano-system_fluentd*.log"] 170 tag kubernetes.* 171 read_from_head true 172 # @log_level debug 173 <parse> 174 @type multi_format 175 <pattern> 176 format json 177 time_format %Y-%m-%dT%H:%M:%S.%NZ 178 </pattern> 179 # KIND CRI pattern/format 180 <pattern> 181 format /^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<flags>[^ ]+) (?<log>.*)$/ 182 time_format %Y-%m-%dT%H:%M:%S.%NZ 183 </pattern> 184 # OKE v1.20.8 185 <pattern> 186 format /^(?<time>[^ ]+) (?<stream>stdout|stderr) (?<flags>[^ ]+) (?<log>.*)$/ 187 time_format %Y-%m-%dT%H:%M:%S.%N%:z 188 </pattern> 189 </parse> 190 </source> 191 192 components-filter.conf: | 193 # filter to parse istio-proxy and istiod container log files 194 <filter kubernetes.**istio-proxy** kubernetes.**istiod**istio-system_discovery**> 195 @type parser 196 @id istio 197 key_name log 198 reserve_data true 199 emit_invalid_record_to_error true 200 <parse> 201 # istio containers have two formats for log records 202 # one has a timestamp, log level and other fields 203 # the other is just a messsage 204 @type multi_format 205 <pattern> 206 format /^(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}Z)\t(?<level>.*?)\t(?<message>[\s\S]*?)$/ 207 time_key logtime 208 time_format %Y-%m-%dT%H:%M:%S.%NZ 209 </pattern> 210 <pattern> 211 format none 212 </pattern> 213 </parse> 214 </filter> 215 216 # filter to parse opensearch log files which includes es-master, es-data, es-ingest 217 <filter kubernetes.**vmi-system-es-**verrazzano-system_es-**> 218 @type parser 219 @id opensearch 220 key_name log 221 reserve_data true 222 emit_invalid_record_to_error true 223 <parse> 224 # opensearch have two formats for log records 225 # one has a timestamp, log level and message (already been parsed under istio-proxy container) 226 # the other is timestamp, log level, other field, pod-name and message 227 @type multi_format 228 <pattern> 229 format /^\[(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2},\d{3})\]\[(?<level>.*?)\]\[.*vmi-system-es-.*?\]\s(?<message>.*?)$/ 230 time_key logtime 231 time_format %Y-%m-%dT%H:%M:%S,%N 232 </pattern> 233 <pattern> 234 format none 235 </pattern> 236 </parse> 237 </filter> 238 239 # filter to parse authproxy container log files 240 <filter kubernetes.**verrazzano-authproxy**verrazzano-authproxy**> 241 @type parser 242 @id authproxy 243 key_name log 244 reserve_data true 245 emit_invalid_record_to_error true 246 <parse> 247 # authproxy has two formats for log records 248 # one has a timestamp, log level and other fields 249 # the other is just a messsage 250 @type multi_format 251 <pattern> 252 format json 253 time_key @timestamp 254 time_format %Y-%m-%dT%H:%M:%S+%N 255 </pattern> 256 <pattern> 257 format none 258 </pattern> 259 </parse> 260 </filter> 261 262 # filter to parse kiali container log files 263 <filter kubernetes.**vmi-system-kiali**verrazzano-system_vmi-system-kiali**> 264 @type parser 265 @id vmi-system-kiali 266 key_name log 267 reserve_data true 268 emit_invalid_record_to_error true 269 <parse> 270 # Kiali has two formats for log records 271 # Kiali format and a klog format 272 @type multi_format 273 <pattern> 274 format /^(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z) (?<level>.*?) (?<message>[\s\S]*?)$/ 275 time_key logtime 276 time_format %Y-%m-%dT%H:%M:%SZ 277 </pattern> 278 <pattern> 279 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 280 time_key logtime 281 time_format %H:%M:%S.%N 282 </pattern> 283 <pattern> 284 format none 285 </pattern> 286 </parse> 287 </filter> 288 289 # filter to parse Coherence operator container log files 290 <filter kubernetes.**coherence-operator**verrazzano-system_manager**> 291 @type parser 292 @id coh-operator 293 key_name log 294 reserve_data true 295 emit_invalid_record_to_error true 296 <parse> 297 # Coherence operator has two formats for log records 298 # Coherence operator format and a klog format 299 @type multi_format 300 <pattern> 301 format /^(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)\t(?<level>.*?)\t(?<message>[\s\S]*?)$/ 302 time_key logtime 303 time_format %Y-%m-%dT%H:%M:%S.%NZ 304 </pattern> 305 <pattern> 306 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 307 time_key logtime 308 time_format %H:%M:%S.%N 309 </pattern> 310 <pattern> 311 format none 312 </pattern> 313 </parse> 314 </filter> 315 316 # filter to parse oam-kubernetes-runtime container log files 317 <filter kubernetes.**oam-kubernetes-runtime**verrazzano-system_oam-kubernetes-runtime**> 318 @type parser 319 @id oam-kubernetes-runtime 320 key_name log 321 reserve_data true 322 emit_invalid_record_to_error true 323 <parse> 324 # oam-kubernetes-runtime has two formats for log records 325 # oam-kubernetes-runtime format and a klog format 326 @type multi_format 327 <pattern> 328 format /^(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)\t(?<level>.*?)\t(?<message>[\s\S]*?)$/ 329 time_key logtime 330 time_format %Y-%m-%dT%H:%M:%S.%NZ 331 </pattern> 332 <pattern> 333 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 334 time_key logtime 335 time_format %H:%M:%S.%N 336 </pattern> 337 <pattern> 338 format none 339 </pattern> 340 </parse> 341 </filter> 342 343 # filter to parse cert-manager container log files 344 # includes cert-manager, ca-injector, webhook 345 <filter kubernetes.**cert-manager**cert-manager**> 346 @type parser 347 @id cert-manager 348 key_name log 349 reserve_data true 350 emit_invalid_record_to_error true 351 <parse> 352 # cert-manager has a klog format 353 @type multi_format 354 <pattern> 355 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 356 time_key logtime 357 time_format %H:%M:%S.%N 358 </pattern> 359 <pattern> 360 format none 361 </pattern> 362 </parse> 363 </filter> 364 365 # filter to parse Verrazzano platform operator container log files 366 <filter kubernetes.**verrazzano-**-operator** kubernetes.**verrazzano-**_webhook-init**> 367 @type parser 368 @id verrazzano-operators 369 key_name log 370 reserve_data true 371 emit_invalid_record_to_error true 372 <parse> 373 @type multi_format 374 <pattern> 375 format json 376 time_key @timestamp 377 time_format %Y-%m-%dT%H:%M:%S.%NZ 378 </pattern> 379 # Kubernetes klog format 380 <pattern> 381 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 382 time_key logtime 383 time_format %H:%M:%S.%N 384 </pattern> 385 <pattern> 386 format none 387 </pattern> 388 </parse> 389 </filter> 390 391 # filter to parse Keycloak container log files 392 <filter kubernetes.**keycloak**keycloak_keycloak**> 393 @type parser 394 @id keycloak 395 key_name log 396 reserve_data true 397 emit_invalid_record_to_error true 398 <parse> 399 # Keycloak has one format for log records 400 @type multi_format 401 <pattern> 402 format /^.*?(?<logtime>\d{2}:\d{2}:\d{2},\d{3}) (?<level>.*?)( |\t)+\[.*?\]( |\t)+\(.*?\)( |\t)+(?<message>.*)$/ 403 time_key logtime 404 time_format %H:%M:%S,%N 405 </pattern> 406 <pattern> 407 format none 408 </pattern> 409 </parse> 410 </filter> 411 412 # filter to parse MySQL container log files 413 <filter kubernetes.**mysql**keycloak_mysql**> 414 @type parser 415 @id mysql 416 key_name log 417 reserve_data true 418 emit_invalid_record_to_error true 419 <parse> 420 # MySQL has two formats for log records 421 @type multi_format 422 <pattern> 423 format /^(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}Z) \d+ \[(?<level>.*?)\] (\[.*?\] ){2}(?<message>.*?)$/ 424 time_key logtime 425 time_format %Y-%m-%dT%H:%M:%S.%NZ 426 </pattern> 427 <pattern> 428 format /^(?<logtime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\+\d{2}:\d{2} \[(?<level>.*?)\] \[.*?\]: (?<message>.*?)$/ 429 time_key logtime 430 time_format %Y-%m-%d %H:%M:%S 431 </pattern> 432 <pattern> 433 format none 434 </pattern> 435 </parse> 436 </filter> 437 438 # filter to parse MySQLOperator container log files 439 <filter kubernetes.**mysql-operator**mysql-operator_mysql-operator**> 440 @type parser 441 @id mysql-operator 442 key_name log 443 reserve_data true 444 emit_invalid_record_to_error true 445 <parse> 446 @type multi_format 447 <pattern> 448 format /^\[(?<logtime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\s(?<component>.*?)\s\[(?<level>.*?)\]\s(?<message>.*?)$/ 449 time_key logtime 450 time_format %Y-%m-%d %H:%M:%S,%N 451 </pattern> 452 <pattern> 453 format /^(?<logtime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}):\s(?<level>.*?):\s(?<message>.*?)$/ 454 time_key logtime 455 time_format %Y-%m-%d %H:%M:%S 456 </pattern> 457 <pattern> 458 format none 459 </pattern> 460 </parse> 461 </filter> 462 463 # filter to parse Grafana container log files 464 <filter kubernetes.**vmi-system-grafana**verrazzano-system_grafana**> 465 @type parser 466 @id grafana 467 key_name log 468 reserve_data true 469 emit_invalid_record_to_error true 470 <parse> 471 # Grafana has two formats for log records 472 # one for json logs and one for string logs 473 @type multi_format 474 <pattern> 475 format json 476 time_key @timestamp 477 time_format %Y-%m-%dT%H:%M:%S.%NZ 478 </pattern> 479 <pattern> 480 format /^t=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2})\+\d{4} lvl=(?<level>\S+) msg="(?<message>.*?)".*?$/ 481 time_key logtime 482 time_format %Y-%m-%dT%H:%M:%S 483 </pattern> 484 <pattern> 485 format none 486 </pattern> 487 </parse> 488 </filter> 489 490 # filter to apply a record transformer Grafana container log files in JSON format 491 <filter kubernetes.**vmi-system-grafana**verrazzano-system_grafana**> 492 @type record_transformer 493 @id grafana-json 494 enable_ruby true 495 <record> 496 message ${record["@message"] ? record["@message"] : record["message"] ? record["message"] : ""} 497 level ${record["@level"] ? record["@level"] : record["level"] ? record["level"] : ""} 498 </record> 499 </filter> 500 501 502 # filter to parse verrazzano-system Prometheus container log files 503 <filter kubernetes.**vmi-system-prometheus**verrazzano-system_prometheus**> 504 @type parser 505 @id vmi-system-prometheus 506 key_name log 507 reserve_data true 508 emit_invalid_record_to_error true 509 <parse> 510 # Prometheus has two formats for log records 511 # One with a level and msg 512 # One with a level but not msg 513 @type multi_format 514 <pattern> 515 format /^ts=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)(.*)level=(?<level>.*?) (.*?)msg="(?<message>.*?)"([\s\S]*?)$/ 516 time_key logtime 517 time_format %Y-%m-%dT%H:%M:%S.%NZ 518 </pattern> 519 <pattern> 520 format /^ts=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)(.*)level=(?<level>.*?) (?<message>[\s\S]*?)$/ 521 time_key logtime 522 time_format %Y-%m-%dT%H:%M:%S.%NZ 523 </pattern> 524 # Kubernetes klog format 525 <pattern> 526 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 527 time_key logtime 528 time_format %H:%M:%S.%N 529 </pattern> 530 <pattern> 531 format none 532 </pattern> 533 </parse> 534 </filter> 535 536 # filter to parse verrazzano-system Prometheus config-reloader container log files 537 <filter kubernetes.**vmi-system-prometheus**verrazzano-system_config-reloader**> 538 @type parser 539 @id config-reloader 540 key_name log 541 reserve_data true 542 emit_invalid_record_to_error true 543 <parse> 544 # config-reloader log messages do not have a log level 545 @type multi_format 546 <pattern> 547 format /^(?<logtime>\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}) (?<message>[\s\S]*?)$/ 548 time_key logtime 549 time_format %Y/%m/%d %H:%M:%S 550 </pattern> 551 <pattern> 552 format none 553 </pattern> 554 </parse> 555 </filter> 556 557 # filter to parse Weblogic Operator Dashboard container log files 558 <filter kubernetes.**weblogic-operator**verrazzano-system_weblogic-operator**> 559 @type parser 560 @id weblogic-operator 561 key_name log 562 reserve_data true 563 emit_invalid_record_to_error true 564 <parse> 565 # WebLogic Operator Dashboard has one format for log records 566 @type multi_format 567 <pattern> 568 format json 569 time_key timestamp 570 time_format %Y-%m-%dT%H:%M:%S.%NZ 571 </pattern> 572 <pattern> 573 format none 574 </pattern> 575 </parse> 576 </filter> 577 578 # filter to apply a record transformer into WebLogic Component log files in JSON format 579 <filter kubernetes.**weblogic-operator**verrazzano-system_weblogic-operator**> 580 @type record_transformer 581 @id weblogic-operator-json 582 remove_keys timestamp 583 </filter> 584 585 # filter to parse OpenSearch Dashboard container log files 586 <filter kubernetes.**vmi-system-osd**verrazzano-system_kibana**> 587 @type parser 588 @id kibana 589 key_name log 590 reserve_data true 591 emit_invalid_record_to_error true 592 <parse> 593 # OpenSearch Dashboard has one format for log records 594 @type multi_format 595 <pattern> 596 format json 597 time_key @timestamp 598 time_format %Y-%m-%dT%H:%M:%SZ 599 </pattern> 600 <pattern> 601 format none 602 </pattern> 603 </parse> 604 </filter> 605 606 # filter to parse OpenSearch Dashboard container log files 607 <filter kubernetes.**vmi-system-osd**verrazzano-system_kibana**> 608 @type record_transformer 609 @id kibana-json 610 enable_ruby true 611 <record> 612 # the following Ruby code looks for an intersection between the "tags" values and the log level values 613 # it returns the first intersection value if found or empty string if not 614 level ${!!record["tags"] ? !(['trace', 'debug', 'info', 'warn', 'error', 'fail'] & record["tags"])[0].nil? ? (['trace', 'debug', 'info', 'warn', 'error', 'fail'] & record["tags"])[0] : "" : ""} 615 </record> 616 </filter> 617 618 # filter to parse NGINX Ingress Controller container log files 619 <filter **ingress-nginx-controller**ingress-nginx_controller-**> 620 @type parser 621 @id nginx-ingress-controller 622 key_name log 623 reserve_data true 624 emit_invalid_record_to_error true 625 <parse> 626 @type multi_format 627 <pattern> 628 format json 629 time_key @timestamp 630 time_format %Y-%m-%dT%H:%M:%S+%N 631 </pattern> 632 # Kubernetes klog format 633 <pattern> 634 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 635 time_key logtime 636 time_format %H:%M:%S.%N 637 </pattern> 638 <pattern> 639 format none 640 </pattern> 641 </parse> 642 </filter> 643 644 # filter to parse Rancher namespace container log files 645 <filter kubernetes.**cattle-system** kubernetes.**fleet-system** kubernetes.**local-path-provisioner**> 646 @type parser 647 @id rancher 648 key_name log 649 reserve_data true 650 emit_invalid_record_to_error true 651 <parse> 652 @type multi_format 653 # Rancher pattern #1 654 <pattern> 655 format /^time="(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)" level=(?<level>.*?) msg="(?<message>.*?)"[\s\S]*?$/ 656 time_key logtime 657 time_format %Y-%m-%dT%H:%M:%SZ 658 </pattern> 659 # Rancher pattern #2 660 <pattern> 661 format /^(?<logtime>\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}) \[(?<level>.*?)\] (?<message>[\s\S]*?)?$/ 662 time_key logtime 663 time_format %Y/%m/%d %H:%M:%S 664 </pattern> 665 # Kubernetes klog format 666 <pattern> 667 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 668 time_key logtime 669 time_format %H:%M:%S.%N 670 </pattern> 671 <pattern> 672 format none 673 </pattern> 674 </parse> 675 </filter> 676 677 # filter to parse verrazzano-capi namespace container log files 678 <filter kubernetes.**verrazzano-capi**> 679 @type parser 680 @id clusterapi 681 key_name log 682 reserve_data true 683 emit_invalid_record_to_error true 684 <parse> 685 @type multi_format 686 # Kubernetes klog format 687 <pattern> 688 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 689 time_key logtime 690 time_format %H:%M:%S.%N 691 </pattern> 692 <pattern> 693 format none 694 </pattern> 695 </parse> 696 </filter> 697 698 # filter to parse External-dns container log files 699 <filter kubernetes.**external-dns**external-dns**> 700 @type parser 701 @id external-dns 702 key_name log 703 reserve_data true 704 emit_invalid_record_to_error true 705 <parse> 706 @type multi_format 707 # external-dns pattern 708 <pattern> 709 format /^time="(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)" level=(?<level>.*?) msg="(?<message>.*?)"[\s\S]*?$/ 710 time_key logtime 711 time_format %Y-%m-%dT%H:%M:%SZ 712 </pattern> 713 # Kubernetes klog format 714 <pattern> 715 format /^(?<level>.)(\d{2}\d{2}) (?<logtime>\d{2}:\d{2}:\d{2}.\d{6})\s*?(?<message>[\s\S]*?)$/ 716 time_key logtime 717 time_format %H:%M:%S.%N 718 </pattern> 719 <pattern> 720 format none 721 </pattern> 722 </parse> 723 </filter> 724 725 726 # filter to parse node-exporter container log files 727 <filter kubernetes.**node-exporter**monitoring_node-exporter**> 728 @type parser 729 @id node-exporter 730 key_name log 731 reserve_data true 732 emit_invalid_record_to_error true 733 <parse> 734 # Node exporter has two formats for log records 735 # One with a level and msg 736 # One with a level but not msg 737 @type multi_format 738 <pattern> 739 format /^level=(?<level>.*?) ts=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)(.*?)msg="(?<message>.*?)"([\s\S]*?)$/ 740 time_key logtime 741 time_format %Y-%m-%dT%H:%M:%S.%NZ 742 </pattern> 743 <pattern> 744 format /^level=(?<level>.*?) ts=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z) (?<message>[\s\S]*?)$/ 745 time_key logtime 746 time_format %Y-%m-%dT%H:%M:%S.%NZ 747 </pattern> 748 <pattern> 749 format none 750 </pattern> 751 </parse> 752 </filter> 753 754 # filter to parse jaeger operator and jaeger resource log files 755 <filter kubernetes.**jaeger-operator-**verrazzano-monitoring_jaeger-**> 756 @type parser 757 @id jaeger 758 key_name log 759 reserve_data true 760 emit_invalid_record_to_error true 761 <parse> 762 @type multi_format 763 # zap log format for Jaeger components 764 <pattern> 765 format json 766 time_key ts 767 time_type float 768 </pattern> 769 # Log patterns for Jaeger operator 770 <pattern> 771 format /^(?<logtime>.*?) (?<level>\S+?) (?<message>[\s\S]*?)$/ 772 time_key logtime 773 time_type float 774 </pattern> 775 <pattern> 776 format /^(?<logtime>.*?) (?<level>\S+?) (?<component>[\.\S]+?) (?<message>[\s\S]*?)$/ 777 time_key logtime 778 time_type float 779 </pattern> 780 <pattern> 781 format /^time=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z) level=(?<level>.*?) message=(?<message>[\s\S]*?) error=(?<error>[\s\S]*?) execution=(?<executiontime>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}.\d+ \+\d{3} \w+?) instance=(?<instance>[\s\S]*?) namespace=(?<namespace>[\s\S]*?)$/ 782 time_key logtime 783 time_format %Y-%m-%dT%H:%M:%S.%NZ 784 </pattern> 785 <pattern> 786 format none 787 </pattern> 788 </parse> 789 </filter> 790 791 # filter to parse Thanos container log files 792 <filter kubernetes.**thanos**verrazzano-monitoring**> 793 @type parser 794 @id thanos 795 key_name log 796 reserve_data true 797 emit_invalid_record_to_error true 798 <parse> 799 # Thanos has multiple formats for log records 800 # Some records do not have a "msg" field, they instead have an "err" field 801 @type multi_format 802 <pattern> 803 format /^level=(?<level>.*?) ts=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3,}Z)(.*?)msg="(?<message>.*?)"([\s\S]*?)$/ 804 time_key logtime 805 time_format %Y-%m-%dT%H:%M:%S.%NZ 806 </pattern> 807 <pattern> 808 format /^level=(?<level>.*?) ts=(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3,}Z)(.*?)err="(?<message>.*?)"$/ 809 time_key logtime 810 time_format %Y-%m-%dT%H:%M:%S.%NZ 811 </pattern> 812 # Istio proxy log pattern 813 <pattern> 814 format /^(?<logtime>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}Z)\t(?<level>.*?)\t(?<message>[\s\S]*?)$/ 815 time_key logtime 816 time_format %Y-%m-%dT%H:%M:%S.%NZ 817 </pattern> 818 <pattern> 819 format none 820 </pattern> 821 </parse> 822 </filter> 823 824 kubernetes-filter.conf: | 825 # Query the API for extra metadata. 826 <filter kubernetes.**> 827 @type kubernetes_metadata 828 @id kubernetes_metadata 829 watch_retry_interval 20 830 </filter> 831 832 # rewrite_tag_filter does not support nested fields like 833 # kubernetes.container_name, so this exists to flatten the fields 834 # so we can use them in our rewrite_tag_filter 835 <filter kubernetes.**> 836 @type record_transformer 837 @id kubernetes_record_transformer 838 enable_ruby true 839 <record> 840 kubernetes_namespace_container_name ${record["kubernetes"]["namespace_name"]}.${record["kubernetes"]["container_name"]} 841 </record> 842 </filter> 843 844 # parse sidecar stdout 845 <filter kubernetes.**_fluentd-stdout-sidecar-**> 846 @type parser 847 @id stdout_log_text 848 key_name log 849 reserve_data true 850 emit_invalid_record_to_error true 851 <parse> 852 @type multi_format 853 <pattern> 854 format /^(?<time>[^ ]* [^ ]* [^ ]*) (?<flags>[^\s]+): (?<log>[\s\S]*)$/ 855 </pattern> 856 <pattern> 857 format none 858 </pattern> 859 </parse> 860 </filter> 861 862 # parse log record 863 <filter kubernetes.**> 864 @type parser 865 @id parse_log_to_json 866 key_name log 867 reserve_data true 868 emit_invalid_record_to_error true 869 <parse> 870 @type multi_format 871 <pattern> 872 format json 873 time_key @timestamp 874 time_format %Y-%m-%dT%H:%M:%S.%NZ 875 </pattern> 876 <pattern> 877 format none 878 </pattern> 879 </parse> 880 </filter> 881 882 # Remove the unnecessary field as the information is already available on 883 # other fields. 884 <filter kube.**> 885 @type record_transformer 886 @id kube_record_transformer 887 remove_keys kubernetes_namespace_container_name 888 </filter> 889 890 <filter kube.kube-system.**> 891 @type parser 892 @id kube_parser 893 format kubernetes 894 reserve_data true 895 key_name log 896 </filter> 897 898 <filter kube.**> 899 @type parser 900 key_name log 901 reserve_data true 902 remove_key_name_field false 903 emit_invalid_record_to_error false 904 <parse> 905 @type multi_format 906 <pattern> 907 format json 908 time_format %Y-%m-%dT%H:%M:%S.%N%Z 909 </pattern> 910 <pattern> 911 format json 912 time_format %Y-%m-%dT%H:%M:%S%z 913 </pattern> 914 </parse> 915 </filter> 916 917 output.conf: | 918 <filter **> 919 @type record_transformer 920 @id cluster_name 921 <record> 922 cluster_name "#{ENV['CLUSTER_NAME']}" 923 </record> 924 </filter> 925 926 # Force the timestamp field into ISO 8601 format 927 <filter **> 928 @type record_transformer 929 @id time_format 930 enable_ruby true 931 <record> 932 @timestamp ${time.iso8601(3)} 933 </record> 934 </filter> 935 936 es-output.conf: | 937 # Matches anything that Verrazzano installs 938 <match kubernetes.**_kube-** kubernetes.**_verrazzano-** kubernetes.**cattle-** kubernetes.**rancher-** kubernetes.**fleet-** kubernetes.**ingress-nginx** kubernetes.**istio-system** kubernetes.**keycloak** kubernetes.**mysql-operator** kubernetes.**cert-manager** kubernetes.**_monitoring_** kubernetes.**_metallb-** kubernetes.**_local-path-storage_** kubernetes.**_local_** systemd.** kubernetes.**argocd** > 939 @type opensearch_data_stream 940 @id out_systemd 941 @log_level info 942 log_es_400_reason true 943 suppress_type_name true 944 945 data_stream_name verrazzano-system 946 data_stream_template_name verrazzano-data-stream 947 template_file /fluentd/etc/opensearch-template-verrazzano.json 948 949 time_precision 9 950 951 # Prevent reloading connections to Elasticsearch 952 reload_connections false 953 reconnect_on_error true 954 reload_on_failure true 955 slow_flush_log_threshold 120s 956 957 hosts "#{ENV['ELASTICSEARCH_URL']}" 958 ca_file "#{ENV['CA_FILE']}" 959 # ssl_version TLSv1_2 960 user "#{ENV['ELASTICSEARCH_USER']}" 961 password "#{ENV['ELASTICSEARCH_PASSWORD']}" 962 963 bulk_message_request_threshold 16M 964 request_timeout 2147483648 965 <buffer> 966 @type file 967 path /fluentd/log/system-buffer 968 flush_thread_count 8 969 flush_interval 5s 970 retry_forever 971 retry_max_interval 10 972 # Cap buffer memory usage to 16MiB/chunk * 10 chunks = 160 MiB 973 chunk_limit_size 16M 974 queue_limit_length 10 975 chunk_full_threshold 0.9 976 overflow_action throw_exception 977 </buffer> 978 </match> 979 <match **> 980 @type opensearch_data_stream 981 @id out_all 982 @log_level info 983 log_es_400_reason true 984 suppress_type_name true 985 986 data_stream_name verrazzano-application-${$.kubernetes.namespace_name} 987 data_stream_template_name verrazzano-data-stream 988 template_file /fluentd/etc/opensearch-template-verrazzano.json 989 990 time_precision 9 991 # Prevent reloading connections to Elasticsearch 992 reload_connections false 993 reconnect_on_error true 994 reload_on_failure true 995 slow_flush_log_threshold 120s 996 997 hosts "#{ENV['ELASTICSEARCH_URL']}" 998 ca_file "#{ENV['CA_FILE']}" 999 # ssl_version TLSv1_2 1000 user "#{ENV['ELASTICSEARCH_USER']}" 1001 password "#{ENV['ELASTICSEARCH_PASSWORD']}" 1002 1003 bulk_message_request_threshold 16M 1004 request_timeout 2147483648 1005 <buffer tag, $.kubernetes.namespace_name> 1006 @type file 1007 path /fluentd/log/output-buffer 1008 flush_thread_count 8 1009 flush_interval 5s 1010 retry_forever 1011 retry_max_interval 10 1012 # Cap buffer memory usage to 16MiB/chunk * 10 chunks = 160 MiB 1013 chunk_limit_size 16M 1014 queue_limit_length 10 1015 chunk_full_threshold 0.9 1016 overflow_action throw_exception 1017 </buffer> 1018 </match> 1019 1020 {{- if .Values.fluentd.oci }} 1021 oci-logging-system.conf: | 1022 # Match all "system" namespaces so system log records are sent to a separate OCI Log object 1023 <match kubernetes.**_kube-** kubernetes.**_verrazzano-** kubernetes.**cattle-** kubernetes.**rancher-** kubernetes.**fleet-** kubernetes.**ingress-nginx** kubernetes.**istio-system** kubernetes.**keycloak** kubernetes.**mysql-operator** kubernetes.**cert-manager** kubernetes.**_monitoring_** kubernetes.**_metallb-** kubernetes.**_local-path-storage_** kubernetes.**_local_** systemd.** kubernetes.**argocd**> 1024 @type oci_logging 1025 log_object_id {{ .Values.fluentd.oci.systemLogId }} 1026 <buffer> 1027 @type file 1028 path /fluentd/log/oci-logging-system 1029 disable_chunk_backup true 1030 chunk_limit_size 5MB 1031 flush_interval 180s 1032 total_limit_size 1GB 1033 overflow_action throw_exception 1034 retry_type exponential_backoff 1035 </buffer> 1036 </match> 1037 1038 oci-logging-default-app.conf: | 1039 <match **> 1040 @type oci_logging 1041 log_object_id {{ .Values.fluentd.oci.defaultAppLogId }} 1042 <buffer> 1043 @type file 1044 path /fluentd/log/oci-logging-default-app 1045 disable_chunk_backup true 1046 chunk_limit_size 5MB 1047 flush_interval 180s 1048 total_limit_size 1GB 1049 overflow_action throw_exception 1050 retry_type exponential_backoff 1051 </buffer> 1052 </match> 1053 {{- end }} 1054 1055 opensearch-template-verrazzano.json: | 1056 { 1057 "index_patterns":[ 1058 "verrazzano-system", 1059 "verrazzano-application*" 1060 ], 1061 "version":60001, 1062 "priority": 101, 1063 "data_stream": {}, 1064 "template": { 1065 "settings":{ 1066 "index.refresh_interval":"5s", 1067 "index.mapping.total_fields.limit":"2000", 1068 "number_of_shards":1, 1069 "index.number_of_replicas":0, 1070 "index.auto_expand_replicas":"0-1" 1071 }, 1072 "mappings":{ 1073 "dynamic_templates":[ 1074 { 1075 "message_field":{ 1076 "path_match":"message", 1077 "match_mapping_type":"string", 1078 "mapping":{ 1079 "type":"text", 1080 "norms":false 1081 } 1082 } 1083 }, 1084 { 1085 "object_fields": { 1086 "match": "*", 1087 "match_mapping_type": "object", 1088 "mapping": { 1089 "type": "object" 1090 } 1091 } 1092 }, 1093 { 1094 "all_non_object_fields":{ 1095 "match":"*", 1096 "mapping":{ 1097 "type":"text", 1098 "norms":false, 1099 "fields":{ 1100 "keyword":{ 1101 "type":"keyword", 1102 "ignore_above":256 1103 } 1104 } 1105 } 1106 } 1107 } 1108 ], 1109 "properties" : { 1110 "@timestamp": { "type": "date", "format": "strict_date_time||strict_date_optional_time||epoch_millis"}, 1111 "kubernetes.pod_ip": { 1112 "type": "text", 1113 "norms": false, 1114 "fields":{ 1115 "keyword":{ 1116 "type":"keyword", 1117 "ignore_above":256 1118 }, 1119 "ip":{ 1120 "type": "ip", 1121 "ignore_malformed": true 1122 } 1123 } 1124 }, 1125 "http_request.remoteIp": { 1126 "type": "text", 1127 "norms": false, 1128 "fields":{ 1129 "keyword":{ 1130 "type":"keyword", 1131 "ignore_above":256 1132 }, 1133 "ip":{ 1134 "type": "ip", 1135 "ignore_malformed": true 1136 } 1137 } 1138 }, 1139 "http_request.responseSize": { 1140 "type": "text", 1141 "norms": false, 1142 "fields":{ 1143 "keyword":{ 1144 "type":"keyword", 1145 "ignore_above":256 1146 }, 1147 "integer":{ 1148 "type": "integer" 1149 } 1150 } 1151 }, 1152 "http_request.status": { 1153 "type": "text", 1154 "norms": false, 1155 "fields":{ 1156 "keyword":{ 1157 "type":"keyword", 1158 "ignore_above":256 1159 }, 1160 "integer":{ 1161 "type": "integer" 1162 } 1163 } 1164 }, 1165 "http_request.requestSize": { 1166 "type": "text", 1167 "norms": false, 1168 "fields":{ 1169 "keyword":{ 1170 "type":"keyword", 1171 "ignore_above":256 1172 }, 1173 "integer":{ 1174 "type": "integer" 1175 } 1176 } 1177 } 1178 } 1179 } 1180 } 1181 }