github.com/kiali/kiali@v1.84.0/business/dashboards.go (about) 1 package business 2 3 import ( 4 "fmt" 5 "sort" 6 "strings" 7 "sync" 8 9 "github.com/kiali/kiali/config" 10 "github.com/kiali/kiali/config/dashboards" 11 "github.com/kiali/kiali/log" 12 "github.com/kiali/kiali/models" 13 "github.com/kiali/kiali/prometheus" 14 ) 15 16 const defaultNamespaceLabel = "namespace" 17 18 // DashboardsService deals with fetching dashboards from config 19 type DashboardsService struct { 20 promClient prometheus.ClientInterface 21 dashboards map[string]dashboards.MonitoringDashboard 22 promConfig config.PrometheusConfig 23 globalNamespace string 24 namespaceLabel string 25 CustomEnabled bool 26 } 27 28 // NewDashboardsService initializes this business service 29 func NewDashboardsService(namespace *models.Namespace, workload *models.Workload) *DashboardsService { 30 cfg := config.Get() 31 customEnabled := cfg.ExternalServices.CustomDashboards.Enabled 32 prom := cfg.ExternalServices.Prometheus 33 if customEnabled && cfg.ExternalServices.CustomDashboards.Prometheus.URL != "" { 34 prom = cfg.ExternalServices.CustomDashboards.Prometheus 35 } 36 nsLabel := cfg.ExternalServices.CustomDashboards.NamespaceLabel 37 if nsLabel == "" { 38 nsLabel = "namespace" 39 } 40 41 // Overwrite Custom dashboards defined at Namespace level 42 builtInDashboards := cfg.CustomDashboards 43 if namespace != nil { 44 nsDashboards := dashboards.GetNamespaceMonitoringDashboards(namespace.Name, namespace.Annotations) 45 builtInDashboards = dashboards.AddMonitoringDashboards(builtInDashboards, nsDashboards) 46 } 47 if workload != nil { 48 wkDashboards := dashboards.GetWorkloadMonitoringDashboards(namespace.Name, workload.Name, workload.DashboardAnnotations) 49 builtInDashboards = dashboards.AddMonitoringDashboards(builtInDashboards, wkDashboards) 50 } 51 52 return &DashboardsService{ 53 CustomEnabled: customEnabled, 54 promConfig: prom, 55 globalNamespace: cfg.Deployment.Namespace, 56 namespaceLabel: nsLabel, 57 dashboards: builtInDashboards.OrganizeByName(), 58 } 59 } 60 61 func (in *DashboardsService) prom() (prometheus.ClientInterface, error) { 62 // Lazy init 63 if in.promClient == nil { 64 client, err := prometheus.NewClientForConfig(in.promConfig) 65 if err != nil { 66 return nil, fmt.Errorf("cannot initialize Prometheus Client: %v", err) 67 } 68 in.promClient = client 69 } 70 return in.promClient, nil 71 } 72 73 func (in *DashboardsService) loadRawDashboardResource(template string) (*dashboards.MonitoringDashboard, error) { 74 dashboard, ok := in.dashboards[template] 75 if !ok { 76 return nil, fmt.Errorf("Dashboard [%v] does not exist or is disabled", template) 77 } 78 79 return &dashboard, nil 80 } 81 82 func (in *DashboardsService) loadAndResolveDashboardResource(template string, loaded map[string]bool) (*dashboards.MonitoringDashboard, error) { 83 // Circular dependency check 84 if _, ok := loaded[template]; ok { 85 return nil, fmt.Errorf("cannot load dashboard %s due to circular dependency detected. Already loaded dependencies: %v", template, loaded) 86 } 87 loaded[template] = true 88 dashboard, err := in.loadRawDashboardResource(template) 89 if err != nil { 90 return nil, err 91 } 92 err = in.resolveReferences(dashboard, loaded) 93 return dashboard, err 94 } 95 96 // resolveReferences resolves the composition mechanism that allows to reference a dashboard from another one 97 func (in *DashboardsService) resolveReferences(dashboard *dashboards.MonitoringDashboard, loaded map[string]bool) error { 98 resolved := []dashboards.MonitoringDashboardItem{} 99 for _, item := range dashboard.Items { 100 reference := strings.TrimSpace(item.Include) 101 if reference != "" { 102 // reference can point to a whole dashboard (ex: microprofile-1.0) or a chart within a dashboard (ex: microprofile-1.0$Thread count) 103 parts := strings.Split(reference, "$") 104 dashboardRefName := parts[0] 105 composedDashboard, err := in.loadAndResolveDashboardResource(dashboardRefName, loaded) 106 if err != nil { 107 return err 108 } 109 for _, item2 := range composedDashboard.Items { 110 if len(parts) > 1 { 111 // Reference a specific chart 112 if item2.Chart.Name == parts[1] { 113 resolved = append(resolved, item2) 114 break 115 } 116 } else { 117 // Reference the whole dashboard 118 resolved = append(resolved, item2) 119 } 120 } 121 } else { 122 resolved = append(resolved, item) 123 } 124 } 125 dashboard.Items = resolved 126 return nil 127 } 128 129 // GetDashboard returns a dashboard filled-in with target data 130 func (in *DashboardsService) GetDashboard(params models.DashboardQuery, template string) (*models.MonitoringDashboard, error) { 131 promClient, err := in.prom() 132 if err != nil { 133 return nil, err 134 } 135 136 dashboard, err := in.loadAndResolveDashboardResource(template, map[string]bool{}) 137 if err != nil { 138 return nil, err 139 } 140 141 filters := in.buildLabelsQueryString(params.Namespace, params.LabelsFilters) 142 aggLabels := append(params.AdditionalLabels, models.ConvertAggregations(*dashboard)...) 143 if len(aggLabels) == 0 { 144 // Prevent null in json 145 aggLabels = []models.Aggregation{} 146 } 147 148 wg := sync.WaitGroup{} 149 wg.Add(len(dashboard.Items) + 1) 150 filledCharts := make([]models.Chart, len(dashboard.Items)) 151 152 for i, item := range dashboard.Items { 153 go func(idx int, chart dashboards.MonitoringDashboardChart) { 154 defer wg.Done() 155 conversionParams := models.ConversionParams{Scale: 1.0, SortLabel: chart.SortLabel, SortLabelParseAs: chart.SortLabelParseAs} 156 if chart.UnitScale != 0.0 { 157 conversionParams.Scale = chart.UnitScale 158 } 159 // Group by labels is concat of what is defined in CR + what is passed as parameters 160 byLabels := append(chart.GroupLabels, params.ByLabels...) 161 if len(chart.SortLabel) > 0 { 162 // We also need to group by the label used for sorting, if not explicitly present 163 present := false 164 for _, lbl := range byLabels { 165 if lbl == chart.SortLabel { 166 present = true 167 break 168 } 169 } 170 if !present { 171 byLabels = append(byLabels, chart.SortLabel) 172 // Mark the sort label to not be kept during conversion 173 conversionParams.RemoveSortLabel = true 174 } 175 } 176 grouping := strings.Join(byLabels, ",") 177 178 filledCharts[idx] = models.ConvertChart(chart) 179 metrics := chart.GetMetrics() 180 for _, ref := range metrics { 181 var converted []models.Metric 182 var err error 183 if chart.DataType == dashboards.Raw { 184 aggregator := params.RawDataAggregator 185 if chart.Aggregator != "" { 186 aggregator = chart.Aggregator 187 } 188 metric := promClient.FetchRange(ref.MetricName, filters, grouping, aggregator, ¶ms.RangeQuery) 189 converted, err = models.ConvertMetric(ref.DisplayName, metric, conversionParams) 190 } else if chart.DataType == dashboards.Rate { 191 metric := promClient.FetchRateRange(ref.MetricName, []string{filters}, grouping, ¶ms.RangeQuery) 192 converted, err = models.ConvertMetric(ref.DisplayName, metric, conversionParams) 193 } else { 194 histo := promClient.FetchHistogramRange(ref.MetricName, filters, grouping, ¶ms.RangeQuery) 195 converted, err = models.ConvertHistogram(ref.DisplayName, histo, conversionParams) 196 } 197 198 // Fill in chart 199 if err != nil { 200 filledCharts[idx].Error = err.Error() 201 } else { 202 filledCharts[idx].Metrics = append(filledCharts[idx].Metrics, converted...) 203 } 204 } 205 }(i, item.Chart) 206 } 207 208 var externalLinks []models.ExternalLink 209 go func() { 210 defer wg.Done() 211 links, _, err := GetGrafanaLinks(dashboard.ExternalLinks) 212 if err != nil { 213 log.Errorf("Error while getting Grafana links: %v", err) 214 } 215 if links != nil { 216 externalLinks = links 217 } else { 218 externalLinks = []models.ExternalLink{} 219 } 220 }() 221 222 wg.Wait() 223 // A dashboard can define the rows used, if not defined, by default it will use 2 rows 224 rows := dashboard.Rows 225 if rows < 1 { 226 rows = 2 227 } 228 return &models.MonitoringDashboard{ 229 Name: dashboard.Name, 230 Title: dashboard.Title, 231 Charts: filledCharts, 232 Aggregations: aggLabels, 233 ExternalLinks: externalLinks, 234 Rows: rows, 235 }, nil 236 } 237 238 // SearchExplicitDashboards will check annotations of all supplied pods to extract a unique list of dashboards 239 // Accepted annotations are "kiali.io/runtimes" and "kiali.io/dashboards" 240 func (in *DashboardsService) SearchExplicitDashboards(pods []models.Pod) []models.Runtime { 241 uniqueRefsList := extractUniqueDashboards(pods) 242 if len(uniqueRefsList) > 0 { 243 log.Tracef("getting dashboards from refs list: %v", uniqueRefsList) 244 return in.buildRuntimesList(uniqueRefsList) 245 } 246 return []models.Runtime{} 247 } 248 249 func (in *DashboardsService) buildRuntimesList(templatesNames []string) []models.Runtime { 250 dashboards := make([]*dashboards.MonitoringDashboard, len(templatesNames)) 251 wg := sync.WaitGroup{} 252 wg.Add(len(templatesNames)) 253 for idx, template := range templatesNames { 254 go func(i int, tpl string) { 255 defer wg.Done() 256 dashboard, err := in.loadRawDashboardResource(tpl) 257 if err != nil { 258 log.Errorf("cannot get dashboard [%s]: %v", tpl, err) 259 } else { 260 dashboards[i] = dashboard 261 } 262 }(idx, template) 263 } 264 265 wg.Wait() 266 267 runtimes := []models.Runtime{} 268 for _, dashboard := range dashboards { 269 if dashboard == nil { 270 continue 271 } 272 runtimes = addDashboardToRuntimes(dashboard, runtimes) 273 } 274 return runtimes 275 } 276 277 func (in *DashboardsService) fetchDashboardMetricNames(namespace string, labelsFilters map[string]string) []string { 278 promClient, err := in.prom() 279 if err != nil { 280 return []string{} 281 } 282 283 // Get the list of metrics that we look for to determine which dashboards can be used. 284 // Some dashboards cannot be discovered using metric lookups - ignore those. 285 discoverOnMetrics := make([]string, 0, len(in.dashboards)) 286 for _, md := range in.dashboards { 287 if md.DiscoverOn != "" { 288 discoverOnMetrics = append(discoverOnMetrics, md.DiscoverOn) 289 } 290 } 291 292 labels := in.buildLabelsQueryString(namespace, labelsFilters) 293 metrics, err := promClient.GetMetricsForLabels(discoverOnMetrics, labels) 294 if err != nil { 295 log.Errorf("custom dashboard discovery failed, cannot load metrics for labels [%s]: %v", labels, err) 296 } 297 return metrics 298 } 299 300 // discoverDashboards tries to discover dashboards based on existing metrics 301 func (in *DashboardsService) discoverDashboards(namespace string, labelsFilters map[string]string) []models.Runtime { 302 log.Tracef("starting custom dashboard discovery on namespace [%s] with filters [%v]", namespace, labelsFilters) 303 304 var metrics []string 305 wg := sync.WaitGroup{} 306 wg.Add(1) 307 go func() { 308 defer wg.Done() 309 metrics = in.fetchDashboardMetricNames(namespace, labelsFilters) 310 }() 311 312 wg.Wait() 313 return runDiscoveryMatcher(metrics, in.dashboards) 314 } 315 316 func runDiscoveryMatcher(metrics []string, allDashboards map[string]dashboards.MonitoringDashboard) []models.Runtime { 317 // In all dashboards, finds the ones that match the metrics set 318 // We must exclude from the results included dashboards when both the including and the included dashboards are matching 319 runtimesMap := make(map[string]*dashboards.MonitoringDashboard) 320 for _, d := range allDashboards { 321 dashboard := d // sticky reference 322 matchReference := strings.TrimSpace(dashboard.DiscoverOn) 323 if matchReference != "" { 324 for _, metric := range metrics { 325 if matchReference == metric { 326 if _, exists := runtimesMap[dashboard.Name]; !exists { 327 runtimesMap[dashboard.Name] = &dashboard 328 } 329 // Mark included dashboards as already found 330 // and set them "nil" to not show them as standalone dashboards even if they match 331 for _, item := range dashboard.Items { 332 if item.Include != "" { 333 runtimesMap[item.Include] = nil 334 } 335 } 336 break 337 } 338 } 339 } 340 } 341 runtimes := []models.Runtime{} 342 for _, dashboard := range runtimesMap { 343 if dashboard != nil { 344 runtimes = addDashboardToRuntimes(dashboard, runtimes) 345 } 346 } 347 sort.Slice(runtimes, func(i, j int) bool { return runtimes[i].Name < runtimes[j].Name }) 348 return runtimes 349 } 350 351 func addDashboardToRuntimes(dashboard *dashboards.MonitoringDashboard, runtimes []models.Runtime) []models.Runtime { 352 runtime := dashboard.Runtime 353 ref := models.DashboardRef{ 354 Template: dashboard.Name, 355 Title: dashboard.Title, 356 } 357 found := false 358 for i := range runtimes { 359 rtObj := &runtimes[i] 360 if rtObj.Name == runtime { 361 rtObj.DashboardRefs = append(rtObj.DashboardRefs, ref) 362 found = true 363 break 364 } 365 } 366 if !found { 367 runtimes = append(runtimes, models.Runtime{ 368 Name: runtime, 369 DashboardRefs: []models.DashboardRef{ref}, 370 }) 371 } 372 return runtimes 373 } 374 375 func (in *DashboardsService) buildLabelsQueryString(namespace string, labelsFilters map[string]string) string { 376 namespaceLabel := in.namespaceLabel 377 if namespaceLabel == "" { 378 namespaceLabel = defaultNamespaceLabel 379 } 380 labels := fmt.Sprintf(`{%s="%s"`, namespaceLabel, namespace) 381 for k, v := range labelsFilters { 382 labels += fmt.Sprintf(`,%s="%s"`, prometheus.SanitizeLabelName(k), v) 383 } 384 for labelName, labelValue := range in.promConfig.QueryScope { 385 labels += fmt.Sprintf(`,%s="%s"`, prometheus.SanitizeLabelName(labelName), labelValue) 386 } 387 388 labels += "}" 389 return labels 390 } 391 392 type istioChart struct { 393 models.Chart 394 refName string 395 scale float64 396 } 397 398 func getIstioCharts() []istioChart { 399 istioCharts := []istioChart{ 400 { 401 Chart: models.Chart{ 402 Name: "Request volume", 403 Unit: "ops", 404 Spans: 3, 405 }, 406 refName: "request_count", 407 }, 408 { 409 Chart: models.Chart{ 410 Name: "Request duration", 411 Unit: "seconds", 412 Spans: 3, 413 }, 414 refName: "request_duration_millis", 415 scale: 0.001, 416 }, 417 { 418 Chart: models.Chart{ 419 Name: "Request size", 420 Unit: "bytes", 421 Spans: 3, 422 }, 423 refName: "request_size", 424 }, 425 { 426 Chart: models.Chart{ 427 Name: "Response size", 428 Unit: "bytes", 429 Spans: 3, 430 }, 431 refName: "response_size", 432 }, 433 { 434 Chart: models.Chart{ 435 Name: "Request throughput", 436 Unit: "bitrate", 437 Spans: 3, 438 }, 439 refName: "request_throughput", 440 scale: 8, // Bps to bps 441 }, 442 { 443 Chart: models.Chart{ 444 Name: "Response throughput", 445 Unit: "bitrate", 446 Spans: 3, 447 }, 448 refName: "response_throughput", 449 scale: 8, // Bps to bps 450 }, 451 { 452 Chart: models.Chart{ 453 Name: "gRPC received", 454 Unit: "msgrate", 455 Spans: 3, 456 }, 457 refName: "grpc_received", 458 }, 459 { 460 Chart: models.Chart{ 461 Name: "gRPC sent", 462 Unit: "msgrate", 463 Spans: 3, 464 }, 465 refName: "grpc_sent", 466 }, 467 { 468 Chart: models.Chart{ 469 Name: "TCP opened", 470 Unit: "connrate", 471 Spans: 3, 472 }, 473 refName: "tcp_opened", 474 }, 475 { 476 Chart: models.Chart{ 477 Name: "TCP closed", 478 Unit: "connrate", 479 Spans: 3, 480 }, 481 refName: "tcp_closed", 482 }, 483 { 484 Chart: models.Chart{ 485 Name: "TCP received", 486 Unit: "bitrate", 487 Spans: 3, 488 }, 489 refName: "tcp_received", 490 }, 491 { 492 Chart: models.Chart{ 493 Name: "TCP sent", 494 Unit: "bitrate", 495 Spans: 3, 496 }, 497 refName: "tcp_sent", 498 }, 499 } 500 return istioCharts 501 } 502 503 func GetIstioScaler() func(name string) float64 { 504 charts := getIstioCharts() 505 return func(name string) float64 { 506 for _, c := range charts { 507 if c.refName == name { 508 return c.scale 509 } 510 } 511 return 1.0 512 } 513 } 514 515 // BuildIstioDashboard returns Istio dashboard filled-in with metrics 516 func (in *DashboardsService) BuildIstioDashboard(metrics models.MetricsMap, direction string) *models.MonitoringDashboard { 517 var dashboard models.MonitoringDashboard 518 // Copy dashboard 519 if direction == "inbound" { 520 dashboard = models.PrepareIstioDashboard("Inbound") 521 } else { 522 dashboard = models.PrepareIstioDashboard("Outbound") 523 } 524 525 istioCharts := getIstioCharts() 526 527 for _, chartTpl := range istioCharts { 528 newChart := chartTpl.Chart 529 conversionParams := models.ConversionParams{Scale: 1.0} 530 if chartTpl.scale != 0.0 { 531 conversionParams.Scale = chartTpl.scale 532 } 533 if metrics := metrics[chartTpl.refName]; metrics != nil { 534 newChart.Metrics = metrics 535 } else { 536 newChart.Metrics = []models.Metric{} 537 } 538 dashboard.Charts = append(dashboard.Charts, newChart) 539 } 540 return &dashboard 541 } 542 543 // GetCustomDashboardRefs finds all dashboard IDs and Titles associated to this app and add them to the model 544 func (in *DashboardsService) GetCustomDashboardRefs(namespace, app, version string, pods []*models.Pod) []models.Runtime { 545 if !in.CustomEnabled || app == "" { 546 // Custom dashboards are disabled or the app label is not configured 547 return []models.Runtime{} 548 } 549 550 // A better way to do? 551 var podsCast []models.Pod 552 for _, p := range pods { 553 podsCast = append(podsCast, *p) 554 } 555 runtimes := in.SearchExplicitDashboards(podsCast) 556 557 if len(runtimes) == 0 { 558 cfg := config.Get() 559 discoveryEnabled := cfg.ExternalServices.CustomDashboards.DiscoveryEnabled 560 if discoveryEnabled == config.DashboardsDiscoveryEnabled || 561 (discoveryEnabled == config.DashboardsDiscoveryAuto && 562 len(pods) <= cfg.ExternalServices.CustomDashboards.DiscoveryAutoThreshold) { 563 filters := make(map[string]string) 564 filters[cfg.IstioLabels.AppLabelName] = app 565 if version != "" { 566 filters[cfg.IstioLabels.VersionLabelName] = version 567 } 568 runtimes = in.discoverDashboards(namespace, filters) 569 } 570 } 571 return runtimes 572 } 573 574 func extractUniqueDashboards(pods []models.Pod) []string { 575 // Get uniqueness from plain list rather than map to preserve ordering; anyway, very low amount of objects is expected 576 uniqueRefs := []string{} 577 for _, pod := range pods { 578 // Check for custom dashboards annotation 579 dashboards := extractDashboardsFromAnnotation(pod, "kiali.io/runtimes") 580 dashboards = append(dashboards, extractDashboardsFromAnnotation(pod, "kiali.io/dashboards")...) 581 for _, ref := range dashboards { 582 if ref != "" { 583 exists := false 584 for _, existingRef := range uniqueRefs { 585 if ref == existingRef { 586 exists = true 587 break 588 } 589 } 590 if !exists { 591 uniqueRefs = append(uniqueRefs, ref) 592 } 593 } 594 } 595 } 596 return uniqueRefs 597 } 598 599 func extractDashboardsFromAnnotation(pod models.Pod, annotation string) []string { 600 dashboards := []string{} 601 if rawDashboards, ok := pod.Annotations[annotation]; ok { 602 rawDashboardsSlice := strings.Split(rawDashboards, ",") 603 for _, dashboard := range rawDashboardsSlice { 604 dashboards = append(dashboards, strings.TrimSpace(dashboard)) 605 } 606 } 607 return dashboards 608 }