github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/verify-infra/vmi/vmi_test.go (about) 1 // Copyright (c) 2020, 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 package vmi_test 5 6 import ( 7 "context" 8 "encoding/json" 9 "fmt" 10 "io" 11 "net/http" 12 "os" 13 "time" 14 15 cmconstants "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/certmanager/constants" 16 17 "github.com/Jeffail/gabs/v2" 18 "github.com/hashicorp/go-retryablehttp" 19 . "github.com/onsi/ginkgo/v2" 20 . "github.com/onsi/gomega" 21 "github.com/verrazzano/verrazzano/pkg/k8sutil" 22 "github.com/verrazzano/verrazzano/pkg/vzcr" 23 vzbeta1 "github.com/verrazzano/verrazzano/platform-operator/apis/verrazzano/v1beta1" 24 "github.com/verrazzano/verrazzano/platform-operator/constants" 25 "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/grafana" 26 "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/keycloak" 27 "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/kiali" 28 "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/nginx" 29 "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/opensearch" 30 "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/opensearchdashboards" 31 "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/prometheus/operator" 32 "github.com/verrazzano/verrazzano/tests/e2e/pkg" 33 "github.com/verrazzano/verrazzano/tests/e2e/pkg/test/framework" 34 "github.com/verrazzano/verrazzano/tests/e2e/pkg/vmi" 35 corev1 "k8s.io/api/core/v1" 36 apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" 37 v1 "k8s.io/apimachinery/pkg/apis/meta/v1" 38 ) 39 40 const ( 41 verrazzanoNamespace = "verrazzano-system" 42 loggingNamespace = "verrazzano-logging" 43 esMasterPrefix = "elasticsearch-master-vmi-system-es-master" 44 esMaster0 = esMasterPrefix + "-0" 45 esMaster1 = esMasterPrefix + "-1" 46 esMaster2 = esMasterPrefix + "-2" 47 esData = "vmi-system-es-data" 48 esData1 = esData + "-1" 49 esData2 = esData + "-2" 50 ) 51 52 var ( 53 opensearchIngress = "vmi-system-os-ingest" 54 osdIngress = "vmi-system-osd" 55 prometheusIngress = "vmi-system-prometheus" 56 grafanaIngress = "vmi-system-grafana" 57 ) 58 59 var t = framework.NewTestFramework("vmi") 60 61 func getIngressURLs() (map[string]string, error) { 62 clientset, err := k8sutil.GetKubernetesClientset() 63 if err != nil { 64 return nil, err 65 } 66 ingressList, err := clientset.NetworkingV1().Ingresses(verrazzanoNamespace).List(context.TODO(), v1.ListOptions{}) 67 if err != nil { 68 return nil, err 69 } 70 71 ingressURLs := make(map[string]string) 72 73 for _, ingress := range ingressList.Items { 74 var ingressRules = ingress.Spec.Rules 75 if len(ingressRules) != 1 { 76 return nil, fmt.Errorf("expected ingress %s in namespace %s to have 1 ingress rule, but had %v", 77 ingress.Name, ingress.Namespace, ingressRules) 78 } 79 ingressURLs[ingress.Name] = fmt.Sprintf("https://%s/", ingressRules[0].Host) 80 } 81 return ingressURLs, nil 82 } 83 84 func verrazzanoMonitoringInstanceCRD() (*apiextv1.CustomResourceDefinition, error) { 85 client, err := pkg.APIExtensionsClientSet() 86 if err != nil { 87 return nil, err 88 } 89 crd, err := client.CustomResourceDefinitions().Get(context.TODO(), "verrazzanomonitoringinstances.verrazzano.io", v1.GetOptions{}) 90 if err != nil { 91 return nil, err 92 } 93 return crd, nil 94 } 95 96 func verrazzanoInstallerCRD() (*apiextv1.CustomResourceDefinition, error) { 97 client, err := pkg.APIExtensionsClientSet() 98 if err != nil { 99 return nil, err 100 } 101 crd, err := client.CustomResourceDefinitions().Get(context.TODO(), "verrazzanos.install.verrazzano.io", v1.GetOptions{}) 102 if err != nil { 103 return nil, err 104 } 105 return crd, nil 106 } 107 108 var ( 109 httpClient *retryablehttp.Client 110 creds *pkg.UsernamePassword 111 vmiCRD *apiextv1.CustomResourceDefinition 112 vzCRD *apiextv1.CustomResourceDefinition 113 ingressURLs map[string]string 114 volumeClaims map[string]*corev1.PersistentVolumeClaim 115 elastic *vmi.Opensearch 116 waitTimeout = 15 * time.Minute 117 pollingInterval = 5 * time.Second 118 elasticWaitTimeout = 15 * time.Minute 119 elasticPollingInterval = 5 * time.Second 120 121 vzMonitoringVolumeClaims map[string]*corev1.PersistentVolumeClaim 122 ) 123 124 var beforeSuite = t.BeforeSuiteFunc(func() { 125 var err error 126 kubeconfigPath, err := k8sutil.GetKubeConfigLocation() 127 if err != nil { 128 t.Fail(fmt.Sprintf("Failed to get default kubeconfig path: %s", err.Error())) 129 } 130 vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath) 131 if err != nil { 132 t.Fail(fmt.Sprintf("Failed to get installed Verrazzano resource in the cluster: %v", err)) 133 } 134 if ingressEnabled(vz) { 135 httpClient = pkg.EventuallyVerrazzanoRetryableHTTPClient() 136 } 137 138 Eventually(func() (*apiextv1.CustomResourceDefinition, error) { 139 vzCRD, err = verrazzanoInstallerCRD() 140 return vzCRD, err 141 }, waitTimeout, pollingInterval).ShouldNot(BeNil()) 142 143 Eventually(func() error { 144 ingressURLs, err = getIngressURLs() 145 return err 146 }, waitTimeout, pollingInterval).ShouldNot(HaveOccurred()) 147 148 Eventually(func() error { 149 volumeClaims, err = pkg.GetPersistentVolumeClaims(verrazzanoNamespace) 150 return err 151 }, waitTimeout, pollingInterval).ShouldNot(HaveOccurred()) 152 153 Eventually(func() error { 154 vzMonitoringVolumeClaims, err = pkg.GetPersistentVolumeClaims(constants.VerrazzanoMonitoringNamespace) 155 return err 156 }, waitTimeout, pollingInterval).ShouldNot(HaveOccurred()) 157 158 ok, _ := pkg.IsVerrazzanoMinVersion("1.7.0", kubeconfigPath) 159 elastic = vmi.GetOpensearch("system", ok) 160 if verrazzanoSecretRequired(vz) { 161 creds = pkg.EventuallyGetSystemVMICredentials() 162 } 163 opensearchIngress = elastic.GetOSIngressName() 164 osdIngress = elastic.GetOSDIngressName() 165 }) 166 167 var _ = BeforeSuite(beforeSuite) 168 169 var _ = t.AfterEach(func() {}) 170 171 var _ = t.Describe("VMI", Label("f:infra-lcm"), func() { 172 kubeconfigPath, err := k8sutil.GetKubeConfigLocation() 173 if err != nil { 174 AbortSuite(fmt.Sprintf("Failed to get default kubeconfig path: %s", err.Error())) 175 } 176 vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath) 177 if err != nil { 178 AbortSuite(fmt.Sprintf("Failed to get installed Verrazzano resource in the cluster: %v", err)) 179 } 180 181 t.Context("Check that OpenSearch", func() { 182 if vzcr.IsComponentStatusEnabled(vz, opensearch.ComponentName) { 183 if ingressEnabled(vz) { 184 t.It("endpoint is accessible", Label("f:mesh.ingress"), func() { 185 elasticPodsRunning := func() bool { 186 podName := "vmi-system-es-master" 187 podNamespace := verrazzanoNamespace 188 if elastic.OperatorManaged { 189 podName = "opensearch-es-master" 190 podNamespace = loggingNamespace 191 } 192 result, err := pkg.PodsRunning(podNamespace, []string{podName}) 193 if err != nil { 194 AbortSuite(fmt.Sprintf("Pod %v is not running in the namespace: %v, error: %v", "vmi-system-es-master", verrazzanoNamespace, err)) 195 } 196 return result 197 } 198 Eventually(elasticPodsRunning, waitTimeout, pollingInterval).Should(BeTrue(), "pods did not all show up") 199 Eventually(elasticIngress, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "ingress did not show up") 200 Expect(ingressURLs).To(HaveKey(opensearchIngress), fmt.Sprintf("Ingress %s not found", opensearchIngress)) 201 Eventually(elasticConnected, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "never connected") 202 Eventually(elasticIndicesCreated, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "indices never created") 203 assertOidcIngressByName(opensearchIngress, vz, opensearch.ComponentName) 204 Expect(vz.Status.VerrazzanoInstance.OpenSearchURL).ToNot(BeNil()) 205 }) 206 207 t.It("verrazzano-system Index is accessible", Label("f:observability.logging.es"), 208 func() { 209 if os.Getenv("TEST_ENV") != "LRE" { 210 indexName, err := pkg.GetOpenSearchSystemIndex(verrazzanoNamespace) 211 Expect(err).ShouldNot(HaveOccurred()) 212 pkg.Concurrently( 213 func() { 214 Eventually(func() bool { 215 return pkg.FindLog(indexName, 216 []pkg.Match{ 217 {Key: "kubernetes.container_name", Value: "verrazzano-monitoring-operator"}, 218 {Key: "cluster_name", Value: constants.MCLocalCluster}}, 219 []pkg.Match{}) 220 }, elasticWaitTimeout, pollingInterval).Should(BeTrue(), "Expected to find a verrazzano-monitoring-operator log record") 221 }, 222 func() { 223 Eventually(func() bool { 224 return pkg.FindLog(indexName, 225 []pkg.Match{ 226 {Key: "kubernetes.container_name", Value: "verrazzano-application-operator"}, 227 {Key: "cluster_name", Value: constants.MCLocalCluster}}, 228 []pkg.Match{}) 229 }, elasticWaitTimeout, pollingInterval).Should(BeTrue(), "Expected to find a verrazzano-application-operator log record") 230 }, 231 ) 232 } 233 }) 234 235 t.It("health is green", func() { 236 Eventually(elasticHealth, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "cluster health status not green") 237 Eventually(elasticIndicesHealth, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "indices health status not green") 238 }) 239 240 t.It("systemd journal Index is accessible", Label("f:observability.logging.es"), 241 func() { 242 indexName, err := pkg.GetOpenSearchSystemIndex("systemd-journal") 243 Expect(err).ShouldNot(HaveOccurred()) 244 Eventually(func() bool { 245 return pkg.FindAnyLog(indexName, 246 []pkg.Match{ 247 {Key: "tag", Value: "systemd"}, 248 {Key: "TRANSPORT", Value: "journal"}, 249 {Key: "cluster_name", Value: constants.MCLocalCluster}}, 250 []pkg.Match{}) 251 }, elasticWaitTimeout, pollingInterval).Should(BeTrue(), "Expected to find a systemd log record") 252 }) 253 } 254 } else { 255 t.It("is not running", func() { 256 // Verify ES not present 257 Eventually(func() (bool, error) { 258 podPrefix := []string{"vmi-system-es"} 259 podNamespace := verrazzanoNamespace 260 if elastic.OperatorManaged { 261 podPrefix = []string{"opensearch-es"} 262 podNamespace = loggingNamespace 263 } 264 return pkg.PodsNotRunning(podNamespace, podPrefix) 265 }, waitTimeout, pollingInterval).Should(BeTrue()) 266 Expect(elastic.CheckIngress()).To(BeFalse()) 267 Expect(ingressURLs).NotTo(HaveKey(opensearchIngress), fmt.Sprintf("Ingress %s should not exist", opensearchIngress)) 268 Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.OpenSearchURL == nil).To(BeTrue()) 269 }) 270 } 271 }) 272 273 t.Context("Check that OpenSearch-Dashboards", func() { 274 if vzcr.IsComponentStatusEnabled(vz, opensearchdashboards.ComponentName) { 275 if ingressEnabled(vz) { 276 t.It("endpoint is accessible", Label("f:mesh.ingress", 277 "f:observability.logging.kibana"), func() { 278 osdPodsRunning := func() bool { 279 podName := "vmi-system-osd" 280 podNamespace := verrazzanoNamespace 281 if elastic.OperatorManaged { 282 podName = "opensearch-dashboards" 283 podNamespace = loggingNamespace 284 } 285 result, err := pkg.PodsRunning(podNamespace, []string{podName}) 286 if err != nil { 287 AbortSuite(fmt.Sprintf("Pod %v is not running in the namespace: %v, error: %v", "vmi-system-osd", verrazzanoNamespace, err)) 288 } 289 return result 290 } 291 Eventually(osdPodsRunning, waitTimeout, pollingInterval).Should(BeTrue(), "osd pods did not all show up") 292 Expect(ingressURLs).To(HaveKey(osdIngress), fmt.Sprintf("Ingress %s not found", osdIngress)) 293 assertOidcIngressByName(osdIngress, vz, opensearchdashboards.ComponentName) 294 Expect(vz.Status.VerrazzanoInstance.OpenSearchURL).ToNot(BeNil()) 295 }) 296 } 297 } else { 298 t.It("is not running", func() { 299 300 Eventually(func() (bool, error) { 301 podPrefix := []string{"opensearch-dashboards"} 302 podNamespace := verrazzanoNamespace 303 if elastic.OperatorManaged { 304 podPrefix = []string{"opensearch-es"} 305 podNamespace = loggingNamespace 306 } 307 return pkg.PodsNotRunning(podNamespace, podPrefix) 308 }, waitTimeout, pollingInterval).Should(BeTrue()) 309 Expect(ingressURLs).NotTo(HaveKey(osdIngress), fmt.Sprintf("Ingress %s should not exist", osdIngress)) 310 Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.OpenSearchURL == nil).To(BeTrue()) 311 }) 312 } 313 }) 314 315 t.Context("Check that Prometheus", func() { 316 const stsName = "prometheus-prometheus-operator-kube-p-prometheus" 317 if vzcr.IsComponentStatusEnabled(vz, operator.ComponentName) { 318 t.It("helm override for replicas is in effect", Label("f:observability.monitoring.prom"), func() { 319 expectedReplicas, err := getExpectedPrometheusReplicaCount(kubeconfigPath) 320 Expect(err).ToNot(HaveOccurred()) 321 322 // expect Prometheus statefulset to be configured for the expected number of replicas 323 sts, err := pkg.GetStatefulSet(constants.VerrazzanoMonitoringNamespace, stsName) 324 Expect(err).ToNot(HaveOccurred()) 325 Expect(sts.Spec.Replicas).ToNot(BeNil()) 326 Expect(*sts.Spec.Replicas).To(Equal(expectedReplicas)) 327 328 // expect the replicas to be ready 329 Eventually(func() (int32, error) { 330 sts, err := pkg.GetStatefulSet(constants.VerrazzanoMonitoringNamespace, stsName) 331 if err != nil { 332 return 0, err 333 } 334 return sts.Status.ReadyReplicas, nil 335 }, waitTimeout, pollingInterval).Should(Equal(expectedReplicas), 336 fmt.Sprintf("Statefulset %s in namespace %s does not have the expected number of ready replicas", stsName, constants.VerrazzanoMonitoringNamespace)) 337 }) 338 if ingressEnabled(vz) { 339 t.It("endpoint is accessible", Label("f:mesh.ingress", 340 "f:observability.monitoring.prom"), func() { 341 assertOidcIngressByName(prometheusIngress, vz, operator.ComponentName) 342 Expect(vz.Status.VerrazzanoInstance.PrometheusURL).ToNot(BeNil()) 343 }) 344 } 345 346 } else { 347 t.It("is not running", func() { 348 Eventually(func() (bool, error) { 349 return pkg.PodsNotRunning(verrazzanoNamespace, []string{stsName}) 350 }, waitTimeout, pollingInterval).Should(BeTrue()) 351 Expect(ingressURLs).NotTo(HaveKey(prometheusIngress), fmt.Sprintf("Ingress %s should not exist", prometheusIngress)) 352 Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.PrometheusURL == nil).To(BeTrue()) 353 }) 354 } 355 }) 356 357 t.Context("Check that Grafana", func() { 358 if vzcr.IsComponentStatusEnabled(vz, grafana.ComponentName) { 359 t.It("VMI is created successfully", func() { 360 Eventually(func() (*apiextv1.CustomResourceDefinition, error) { 361 vmiCRD, err = verrazzanoMonitoringInstanceCRD() 362 return vmiCRD, err 363 }, waitTimeout, pollingInterval).ShouldNot(BeNil()) 364 }) 365 366 if ingressEnabled(vz) { 367 t.It("Grafana endpoint should be accessible", Label("f:mesh.ingress", 368 "f:observability.monitoring.graf"), func() { 369 Expect(ingressURLs).To(HaveKey("vmi-system-grafana"), "Ingress vmi-system-grafana not found") 370 Expect(vz.Status.VerrazzanoInstance.GrafanaURL).ToNot(BeNil()) 371 }) 372 373 t.It("Default dashboard should be installed in System Grafana for shared VMI", 374 Label("f:observability.monitoring.graf"), func() { 375 pkg.Concurrently( 376 func() { assertDashboard("WebLogic%20Server%20Dashboard") }, 377 func() { assertDashboard("Coherence%20Elastic%20Data%20Summary%20Dashboard") }, 378 func() { assertDashboard("Coherence%20Persistence%20Summary%20Dashboard") }, 379 func() { assertDashboard("Coherence%20Cache%20Details%20Dashboard") }, 380 func() { assertDashboard("Coherence%20Members%20Summary%20Dashboard") }, 381 func() { assertDashboard("Coherence%20Kubernetes%20Summary%20Dashboard") }, 382 func() { assertDashboard("Coherence%20Dashboard%20Main") }, 383 func() { assertDashboard("Coherence%20Caches%20Summary%20Dashboard") }, 384 func() { assertDashboard("Coherence%20Service%20Details%20Dashboard") }, 385 func() { assertDashboard("Coherence%20Proxy%20Servers%20Summary%20Dashboard") }, 386 func() { assertDashboard("Coherence%20Federation%20Details%20Dashboard") }, 387 func() { assertDashboard("Coherence%20Federation%20Summary%20Dashboard") }, 388 func() { assertDashboard("Coherence%20Services%20Summary%20Dashboard") }, 389 func() { assertDashboard("Coherence%20HTTP%20Servers%20Summary%20Dashboard") }, 390 func() { assertDashboard("Coherence%20Proxy%20Server%20Detail%20Dashboard") }, 391 func() { assertDashboard("Coherence%20Alerts%20Dashboard") }, 392 func() { assertDashboard("Coherence%20Member%20Details%20Dashboard") }, 393 func() { assertDashboard("Coherence%20Machines%20Summary%20Dashboard") }, 394 ) 395 }) 396 t.ItMinimumVersion("Grafana should have the verrazzano user with admin privileges", "1.3.0", kubeconfigPath, func() { 397 vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath) 398 if err != nil { 399 t.Logs.Errorf("Error getting Verrazzano resource: %v", err) 400 Fail(err.Error()) 401 } 402 if vz.Spec.Version != "" { 403 t.Logs.Info("Skipping test because Verrazzano has been upgraded %s") 404 } else { 405 Eventually(assertAdminRole, waitTimeout, pollingInterval).Should(BeTrue()) 406 } 407 }) 408 409 t.It("Grafana should have a default datasource present", func() { 410 vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath) 411 if err != nil { 412 t.Logs.Errorf("Error getting Verrazzano resource: %v", err) 413 Fail(err.Error()) 414 } 415 name := "Prometheus" 416 if vzcr.IsThanosEnabled(vz) { 417 name = "Thanos" 418 } 419 Eventually(func() (bool, error) { 420 return grafanaDefaultDatasourceExists(vz, name, kubeconfigPath) 421 }).WithTimeout(waitTimeout).WithPolling(pollingInterval).Should(BeTrue()) 422 }) 423 } 424 425 } else { 426 t.It("is not running", func() { 427 Eventually(func() (bool, error) { 428 return pkg.PodsNotRunning(verrazzanoNamespace, []string{"vmi-system-grafana"}) 429 }, waitTimeout, pollingInterval).Should(BeTrue()) 430 Expect(ingressURLs).NotTo(HaveKey(grafanaIngress), fmt.Sprintf("Ingress %s should not exist", grafanaIngress)) 431 Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.GrafanaURL == nil).To(BeTrue()) 432 }) 433 } 434 }) 435 }) 436 437 func assertOidcIngressByName(key string, vz *vzbeta1.Verrazzano, componentName string) { 438 if ingressEnabled(vz) { 439 Expect(ingressURLs).To(HaveKey(key), fmt.Sprintf("Ingress %s not found", key)) 440 url := ingressURLs[key] 441 assertOidcIngress(url) 442 } else { 443 t.Logs.Infof("Skipping checking ingress %s because ingress-nginx or cert-manager is not installed", key) 444 } 445 446 } 447 448 func assertOidcIngress(url string) { 449 unauthHTTPClient := pkg.EventuallyVerrazzanoRetryableHTTPClient() 450 pkg.Concurrently( 451 func() { 452 Eventually(func() bool { 453 return pkg.AssertOauthURLAccessibleAndUnauthorized(unauthHTTPClient, url) 454 }, waitTimeout, pollingInterval).Should(BeTrue()) 455 }, 456 func() { 457 Eventually(func() bool { 458 return pkg.AssertURLAccessibleAndAuthorized(httpClient, url, creds) 459 }, waitTimeout, pollingInterval).Should(BeTrue()) 460 }, 461 func() { 462 Eventually(func() bool { 463 return pkg.AssertBearerAuthorized(httpClient, url) 464 }, waitTimeout, pollingInterval).Should(BeTrue()) 465 }, 466 ) 467 } 468 469 func elasticIndicesCreated() bool { 470 b, _ := ContainElements(".kibana_1").Match(elastic.ListIndices()) 471 return b 472 } 473 474 func elasticConnected() bool { 475 return elastic.Connect() 476 } 477 478 func elasticHealth() bool { 479 kubeconfigPath, err := k8sutil.GetKubeConfigLocation() 480 if err != nil { 481 t.Logs.Errorf("Failed to get default kubeconfig path: %s", err.Error()) 482 return false 483 } 484 return elastic.CheckHealth(kubeconfigPath) 485 } 486 487 func elasticIndicesHealth() bool { 488 kubeconfigPath, err := k8sutil.GetKubeConfigLocation() 489 if err != nil { 490 t.Logs.Errorf("Failed to get default kubeconfig path: %s", err.Error()) 491 return false 492 } 493 return elastic.CheckIndicesHealth(kubeconfigPath) 494 } 495 496 func elasticIngress() bool { 497 return elastic.CheckIngress() 498 } 499 500 func assertDashboard(url string) { 501 searchURL := fmt.Sprintf("%sapi/search?query=%s", ingressURLs["vmi-system-grafana"], url) 502 fmt.Println("Grafana URL in browseGrafanaDashboard ", searchURL) 503 504 searchDashboard := func() bool { 505 vmiHTTPClient := pkg.EventuallyVerrazzanoRetryableHTTPClient() 506 vmiHTTPClient.HTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { 507 return http.ErrUseLastResponse 508 } 509 510 req, err := retryablehttp.NewRequest("GET", searchURL, nil) 511 if err != nil { 512 t.Logs.Errorf("Error creating HTTP request: %v", err) 513 return false 514 } 515 req.SetBasicAuth(creds.Username, creds.Password) 516 resp, err := vmiHTTPClient.Do(req) 517 if err != nil { 518 t.Logs.Errorf("Error making HTTP request: %v", err) 519 return false 520 } 521 if resp.StatusCode != http.StatusOK { 522 t.Logs.Errorf("Unexpected HTTP status code: %d", resp.StatusCode) 523 return false 524 } 525 // assert that there is a single item in response 526 defer resp.Body.Close() 527 bodyBytes, err := io.ReadAll(resp.Body) 528 if err != nil { 529 t.Logs.Errorf("Unable to read body from response: %v", err) 530 return false 531 } 532 var response []map[string]interface{} 533 if err := json.Unmarshal(bodyBytes, &response); err != nil { 534 t.Logs.Errorf("Error unmarshaling response body: %v", err) 535 return false 536 } 537 if len(response) != 1 { 538 t.Logs.Errorf("Unexpected response length: %d", len(response)) 539 return false 540 } 541 return true 542 } 543 Eventually(searchDashboard, waitTimeout, pollingInterval).Should(BeTrue()) 544 } 545 546 func assertAdminRole() bool { 547 searchURL := fmt.Sprintf("%sapi/users", ingressURLs["vmi-system-grafana"]) 548 vmiHTTPClient := pkg.EventuallyVerrazzanoRetryableHTTPClient() 549 vmiHTTPClient.HTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error { 550 return http.ErrUseLastResponse 551 } 552 553 req, err := retryablehttp.NewRequest("GET", searchURL, nil) 554 if err != nil { 555 t.Logs.Errorf("Error creating HTTP request: %v", err) 556 return false 557 } 558 req.SetBasicAuth(creds.Username, creds.Password) 559 resp, err := vmiHTTPClient.Do(req) 560 if err != nil { 561 t.Logs.Errorf("Error making HTTP request: %v", err) 562 return false 563 } 564 if resp.StatusCode != http.StatusOK { 565 t.Logs.Errorf("Unexpected HTTP status code: %d", resp.StatusCode) 566 return false 567 } 568 // assert that there is a single item in response 569 defer resp.Body.Close() 570 bodyBytes, err := io.ReadAll(resp.Body) 571 if err != nil { 572 t.Logs.Errorf("Unable to read body from response: %v", err) 573 return false 574 } 575 var response []map[string]interface{} 576 if err := json.Unmarshal(bodyBytes, &response); err != nil { 577 t.Logs.Errorf("Error unmarshaling response body: %v", err) 578 return false 579 } 580 if len(response) != 1 { 581 t.Logs.Errorf("Unexpected response length: %d", len(response)) 582 return false 583 } 584 t.Logs.Infof("Grafana users: %s", response) 585 return response[0]["login"] == "verrazzano" && response[0]["isAdmin"] == true 586 } 587 588 // getExpectedPrometheusReplicaCount returns the Prometheus replicas in the values overrides from the 589 // Prometheus Operator component in the Verrazzano CR. If there is no override for replicas then the 590 // default replica count of 1 is returned. 591 func getExpectedPrometheusReplicaCount(kubeconfig string) (int32, error) { 592 vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfig) 593 if err != nil { 594 return 0, err 595 } 596 if !vzcr.IsComponentStatusEnabled(vz, operator.ComponentName) { 597 return 0, nil 598 } 599 var expectedReplicas int32 = 1 600 if vz.Spec.Components.PrometheusOperator == nil { 601 return expectedReplicas, nil 602 } 603 604 for _, override := range vz.Spec.Components.PrometheusOperator.InstallOverrides.ValueOverrides { 605 if override.Values != nil { 606 jsonString, err := gabs.ParseJSON(override.Values.Raw) 607 if err != nil { 608 return 0, err 609 } 610 if container := jsonString.Path("prometheus.prometheusSpec.replicas"); container != nil { 611 if val, ok := container.Data().(float64); ok { 612 expectedReplicas = int32(val) 613 t.Logs.Infof("Found Prometheus replicas override in Verrazzano CR, replica count is: %d", expectedReplicas) 614 break 615 } 616 } 617 } 618 } 619 620 return expectedReplicas, nil 621 } 622 623 func ingressEnabled(vz *vzbeta1.Verrazzano) bool { 624 return vzcr.IsComponentStatusEnabled(vz, nginx.ComponentName) && 625 vzcr.IsComponentStatusEnabled(vz, cmconstants.ClusterIssuerComponentName) && 626 vzcr.IsComponentStatusEnabled(vz, keycloak.ComponentName) 627 } 628 629 func verrazzanoSecretRequired(vz *vzbeta1.Verrazzano) bool { 630 return vzcr.IsComponentStatusEnabled(vz, opensearch.ComponentName) || 631 vzcr.IsComponentStatusEnabled(vz, opensearchdashboards.ComponentName) || 632 vzcr.IsComponentStatusEnabled(vz, operator.ComponentName) || 633 vzcr.IsComponentStatusEnabled(vz, grafana.ComponentName) || 634 vzcr.IsComponentStatusEnabled(vz, kiali.ComponentName) 635 } 636 637 func grafanaDefaultDatasourceExists(vz *vzbeta1.Verrazzano, name, kubeconfigPath string) (bool, error) { 638 password, err := pkg.GetVerrazzanoPasswordInCluster(kubeconfigPath) 639 if err != nil { 640 t.Logs.Error("Failed to get the Verrazzano password from the cluster") 641 return false, err 642 } 643 if vz == nil || vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.GrafanaURL == nil { 644 t.Logs.Error("Grafana URL in the Verrazzano status is empty") 645 return false, nil 646 } 647 resp, err := pkg.GetWebPageWithBasicAuth(*vz.Status.VerrazzanoInstance.GrafanaURL+"/api/datasources", "", "verrazzano", password, kubeconfigPath) 648 if err != nil { 649 t.Logs.Errorf("Failed to get Grafana datasources: %v", err) 650 return false, err 651 } 652 653 var datasources []map[string]interface{} 654 err = json.Unmarshal(resp.Body, &datasources) 655 if err != nil { 656 t.Logs.Errorf("Failed to unmarshal Grafana datasources: %v", err) 657 return false, err 658 } 659 660 for _, source := range datasources { 661 sourceName, ok := source["name"] 662 if !ok { 663 t.Logs.Errorf("Failed to find name for Grafana datasource") 664 continue 665 } 666 667 nameStr, ok := sourceName.(string) 668 if !ok { 669 t.Logs.Errorf("Failed to convert name field to string") 670 continue 671 } 672 if nameStr != name { 673 continue 674 } 675 676 sourceDefault, ok := source["isDefault"] 677 if !ok { 678 t.Logs.Errorf("Failed to verify the datasource was the default") 679 continue 680 } 681 defaultBool, ok := sourceDefault.(bool) 682 if !ok { 683 t.Logs.Errorf("Failed to convert default to bool") 684 continue 685 } 686 return defaultBool, nil 687 } 688 689 t.Logs.Errorf("Failed to find Grafana datasource %s", name) 690 return false, nil 691 }