github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/verify-infra/vmi/vmi_test.go (about)

     1  // Copyright (c) 2020, 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  package vmi_test
     5  
     6  import (
     7  	"context"
     8  	"encoding/json"
     9  	"fmt"
    10  	"io"
    11  	"net/http"
    12  	"os"
    13  	"time"
    14  
    15  	cmconstants "github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/certmanager/constants"
    16  
    17  	"github.com/Jeffail/gabs/v2"
    18  	"github.com/hashicorp/go-retryablehttp"
    19  	. "github.com/onsi/ginkgo/v2"
    20  	. "github.com/onsi/gomega"
    21  	"github.com/verrazzano/verrazzano/pkg/k8sutil"
    22  	"github.com/verrazzano/verrazzano/pkg/vzcr"
    23  	vzbeta1 "github.com/verrazzano/verrazzano/platform-operator/apis/verrazzano/v1beta1"
    24  	"github.com/verrazzano/verrazzano/platform-operator/constants"
    25  	"github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/grafana"
    26  	"github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/keycloak"
    27  	"github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/kiali"
    28  	"github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/nginx"
    29  	"github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/opensearch"
    30  	"github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/opensearchdashboards"
    31  	"github.com/verrazzano/verrazzano/platform-operator/controllers/verrazzano/component/prometheus/operator"
    32  	"github.com/verrazzano/verrazzano/tests/e2e/pkg"
    33  	"github.com/verrazzano/verrazzano/tests/e2e/pkg/test/framework"
    34  	"github.com/verrazzano/verrazzano/tests/e2e/pkg/vmi"
    35  	corev1 "k8s.io/api/core/v1"
    36  	apiextv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
    37  	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
    38  )
    39  
    40  const (
    41  	verrazzanoNamespace = "verrazzano-system"
    42  	loggingNamespace    = "verrazzano-logging"
    43  	esMasterPrefix      = "elasticsearch-master-vmi-system-es-master"
    44  	esMaster0           = esMasterPrefix + "-0"
    45  	esMaster1           = esMasterPrefix + "-1"
    46  	esMaster2           = esMasterPrefix + "-2"
    47  	esData              = "vmi-system-es-data"
    48  	esData1             = esData + "-1"
    49  	esData2             = esData + "-2"
    50  )
    51  
    52  var (
    53  	opensearchIngress = "vmi-system-os-ingest"
    54  	osdIngress        = "vmi-system-osd"
    55  	prometheusIngress = "vmi-system-prometheus"
    56  	grafanaIngress    = "vmi-system-grafana"
    57  )
    58  
    59  var t = framework.NewTestFramework("vmi")
    60  
    61  func getIngressURLs() (map[string]string, error) {
    62  	clientset, err := k8sutil.GetKubernetesClientset()
    63  	if err != nil {
    64  		return nil, err
    65  	}
    66  	ingressList, err := clientset.NetworkingV1().Ingresses(verrazzanoNamespace).List(context.TODO(), v1.ListOptions{})
    67  	if err != nil {
    68  		return nil, err
    69  	}
    70  
    71  	ingressURLs := make(map[string]string)
    72  
    73  	for _, ingress := range ingressList.Items {
    74  		var ingressRules = ingress.Spec.Rules
    75  		if len(ingressRules) != 1 {
    76  			return nil, fmt.Errorf("expected ingress %s in namespace %s to have 1 ingress rule, but had %v",
    77  				ingress.Name, ingress.Namespace, ingressRules)
    78  		}
    79  		ingressURLs[ingress.Name] = fmt.Sprintf("https://%s/", ingressRules[0].Host)
    80  	}
    81  	return ingressURLs, nil
    82  }
    83  
    84  func verrazzanoMonitoringInstanceCRD() (*apiextv1.CustomResourceDefinition, error) {
    85  	client, err := pkg.APIExtensionsClientSet()
    86  	if err != nil {
    87  		return nil, err
    88  	}
    89  	crd, err := client.CustomResourceDefinitions().Get(context.TODO(), "verrazzanomonitoringinstances.verrazzano.io", v1.GetOptions{})
    90  	if err != nil {
    91  		return nil, err
    92  	}
    93  	return crd, nil
    94  }
    95  
    96  func verrazzanoInstallerCRD() (*apiextv1.CustomResourceDefinition, error) {
    97  	client, err := pkg.APIExtensionsClientSet()
    98  	if err != nil {
    99  		return nil, err
   100  	}
   101  	crd, err := client.CustomResourceDefinitions().Get(context.TODO(), "verrazzanos.install.verrazzano.io", v1.GetOptions{})
   102  	if err != nil {
   103  		return nil, err
   104  	}
   105  	return crd, nil
   106  }
   107  
   108  var (
   109  	httpClient             *retryablehttp.Client
   110  	creds                  *pkg.UsernamePassword
   111  	vmiCRD                 *apiextv1.CustomResourceDefinition
   112  	vzCRD                  *apiextv1.CustomResourceDefinition
   113  	ingressURLs            map[string]string
   114  	volumeClaims           map[string]*corev1.PersistentVolumeClaim
   115  	elastic                *vmi.Opensearch
   116  	waitTimeout            = 15 * time.Minute
   117  	pollingInterval        = 5 * time.Second
   118  	elasticWaitTimeout     = 15 * time.Minute
   119  	elasticPollingInterval = 5 * time.Second
   120  
   121  	vzMonitoringVolumeClaims map[string]*corev1.PersistentVolumeClaim
   122  )
   123  
   124  var beforeSuite = t.BeforeSuiteFunc(func() {
   125  	var err error
   126  	kubeconfigPath, err := k8sutil.GetKubeConfigLocation()
   127  	if err != nil {
   128  		t.Fail(fmt.Sprintf("Failed to get default kubeconfig path: %s", err.Error()))
   129  	}
   130  	vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath)
   131  	if err != nil {
   132  		t.Fail(fmt.Sprintf("Failed to get installed Verrazzano resource in the cluster: %v", err))
   133  	}
   134  	if ingressEnabled(vz) {
   135  		httpClient = pkg.EventuallyVerrazzanoRetryableHTTPClient()
   136  	}
   137  
   138  	Eventually(func() (*apiextv1.CustomResourceDefinition, error) {
   139  		vzCRD, err = verrazzanoInstallerCRD()
   140  		return vzCRD, err
   141  	}, waitTimeout, pollingInterval).ShouldNot(BeNil())
   142  
   143  	Eventually(func() error {
   144  		ingressURLs, err = getIngressURLs()
   145  		return err
   146  	}, waitTimeout, pollingInterval).ShouldNot(HaveOccurred())
   147  
   148  	Eventually(func() error {
   149  		volumeClaims, err = pkg.GetPersistentVolumeClaims(verrazzanoNamespace)
   150  		return err
   151  	}, waitTimeout, pollingInterval).ShouldNot(HaveOccurred())
   152  
   153  	Eventually(func() error {
   154  		vzMonitoringVolumeClaims, err = pkg.GetPersistentVolumeClaims(constants.VerrazzanoMonitoringNamespace)
   155  		return err
   156  	}, waitTimeout, pollingInterval).ShouldNot(HaveOccurred())
   157  
   158  	ok, _ := pkg.IsVerrazzanoMinVersion("1.7.0", kubeconfigPath)
   159  	elastic = vmi.GetOpensearch("system", ok)
   160  	if verrazzanoSecretRequired(vz) {
   161  		creds = pkg.EventuallyGetSystemVMICredentials()
   162  	}
   163  	opensearchIngress = elastic.GetOSIngressName()
   164  	osdIngress = elastic.GetOSDIngressName()
   165  })
   166  
   167  var _ = BeforeSuite(beforeSuite)
   168  
   169  var _ = t.AfterEach(func() {})
   170  
   171  var _ = t.Describe("VMI", Label("f:infra-lcm"), func() {
   172  	kubeconfigPath, err := k8sutil.GetKubeConfigLocation()
   173  	if err != nil {
   174  		AbortSuite(fmt.Sprintf("Failed to get default kubeconfig path: %s", err.Error()))
   175  	}
   176  	vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath)
   177  	if err != nil {
   178  		AbortSuite(fmt.Sprintf("Failed to get installed Verrazzano resource in the cluster: %v", err))
   179  	}
   180  
   181  	t.Context("Check that OpenSearch", func() {
   182  		if vzcr.IsComponentStatusEnabled(vz, opensearch.ComponentName) {
   183  			if ingressEnabled(vz) {
   184  				t.It("endpoint is accessible", Label("f:mesh.ingress"), func() {
   185  					elasticPodsRunning := func() bool {
   186  						podName := "vmi-system-es-master"
   187  						podNamespace := verrazzanoNamespace
   188  						if elastic.OperatorManaged {
   189  							podName = "opensearch-es-master"
   190  							podNamespace = loggingNamespace
   191  						}
   192  						result, err := pkg.PodsRunning(podNamespace, []string{podName})
   193  						if err != nil {
   194  							AbortSuite(fmt.Sprintf("Pod %v is not running in the namespace: %v, error: %v", "vmi-system-es-master", verrazzanoNamespace, err))
   195  						}
   196  						return result
   197  					}
   198  					Eventually(elasticPodsRunning, waitTimeout, pollingInterval).Should(BeTrue(), "pods did not all show up")
   199  					Eventually(elasticIngress, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "ingress did not show up")
   200  					Expect(ingressURLs).To(HaveKey(opensearchIngress), fmt.Sprintf("Ingress %s not found", opensearchIngress))
   201  					Eventually(elasticConnected, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "never connected")
   202  					Eventually(elasticIndicesCreated, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "indices never created")
   203  					assertOidcIngressByName(opensearchIngress, vz, opensearch.ComponentName)
   204  					Expect(vz.Status.VerrazzanoInstance.OpenSearchURL).ToNot(BeNil())
   205  				})
   206  
   207  				t.It("verrazzano-system Index is accessible", Label("f:observability.logging.es"),
   208  					func() {
   209  						if os.Getenv("TEST_ENV") != "LRE" {
   210  							indexName, err := pkg.GetOpenSearchSystemIndex(verrazzanoNamespace)
   211  							Expect(err).ShouldNot(HaveOccurred())
   212  							pkg.Concurrently(
   213  								func() {
   214  									Eventually(func() bool {
   215  										return pkg.FindLog(indexName,
   216  											[]pkg.Match{
   217  												{Key: "kubernetes.container_name", Value: "verrazzano-monitoring-operator"},
   218  												{Key: "cluster_name", Value: constants.MCLocalCluster}},
   219  											[]pkg.Match{})
   220  									}, elasticWaitTimeout, pollingInterval).Should(BeTrue(), "Expected to find a verrazzano-monitoring-operator log record")
   221  								},
   222  								func() {
   223  									Eventually(func() bool {
   224  										return pkg.FindLog(indexName,
   225  											[]pkg.Match{
   226  												{Key: "kubernetes.container_name", Value: "verrazzano-application-operator"},
   227  												{Key: "cluster_name", Value: constants.MCLocalCluster}},
   228  											[]pkg.Match{})
   229  									}, elasticWaitTimeout, pollingInterval).Should(BeTrue(), "Expected to find a verrazzano-application-operator log record")
   230  								},
   231  							)
   232  						}
   233  					})
   234  
   235  				t.It("health is green", func() {
   236  					Eventually(elasticHealth, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "cluster health status not green")
   237  					Eventually(elasticIndicesHealth, elasticWaitTimeout, elasticPollingInterval).Should(BeTrue(), "indices health status not green")
   238  				})
   239  
   240  				t.It("systemd journal Index is accessible", Label("f:observability.logging.es"),
   241  					func() {
   242  						indexName, err := pkg.GetOpenSearchSystemIndex("systemd-journal")
   243  						Expect(err).ShouldNot(HaveOccurred())
   244  						Eventually(func() bool {
   245  							return pkg.FindAnyLog(indexName,
   246  								[]pkg.Match{
   247  									{Key: "tag", Value: "systemd"},
   248  									{Key: "TRANSPORT", Value: "journal"},
   249  									{Key: "cluster_name", Value: constants.MCLocalCluster}},
   250  								[]pkg.Match{})
   251  						}, elasticWaitTimeout, pollingInterval).Should(BeTrue(), "Expected to find a systemd log record")
   252  					})
   253  			}
   254  		} else {
   255  			t.It("is not running", func() {
   256  				// Verify ES not present
   257  				Eventually(func() (bool, error) {
   258  					podPrefix := []string{"vmi-system-es"}
   259  					podNamespace := verrazzanoNamespace
   260  					if elastic.OperatorManaged {
   261  						podPrefix = []string{"opensearch-es"}
   262  						podNamespace = loggingNamespace
   263  					}
   264  					return pkg.PodsNotRunning(podNamespace, podPrefix)
   265  				}, waitTimeout, pollingInterval).Should(BeTrue())
   266  				Expect(elastic.CheckIngress()).To(BeFalse())
   267  				Expect(ingressURLs).NotTo(HaveKey(opensearchIngress), fmt.Sprintf("Ingress %s should not exist", opensearchIngress))
   268  				Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.OpenSearchURL == nil).To(BeTrue())
   269  			})
   270  		}
   271  	})
   272  
   273  	t.Context("Check that OpenSearch-Dashboards", func() {
   274  		if vzcr.IsComponentStatusEnabled(vz, opensearchdashboards.ComponentName) {
   275  			if ingressEnabled(vz) {
   276  				t.It("endpoint is accessible", Label("f:mesh.ingress",
   277  					"f:observability.logging.kibana"), func() {
   278  					osdPodsRunning := func() bool {
   279  						podName := "vmi-system-osd"
   280  						podNamespace := verrazzanoNamespace
   281  						if elastic.OperatorManaged {
   282  							podName = "opensearch-dashboards"
   283  							podNamespace = loggingNamespace
   284  						}
   285  						result, err := pkg.PodsRunning(podNamespace, []string{podName})
   286  						if err != nil {
   287  							AbortSuite(fmt.Sprintf("Pod %v is not running in the namespace: %v, error: %v", "vmi-system-osd", verrazzanoNamespace, err))
   288  						}
   289  						return result
   290  					}
   291  					Eventually(osdPodsRunning, waitTimeout, pollingInterval).Should(BeTrue(), "osd pods did not all show up")
   292  					Expect(ingressURLs).To(HaveKey(osdIngress), fmt.Sprintf("Ingress %s not found", osdIngress))
   293  					assertOidcIngressByName(osdIngress, vz, opensearchdashboards.ComponentName)
   294  					Expect(vz.Status.VerrazzanoInstance.OpenSearchURL).ToNot(BeNil())
   295  				})
   296  			}
   297  		} else {
   298  			t.It("is not running", func() {
   299  
   300  				Eventually(func() (bool, error) {
   301  					podPrefix := []string{"opensearch-dashboards"}
   302  					podNamespace := verrazzanoNamespace
   303  					if elastic.OperatorManaged {
   304  						podPrefix = []string{"opensearch-es"}
   305  						podNamespace = loggingNamespace
   306  					}
   307  					return pkg.PodsNotRunning(podNamespace, podPrefix)
   308  				}, waitTimeout, pollingInterval).Should(BeTrue())
   309  				Expect(ingressURLs).NotTo(HaveKey(osdIngress), fmt.Sprintf("Ingress %s should not exist", osdIngress))
   310  				Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.OpenSearchURL == nil).To(BeTrue())
   311  			})
   312  		}
   313  	})
   314  
   315  	t.Context("Check that Prometheus", func() {
   316  		const stsName = "prometheus-prometheus-operator-kube-p-prometheus"
   317  		if vzcr.IsComponentStatusEnabled(vz, operator.ComponentName) {
   318  			t.It("helm override for replicas is in effect", Label("f:observability.monitoring.prom"), func() {
   319  				expectedReplicas, err := getExpectedPrometheusReplicaCount(kubeconfigPath)
   320  				Expect(err).ToNot(HaveOccurred())
   321  
   322  				// expect Prometheus statefulset to be configured for the expected number of replicas
   323  				sts, err := pkg.GetStatefulSet(constants.VerrazzanoMonitoringNamespace, stsName)
   324  				Expect(err).ToNot(HaveOccurred())
   325  				Expect(sts.Spec.Replicas).ToNot(BeNil())
   326  				Expect(*sts.Spec.Replicas).To(Equal(expectedReplicas))
   327  
   328  				// expect the replicas to be ready
   329  				Eventually(func() (int32, error) {
   330  					sts, err := pkg.GetStatefulSet(constants.VerrazzanoMonitoringNamespace, stsName)
   331  					if err != nil {
   332  						return 0, err
   333  					}
   334  					return sts.Status.ReadyReplicas, nil
   335  				}, waitTimeout, pollingInterval).Should(Equal(expectedReplicas),
   336  					fmt.Sprintf("Statefulset %s in namespace %s does not have the expected number of ready replicas", stsName, constants.VerrazzanoMonitoringNamespace))
   337  			})
   338  			if ingressEnabled(vz) {
   339  				t.It("endpoint is accessible", Label("f:mesh.ingress",
   340  					"f:observability.monitoring.prom"), func() {
   341  					assertOidcIngressByName(prometheusIngress, vz, operator.ComponentName)
   342  					Expect(vz.Status.VerrazzanoInstance.PrometheusURL).ToNot(BeNil())
   343  				})
   344  			}
   345  
   346  		} else {
   347  			t.It("is not running", func() {
   348  				Eventually(func() (bool, error) {
   349  					return pkg.PodsNotRunning(verrazzanoNamespace, []string{stsName})
   350  				}, waitTimeout, pollingInterval).Should(BeTrue())
   351  				Expect(ingressURLs).NotTo(HaveKey(prometheusIngress), fmt.Sprintf("Ingress %s should not exist", prometheusIngress))
   352  				Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.PrometheusURL == nil).To(BeTrue())
   353  			})
   354  		}
   355  	})
   356  
   357  	t.Context("Check that Grafana", func() {
   358  		if vzcr.IsComponentStatusEnabled(vz, grafana.ComponentName) {
   359  			t.It("VMI is created successfully", func() {
   360  				Eventually(func() (*apiextv1.CustomResourceDefinition, error) {
   361  					vmiCRD, err = verrazzanoMonitoringInstanceCRD()
   362  					return vmiCRD, err
   363  				}, waitTimeout, pollingInterval).ShouldNot(BeNil())
   364  			})
   365  
   366  			if ingressEnabled(vz) {
   367  				t.It("Grafana endpoint should be accessible", Label("f:mesh.ingress",
   368  					"f:observability.monitoring.graf"), func() {
   369  					Expect(ingressURLs).To(HaveKey("vmi-system-grafana"), "Ingress vmi-system-grafana not found")
   370  					Expect(vz.Status.VerrazzanoInstance.GrafanaURL).ToNot(BeNil())
   371  				})
   372  
   373  				t.It("Default dashboard should be installed in System Grafana for shared VMI",
   374  					Label("f:observability.monitoring.graf"), func() {
   375  						pkg.Concurrently(
   376  							func() { assertDashboard("WebLogic%20Server%20Dashboard") },
   377  							func() { assertDashboard("Coherence%20Elastic%20Data%20Summary%20Dashboard") },
   378  							func() { assertDashboard("Coherence%20Persistence%20Summary%20Dashboard") },
   379  							func() { assertDashboard("Coherence%20Cache%20Details%20Dashboard") },
   380  							func() { assertDashboard("Coherence%20Members%20Summary%20Dashboard") },
   381  							func() { assertDashboard("Coherence%20Kubernetes%20Summary%20Dashboard") },
   382  							func() { assertDashboard("Coherence%20Dashboard%20Main") },
   383  							func() { assertDashboard("Coherence%20Caches%20Summary%20Dashboard") },
   384  							func() { assertDashboard("Coherence%20Service%20Details%20Dashboard") },
   385  							func() { assertDashboard("Coherence%20Proxy%20Servers%20Summary%20Dashboard") },
   386  							func() { assertDashboard("Coherence%20Federation%20Details%20Dashboard") },
   387  							func() { assertDashboard("Coherence%20Federation%20Summary%20Dashboard") },
   388  							func() { assertDashboard("Coherence%20Services%20Summary%20Dashboard") },
   389  							func() { assertDashboard("Coherence%20HTTP%20Servers%20Summary%20Dashboard") },
   390  							func() { assertDashboard("Coherence%20Proxy%20Server%20Detail%20Dashboard") },
   391  							func() { assertDashboard("Coherence%20Alerts%20Dashboard") },
   392  							func() { assertDashboard("Coherence%20Member%20Details%20Dashboard") },
   393  							func() { assertDashboard("Coherence%20Machines%20Summary%20Dashboard") },
   394  						)
   395  					})
   396  				t.ItMinimumVersion("Grafana should have the verrazzano user with admin privileges", "1.3.0", kubeconfigPath, func() {
   397  					vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath)
   398  					if err != nil {
   399  						t.Logs.Errorf("Error getting Verrazzano resource: %v", err)
   400  						Fail(err.Error())
   401  					}
   402  					if vz.Spec.Version != "" {
   403  						t.Logs.Info("Skipping test because Verrazzano has been upgraded %s")
   404  					} else {
   405  						Eventually(assertAdminRole, waitTimeout, pollingInterval).Should(BeTrue())
   406  					}
   407  				})
   408  
   409  				t.It("Grafana should have a default datasource present", func() {
   410  					vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfigPath)
   411  					if err != nil {
   412  						t.Logs.Errorf("Error getting Verrazzano resource: %v", err)
   413  						Fail(err.Error())
   414  					}
   415  					name := "Prometheus"
   416  					if vzcr.IsThanosEnabled(vz) {
   417  						name = "Thanos"
   418  					}
   419  					Eventually(func() (bool, error) {
   420  						return grafanaDefaultDatasourceExists(vz, name, kubeconfigPath)
   421  					}).WithTimeout(waitTimeout).WithPolling(pollingInterval).Should(BeTrue())
   422  				})
   423  			}
   424  
   425  		} else {
   426  			t.It("is not running", func() {
   427  				Eventually(func() (bool, error) {
   428  					return pkg.PodsNotRunning(verrazzanoNamespace, []string{"vmi-system-grafana"})
   429  				}, waitTimeout, pollingInterval).Should(BeTrue())
   430  				Expect(ingressURLs).NotTo(HaveKey(grafanaIngress), fmt.Sprintf("Ingress %s should not exist", grafanaIngress))
   431  				Expect(vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.GrafanaURL == nil).To(BeTrue())
   432  			})
   433  		}
   434  	})
   435  })
   436  
   437  func assertOidcIngressByName(key string, vz *vzbeta1.Verrazzano, componentName string) {
   438  	if ingressEnabled(vz) {
   439  		Expect(ingressURLs).To(HaveKey(key), fmt.Sprintf("Ingress %s not found", key))
   440  		url := ingressURLs[key]
   441  		assertOidcIngress(url)
   442  	} else {
   443  		t.Logs.Infof("Skipping checking ingress %s because ingress-nginx or cert-manager is not installed", key)
   444  	}
   445  
   446  }
   447  
   448  func assertOidcIngress(url string) {
   449  	unauthHTTPClient := pkg.EventuallyVerrazzanoRetryableHTTPClient()
   450  	pkg.Concurrently(
   451  		func() {
   452  			Eventually(func() bool {
   453  				return pkg.AssertOauthURLAccessibleAndUnauthorized(unauthHTTPClient, url)
   454  			}, waitTimeout, pollingInterval).Should(BeTrue())
   455  		},
   456  		func() {
   457  			Eventually(func() bool {
   458  				return pkg.AssertURLAccessibleAndAuthorized(httpClient, url, creds)
   459  			}, waitTimeout, pollingInterval).Should(BeTrue())
   460  		},
   461  		func() {
   462  			Eventually(func() bool {
   463  				return pkg.AssertBearerAuthorized(httpClient, url)
   464  			}, waitTimeout, pollingInterval).Should(BeTrue())
   465  		},
   466  	)
   467  }
   468  
   469  func elasticIndicesCreated() bool {
   470  	b, _ := ContainElements(".kibana_1").Match(elastic.ListIndices())
   471  	return b
   472  }
   473  
   474  func elasticConnected() bool {
   475  	return elastic.Connect()
   476  }
   477  
   478  func elasticHealth() bool {
   479  	kubeconfigPath, err := k8sutil.GetKubeConfigLocation()
   480  	if err != nil {
   481  		t.Logs.Errorf("Failed to get default kubeconfig path: %s", err.Error())
   482  		return false
   483  	}
   484  	return elastic.CheckHealth(kubeconfigPath)
   485  }
   486  
   487  func elasticIndicesHealth() bool {
   488  	kubeconfigPath, err := k8sutil.GetKubeConfigLocation()
   489  	if err != nil {
   490  		t.Logs.Errorf("Failed to get default kubeconfig path: %s", err.Error())
   491  		return false
   492  	}
   493  	return elastic.CheckIndicesHealth(kubeconfigPath)
   494  }
   495  
   496  func elasticIngress() bool {
   497  	return elastic.CheckIngress()
   498  }
   499  
   500  func assertDashboard(url string) {
   501  	searchURL := fmt.Sprintf("%sapi/search?query=%s", ingressURLs["vmi-system-grafana"], url)
   502  	fmt.Println("Grafana URL in browseGrafanaDashboard ", searchURL)
   503  
   504  	searchDashboard := func() bool {
   505  		vmiHTTPClient := pkg.EventuallyVerrazzanoRetryableHTTPClient()
   506  		vmiHTTPClient.HTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
   507  			return http.ErrUseLastResponse
   508  		}
   509  
   510  		req, err := retryablehttp.NewRequest("GET", searchURL, nil)
   511  		if err != nil {
   512  			t.Logs.Errorf("Error creating HTTP request: %v", err)
   513  			return false
   514  		}
   515  		req.SetBasicAuth(creds.Username, creds.Password)
   516  		resp, err := vmiHTTPClient.Do(req)
   517  		if err != nil {
   518  			t.Logs.Errorf("Error making HTTP request: %v", err)
   519  			return false
   520  		}
   521  		if resp.StatusCode != http.StatusOK {
   522  			t.Logs.Errorf("Unexpected HTTP status code: %d", resp.StatusCode)
   523  			return false
   524  		}
   525  		// assert that there is a single item in response
   526  		defer resp.Body.Close()
   527  		bodyBytes, err := io.ReadAll(resp.Body)
   528  		if err != nil {
   529  			t.Logs.Errorf("Unable to read body from response: %v", err)
   530  			return false
   531  		}
   532  		var response []map[string]interface{}
   533  		if err := json.Unmarshal(bodyBytes, &response); err != nil {
   534  			t.Logs.Errorf("Error unmarshaling response body: %v", err)
   535  			return false
   536  		}
   537  		if len(response) != 1 {
   538  			t.Logs.Errorf("Unexpected response length: %d", len(response))
   539  			return false
   540  		}
   541  		return true
   542  	}
   543  	Eventually(searchDashboard, waitTimeout, pollingInterval).Should(BeTrue())
   544  }
   545  
   546  func assertAdminRole() bool {
   547  	searchURL := fmt.Sprintf("%sapi/users", ingressURLs["vmi-system-grafana"])
   548  	vmiHTTPClient := pkg.EventuallyVerrazzanoRetryableHTTPClient()
   549  	vmiHTTPClient.HTTPClient.CheckRedirect = func(req *http.Request, via []*http.Request) error {
   550  		return http.ErrUseLastResponse
   551  	}
   552  
   553  	req, err := retryablehttp.NewRequest("GET", searchURL, nil)
   554  	if err != nil {
   555  		t.Logs.Errorf("Error creating HTTP request: %v", err)
   556  		return false
   557  	}
   558  	req.SetBasicAuth(creds.Username, creds.Password)
   559  	resp, err := vmiHTTPClient.Do(req)
   560  	if err != nil {
   561  		t.Logs.Errorf("Error making HTTP request: %v", err)
   562  		return false
   563  	}
   564  	if resp.StatusCode != http.StatusOK {
   565  		t.Logs.Errorf("Unexpected HTTP status code: %d", resp.StatusCode)
   566  		return false
   567  	}
   568  	// assert that there is a single item in response
   569  	defer resp.Body.Close()
   570  	bodyBytes, err := io.ReadAll(resp.Body)
   571  	if err != nil {
   572  		t.Logs.Errorf("Unable to read body from response: %v", err)
   573  		return false
   574  	}
   575  	var response []map[string]interface{}
   576  	if err := json.Unmarshal(bodyBytes, &response); err != nil {
   577  		t.Logs.Errorf("Error unmarshaling response body: %v", err)
   578  		return false
   579  	}
   580  	if len(response) != 1 {
   581  		t.Logs.Errorf("Unexpected response length: %d", len(response))
   582  		return false
   583  	}
   584  	t.Logs.Infof("Grafana users: %s", response)
   585  	return response[0]["login"] == "verrazzano" && response[0]["isAdmin"] == true
   586  }
   587  
   588  // getExpectedPrometheusReplicaCount returns the Prometheus replicas in the values overrides from the
   589  // Prometheus Operator component in the Verrazzano CR. If there is no override for replicas then the
   590  // default replica count of 1 is returned.
   591  func getExpectedPrometheusReplicaCount(kubeconfig string) (int32, error) {
   592  	vz, err := pkg.GetVerrazzanoInstallResourceInClusterV1beta1(kubeconfig)
   593  	if err != nil {
   594  		return 0, err
   595  	}
   596  	if !vzcr.IsComponentStatusEnabled(vz, operator.ComponentName) {
   597  		return 0, nil
   598  	}
   599  	var expectedReplicas int32 = 1
   600  	if vz.Spec.Components.PrometheusOperator == nil {
   601  		return expectedReplicas, nil
   602  	}
   603  
   604  	for _, override := range vz.Spec.Components.PrometheusOperator.InstallOverrides.ValueOverrides {
   605  		if override.Values != nil {
   606  			jsonString, err := gabs.ParseJSON(override.Values.Raw)
   607  			if err != nil {
   608  				return 0, err
   609  			}
   610  			if container := jsonString.Path("prometheus.prometheusSpec.replicas"); container != nil {
   611  				if val, ok := container.Data().(float64); ok {
   612  					expectedReplicas = int32(val)
   613  					t.Logs.Infof("Found Prometheus replicas override in Verrazzano CR, replica count is: %d", expectedReplicas)
   614  					break
   615  				}
   616  			}
   617  		}
   618  	}
   619  
   620  	return expectedReplicas, nil
   621  }
   622  
   623  func ingressEnabled(vz *vzbeta1.Verrazzano) bool {
   624  	return vzcr.IsComponentStatusEnabled(vz, nginx.ComponentName) &&
   625  		vzcr.IsComponentStatusEnabled(vz, cmconstants.ClusterIssuerComponentName) &&
   626  		vzcr.IsComponentStatusEnabled(vz, keycloak.ComponentName)
   627  }
   628  
   629  func verrazzanoSecretRequired(vz *vzbeta1.Verrazzano) bool {
   630  	return vzcr.IsComponentStatusEnabled(vz, opensearch.ComponentName) ||
   631  		vzcr.IsComponentStatusEnabled(vz, opensearchdashboards.ComponentName) ||
   632  		vzcr.IsComponentStatusEnabled(vz, operator.ComponentName) ||
   633  		vzcr.IsComponentStatusEnabled(vz, grafana.ComponentName) ||
   634  		vzcr.IsComponentStatusEnabled(vz, kiali.ComponentName)
   635  }
   636  
   637  func grafanaDefaultDatasourceExists(vz *vzbeta1.Verrazzano, name, kubeconfigPath string) (bool, error) {
   638  	password, err := pkg.GetVerrazzanoPasswordInCluster(kubeconfigPath)
   639  	if err != nil {
   640  		t.Logs.Error("Failed to get the Verrazzano password from the cluster")
   641  		return false, err
   642  	}
   643  	if vz == nil || vz.Status.VerrazzanoInstance == nil || vz.Status.VerrazzanoInstance.GrafanaURL == nil {
   644  		t.Logs.Error("Grafana URL in the Verrazzano status is empty")
   645  		return false, nil
   646  	}
   647  	resp, err := pkg.GetWebPageWithBasicAuth(*vz.Status.VerrazzanoInstance.GrafanaURL+"/api/datasources", "", "verrazzano", password, kubeconfigPath)
   648  	if err != nil {
   649  		t.Logs.Errorf("Failed to get Grafana datasources: %v", err)
   650  		return false, err
   651  	}
   652  
   653  	var datasources []map[string]interface{}
   654  	err = json.Unmarshal(resp.Body, &datasources)
   655  	if err != nil {
   656  		t.Logs.Errorf("Failed to unmarshal Grafana datasources: %v", err)
   657  		return false, err
   658  	}
   659  
   660  	for _, source := range datasources {
   661  		sourceName, ok := source["name"]
   662  		if !ok {
   663  			t.Logs.Errorf("Failed to find name for Grafana datasource")
   664  			continue
   665  		}
   666  
   667  		nameStr, ok := sourceName.(string)
   668  		if !ok {
   669  			t.Logs.Errorf("Failed to convert name field to string")
   670  			continue
   671  		}
   672  		if nameStr != name {
   673  			continue
   674  		}
   675  
   676  		sourceDefault, ok := source["isDefault"]
   677  		if !ok {
   678  			t.Logs.Errorf("Failed to verify the datasource was the default")
   679  			continue
   680  		}
   681  		defaultBool, ok := sourceDefault.(bool)
   682  		if !ok {
   683  			t.Logs.Errorf("Failed to convert default to bool")
   684  			continue
   685  		}
   686  		return defaultBool, nil
   687  	}
   688  
   689  	t.Logs.Errorf("Failed to find Grafana datasource %s", name)
   690  	return false, nil
   691  }