github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/workloads/weblogic-cluster/weblogic_workload_test.go (about)

     1  // Copyright (c) 2023, Oracle and/or its affiliates.
     2  // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
     3  
     4  package weblogiccluster
     5  
     6  import (
     7  	"fmt"
     8  	"net/http"
     9  	"time"
    10  
    11  	. "github.com/onsi/ginkgo/v2"
    12  	. "github.com/onsi/gomega"
    13  	"github.com/verrazzano/verrazzano/pkg/k8s/resource"
    14  	"github.com/verrazzano/verrazzano/pkg/k8sutil"
    15  	"github.com/verrazzano/verrazzano/tests/e2e/pkg"
    16  	dump "github.com/verrazzano/verrazzano/tests/e2e/pkg/test/clusterdump"
    17  	"github.com/verrazzano/verrazzano/tests/e2e/pkg/test/framework"
    18  	"github.com/verrazzano/verrazzano/tests/e2e/pkg/test/framework/metrics"
    19  	v1 "k8s.io/api/core/v1"
    20  	"k8s.io/apimachinery/pkg/api/errors"
    21  )
    22  
    23  const (
    24  	shortWaitTimeout         = 10 * time.Minute
    25  	shortPollingInterval     = 10 * time.Second
    26  	longPollingInterval      = 20 * time.Second
    27  	imagePullWaitTimeout     = 20 * time.Minute
    28  	imagePullPollingInterval = 30 * time.Second
    29  
    30  	appConfiguration        = "tests/testdata/test-applications/weblogic/hello-weblogic/hello-wls-cluster-app.yaml"
    31  	compConfiguration       = "tests/testdata/test-applications/weblogic/hello-weblogic/hello-wls-cluster-comp.yaml"
    32  	compUpdateConfiguration = "tests/testdata/test-applications/weblogic/hello-weblogic/hello-wls-cluster-comp-update.yaml"
    33  
    34  	appURL         = "hello/weblogic/greetings/message"
    35  	welcomeMessage = "Hello WebLogic"
    36  
    37  	wlsUser           = "weblogic"
    38  	wlDomain          = "hellodomain"
    39  	wlsAdminServer    = "hellodomain-adminserver"
    40  	wlsManagedServer1 = "hellodomain-managedserver1"
    41  	wlsManagedServer2 = "hellodomain-managedserver2"
    42  	trait             = "hello-domain-trait"
    43  
    44  	helloDomainRepoCreds     = "hellodomain-repo-credentials"
    45  	helloDomainWeblogicCreds = "hellodomain-weblogic-credentials"
    46  )
    47  
    48  var (
    49  	t                  = framework.NewTestFramework("weblogicworkload")
    50  	generatedNamespace = pkg.GenerateNamespace("hello-wls")
    51  	expectedPods       = []string{wlsAdminServer, wlsManagedServer1}
    52  	host               = ""
    53  	metricsTest        pkg.MetricsTest
    54  )
    55  
    56  var beforeSuite = t.BeforeSuiteFunc(func() {
    57  	if !skipDeploy {
    58  		start := time.Now()
    59  		deployWebLogicApp(namespace)
    60  		metrics.Emit(t.Metrics.With("deployment_elapsed_time", time.Since(start).Milliseconds()))
    61  
    62  		// Container image pull check
    63  		Eventually(func() bool {
    64  			return pkg.ContainerImagePullWait(namespace, expectedPods)
    65  		}, imagePullWaitTimeout, imagePullPollingInterval).Should(BeTrue())
    66  	}
    67  
    68  	// check expected admin server pod is running
    69  	Eventually(func() bool {
    70  		result, err := pkg.PodsRunning(namespace, []string{wlsAdminServer})
    71  		if err != nil {
    72  			AbortSuite(fmt.Sprintf("WebLogic admin server pod is not running in the namespace: %v, error: %v", namespace, err))
    73  		}
    74  		return result
    75  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Admin server pod is not ready")
    76  
    77  	// check expected managed server pod is running
    78  	Eventually(func() bool {
    79  		result, err := pkg.PodsRunning(namespace, []string{wlsManagedServer1})
    80  		if err != nil {
    81  			AbortSuite(fmt.Sprintf("WebLogic managed server pod is not running in the namespace: %v, error: %v", namespace, err))
    82  		}
    83  		return result
    84  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed server pod is not ready")
    85  
    86  	// check expected admin service is running
    87  	Eventually(func() bool {
    88  		result, err := pkg.DoesServiceExist(namespace, wlsAdminServer)
    89  		if err != nil {
    90  			AbortSuite(fmt.Sprintf("Admin Service %s is not running in the namespace: %v, error: %v", wlsAdminServer, namespace, err))
    91  		}
    92  		return result
    93  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Admin service is not running")
    94  
    95  	// check expected managed service is running
    96  	Eventually(func() bool {
    97  		result, err := pkg.DoesServiceExist(namespace, wlsManagedServer1)
    98  		if err != nil {
    99  			AbortSuite(fmt.Sprintf("Managed Service %s is not running in the namespace: %v, error: %v", wlsManagedServer1, namespace, err))
   100  		}
   101  		return result
   102  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed service is not running")
   103  
   104  	// check expected VirtualService is ready
   105  	Eventually(func() bool {
   106  		result, err := pkg.DoesVirtualServiceExist(namespace, trait)
   107  		if err != nil {
   108  			AbortSuite(fmt.Sprintf("WebLogic VirtualService %s is not running in the namespace: %v, error: %v", trait, namespace, err))
   109  		}
   110  		return result
   111  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: VirtualService is not ready")
   112  
   113  	// check expected Secrets exist
   114  	Eventually(func() bool {
   115  		result, err := pkg.DoesSecretExist(namespace, helloDomainWeblogicCreds)
   116  		if err != nil {
   117  			AbortSuite(fmt.Sprintf("WebLogic Secret %s does not exist in the namespace: %v, error: %v", helloDomainWeblogicCreds, namespace, err))
   118  		}
   119  		return result
   120  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Secret does not exist")
   121  
   122  	Eventually(func() bool {
   123  		result, err := pkg.DoesSecretExist(namespace, helloDomainRepoCreds)
   124  		if err != nil {
   125  			AbortSuite(fmt.Sprintf("WebLogic Secret %s does not exist in the namespace: %v, error: %v", helloDomainRepoCreds, namespace, err))
   126  		}
   127  		return result
   128  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Secret does not exist")
   129  
   130  	// Update the component to specify replicas of 2 for clusters
   131  	Eventually(func() error {
   132  		file, err := pkg.FindTestDataFile(compUpdateConfiguration)
   133  		if err != nil {
   134  			return err
   135  		}
   136  		return resource.CreateOrUpdateResourceFromFileInGeneratedNamespace(file, namespace)
   137  	}, shortWaitTimeout, shortPollingInterval, "Failed to update component resources for WebLogic application").ShouldNot(HaveOccurred())
   138  
   139  	// Container image pull check
   140  	Eventually(func() bool {
   141  		return pkg.ContainerImagePullWait(namespace, []string{wlsManagedServer2})
   142  	}, imagePullWaitTimeout, imagePullPollingInterval).Should(BeTrue())
   143  
   144  	// check expected managed server 2 pod is running after changing the cluster replicas to 2
   145  	Eventually(func() bool {
   146  		result, err := pkg.PodsRunning(namespace, []string{wlsManagedServer2})
   147  		if err != nil {
   148  			AbortSuite(fmt.Sprintf("WebLogic managed server pod is not running in the namespace: %v, error: %v", namespace, err))
   149  		}
   150  		return result
   151  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed server pod is not ready")
   152  
   153  	// check expected managed service 2 is running after changing the cluster replicas to 2
   154  	Eventually(func() bool {
   155  		result, err := pkg.DoesServiceExist(namespace, wlsManagedServer2)
   156  		if err != nil {
   157  			AbortSuite(fmt.Sprintf("Managed Service %s is not running in the namespace: %v, error: %v", wlsManagedServer2, namespace, err))
   158  		}
   159  		return result
   160  	}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed service is not running")
   161  
   162  	var err error
   163  	// Get the host from the Istio gateway resource.
   164  	start := time.Now()
   165  	// check expected Gateway is ready
   166  	Eventually(func() (string, error) {
   167  		host, err = k8sutil.GetHostnameFromGateway(namespace, "")
   168  		return host, err
   169  	}, shortWaitTimeout, shortPollingInterval).Should(Not(BeEmpty()), "Failed to deploy the WebLogic Application: Gateway is not ready")
   170  	metrics.Emit(t.Metrics.With("get_host_name_elapsed_time", time.Since(start).Milliseconds()))
   171  
   172  	kubeconfigPath, err := k8sutil.GetKubeConfigLocation()
   173  	metricsTest, err = pkg.NewMetricsTest(kubeconfigPath, map[string]string{})
   174  	if err != nil {
   175  		AbortSuite(fmt.Sprintf("Failed to create the Metrics test object: %v", err))
   176  	}
   177  
   178  	beforeSuitePassed = true
   179  })
   180  
   181  var _ = BeforeSuite(beforeSuite)
   182  
   183  var failed = false
   184  var beforeSuitePassed = false
   185  var _ = t.AfterEach(func() {
   186  	failed = failed || CurrentSpecReport().Failed()
   187  })
   188  
   189  var afterSuite = t.AfterSuiteFunc(func() {
   190  	if failed || !beforeSuitePassed {
   191  		dump.CaptureContainerLogs(namespace, wlsAdminServer, "weblogic-server", "/scratch/logs/hello-domain")
   192  		dump.ExecuteBugReport(namespace)
   193  	}
   194  	if !skipUndeploy {
   195  		undeployWebLogicApp()
   196  	}
   197  })
   198  
   199  var _ = AfterSuite(afterSuite)
   200  
   201  func deployWebLogicApp(namespace string) {
   202  	t.Logs.Info("Deploy WebLogic application")
   203  	wlsPass := pkg.GetRequiredEnvVarOrFail("WEBLOGIC_PSW")
   204  	regServ := pkg.GetRequiredEnvVarOrFail("OCR_REPO")
   205  	regUser := pkg.GetRequiredEnvVarOrFail("OCR_CREDS_USR")
   206  	regPass := pkg.GetRequiredEnvVarOrFail("OCR_CREDS_PSW")
   207  
   208  	t.Logs.Info("Create namespace")
   209  	Eventually(func() (*v1.Namespace, error) {
   210  		nsLabels := map[string]string{
   211  			"verrazzano-managed": "true",
   212  			"istio-injection":    istioInjection}
   213  		return pkg.CreateNamespace(namespace, nsLabels)
   214  	}, shortWaitTimeout, shortPollingInterval).ShouldNot(BeNil())
   215  
   216  	// Create docker-registry secret to enable pulling image from the registry
   217  	Eventually(func() (*v1.Secret, error) {
   218  		return pkg.CreateDockerSecret(namespace, helloDomainRepoCreds, regServ, regUser, regPass)
   219  	}, shortWaitTimeout, shortPollingInterval).ShouldNot(BeNil())
   220  
   221  	// Create secret for the WebLogic domain
   222  	Eventually(func() (*v1.Secret, error) {
   223  		return pkg.CreateCredentialsSecret(namespace, helloDomainWeblogicCreds, wlsUser, wlsPass, nil)
   224  	}, shortWaitTimeout, shortPollingInterval).ShouldNot(BeNil())
   225  
   226  	// Note: creating the app config first to verify that default metrics traits are created properly if the app config exists before the components
   227  	t.Logs.Info("Create application resources")
   228  	Eventually(func() error {
   229  		file, err := pkg.FindTestDataFile(appConfiguration)
   230  		if err != nil {
   231  			return err
   232  		}
   233  		return resource.CreateOrUpdateResourceFromFileInGeneratedNamespace(file, namespace)
   234  	}, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred())
   235  
   236  	t.Logs.Info("Create component resources")
   237  	Eventually(func() error {
   238  		file, err := pkg.FindTestDataFile(compConfiguration)
   239  		if err != nil {
   240  			return err
   241  		}
   242  		return resource.CreateOrUpdateResourceFromFileInGeneratedNamespace(file, namespace)
   243  	}, shortWaitTimeout, shortPollingInterval, "Failed to create component resources for WebLogic application").ShouldNot(HaveOccurred())
   244  }
   245  
   246  func undeployWebLogicApp() {
   247  	t.Logs.Info("Undeploy WebLogic application")
   248  	t.Logs.Info("Delete application")
   249  	start := time.Now()
   250  	Eventually(func() error {
   251  		file, err := pkg.FindTestDataFile(appConfiguration)
   252  		if err != nil {
   253  			return err
   254  		}
   255  		return resource.DeleteResourceFromFileInGeneratedNamespace(file, namespace)
   256  	}, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred())
   257  
   258  	t.Logs.Info("Delete component")
   259  	Eventually(func() error {
   260  		file, err := pkg.FindTestDataFile(compConfiguration)
   261  		if err != nil {
   262  			return err
   263  		}
   264  		return resource.DeleteResourceFromFileInGeneratedNamespace(file, namespace)
   265  	}, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred())
   266  
   267  	// Wait for pod to terminate
   268  	Eventually(func() bool {
   269  		podsTerminated, _ := pkg.PodsNotRunning(namespace, append(expectedPods, wlsManagedServer2))
   270  		return podsTerminated
   271  	}, shortWaitTimeout, shortPollingInterval).Should(BeTrue())
   272  
   273  	t.Logs.Info("Delete namespace")
   274  	Eventually(func() error {
   275  		return pkg.DeleteNamespace(namespace)
   276  	}, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred())
   277  
   278  	// Wait for namespace finalizer to be removed
   279  	Eventually(func() bool {
   280  		return pkg.CheckNamespaceFinalizerRemoved(namespace)
   281  	}, shortWaitTimeout, shortPollingInterval).Should(BeTrue())
   282  
   283  	// Wait for namespace deletion
   284  	Eventually(func() bool {
   285  		_, err := pkg.GetNamespace(namespace)
   286  		return err != nil && errors.IsNotFound(err)
   287  	}, shortWaitTimeout, shortPollingInterval).Should(BeTrue())
   288  
   289  	metrics.Emit(t.Metrics.With("undeployment_elapsed_time", time.Since(start).Milliseconds()))
   290  }
   291  
   292  var _ = t.Describe("Validate deployment of VerrazzanoWebLogicWorkload", Label("f:app-lcm.oam", "f:app-lcm.weblogic-workload"), func() {
   293  
   294  	t.Context("Ingress", Label("f:mesh.ingress"), FlakeAttempts(8), func() {
   295  		// Verify the application endpoint is working.
   296  		// GIVEN the sample WebLogic app is deployed
   297  		// WHEN the application endpoint is accessed
   298  		// THEN the expected returned page should contain an expected value.
   299  		t.It("Verify application endpoint is working", func() {
   300  			Eventually(func() (*pkg.HTTPResponse, error) {
   301  				url := fmt.Sprintf("https://%s/%s", host, appURL)
   302  				return pkg.GetWebPage(url, host)
   303  			}, shortWaitTimeout, shortPollingInterval).Should(And(pkg.HasStatus(http.StatusOK), pkg.BodyEquals(welcomeMessage)))
   304  		})
   305  	})
   306  
   307  	t.Context("Metrics", Label("f:observability.monitoring.prom"), FlakeAttempts(5), func() {
   308  		// Verify application Prometheus scraped metrics
   309  		// GIVEN the sample WebLogic app is deployed
   310  		// WHEN the application configuration uses a default metrics trait
   311  		// THEN confirm that metrics are being collected
   312  		t.It("Retrieve application Prometheus scraped metrics", func() {
   313  			pkg.Concurrently(
   314  				func() {
   315  					Eventually(func() bool {
   316  						return metricsTest.MetricsExist("wls_jvm_process_cpu_load", map[string]string{"weblogic_domainName": wlDomain})
   317  					}, shortWaitTimeout, longPollingInterval).Should(BeTrue())
   318  				},
   319  				func() {
   320  					Eventually(func() bool {
   321  						return metricsTest.MetricsExist("wls_scrape_mbeans_count_total", map[string]string{"weblogic_domainName": wlDomain})
   322  					}, shortWaitTimeout, longPollingInterval).Should(BeTrue())
   323  				},
   324  				func() {
   325  					Eventually(func() bool {
   326  						return metricsTest.MetricsExist("wls_server_state_val", map[string]string{"weblogic_domainName": wlDomain})
   327  					}, shortWaitTimeout, longPollingInterval).Should(BeTrue())
   328  				},
   329  			)
   330  		})
   331  
   332  		// Verify Istio Prometheus scraped metrics
   333  		// GIVEN the sample WebLogic app is deployed
   334  		// WHEN the application configuration is deployed
   335  		// THEN confirm that Istio metrics are being collected
   336  		if istioInjection == "enabled" {
   337  			t.It("Retrieve Istio Prometheus scraped metrics", func() {
   338  				pkg.Concurrently(
   339  					func() {
   340  						Eventually(func() bool {
   341  							return metricsTest.MetricsExist("istio_tcp_received_bytes_total", map[string]string{"destination_canonical_service": "hello-domain"})
   342  						}, shortWaitTimeout, shortPollingInterval).Should(BeTrue())
   343  					},
   344  					func() {
   345  						Eventually(func() bool {
   346  							return metricsTest.MetricsExist("envoy_cluster_http2_pending_send_bytes", map[string]string{"pod_name": wlsAdminServer})
   347  						}, shortWaitTimeout, longPollingInterval).Should(BeTrue())
   348  					},
   349  					func() {
   350  						Eventually(func() bool {
   351  							return metricsTest.MetricsExist("envoy_cluster_http2_pending_send_bytes", map[string]string{"pod_name": wlsManagedServer1})
   352  						}, shortWaitTimeout, longPollingInterval).Should(BeTrue())
   353  					},
   354  				)
   355  			})
   356  		}
   357  	})
   358  
   359  	t.Context("WebLogic logging", Label("f:observability.logging.es"), func() {
   360  		var indexName string
   361  		var err error
   362  		Eventually(func() error {
   363  			indexName, err = pkg.GetOpenSearchAppIndex(namespace)
   364  			return err
   365  		}, shortWaitTimeout, shortPollingInterval).Should(BeNil(), "Expected to get OpenSearch App Index")
   366  
   367  		// GIVEN a WebLogic application with logging enabled
   368  		// WHEN the Opensearch index is retrieved
   369  		// THEN verify that it is found
   370  		t.It("Verify Opensearch index exists", func() {
   371  			Eventually(func() bool {
   372  				return pkg.LogIndexFound(indexName)
   373  			}, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find log index "+indexName)
   374  		})
   375  		pkg.Concurrently(
   376  			// GIVEN a WebLogic application with logging enabled
   377  			// WHEN the log records are retrieved from the Opensearch index
   378  			// THEN verify that a recent log record of hellodomain-adminserver stdout is found
   379  			func() {
   380  				t.It("Verify recent hellodomain-adminserver log record exists", func() {
   381  					Eventually(func() bool {
   382  						return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{
   383  							"kubernetes.labels.weblogic_domainUID":  wlDomain,
   384  							"kubernetes.labels.weblogic_serverName": "AdminServer",
   385  							"kubernetes.pod_name":                   wlsAdminServer,
   386  							"kubernetes.container_name":             "weblogic-server",
   387  						})
   388  					}, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   389  				})
   390  			},
   391  
   392  			// GIVEN a WebLogic application with logging enabled
   393  			// WHEN the log records are retrieved from the Opensearch index
   394  			// THEN verify that a recent log record of hellodomain-managedserver1 stdout is found
   395  			func() {
   396  				t.It("Verify recent hellodomain-managedserver1 log record exists", func() {
   397  					Eventually(func() bool {
   398  						return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{
   399  							"kubernetes.labels.weblogic_domainUID":  wlDomain,
   400  							"kubernetes.labels.weblogic_serverName": "ManagedServer1",
   401  							"kubernetes.pod_name":                   wlsManagedServer1,
   402  							"kubernetes.container_name":             "weblogic-server",
   403  						})
   404  					}, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   405  				})
   406  			},
   407  
   408  			// GIVEN a WebLogic application with logging enabled
   409  			// WHEN the log records are retrieved from the Opensearch index
   410  			// THEN verify that a recent log record of hellodomain-adminserver log file is found
   411  			func() {
   412  				t.It("Verify recent hellodomain-adminserver log record exists", func() {
   413  					Eventually(func() bool {
   414  						return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{
   415  							"kubernetes.labels.weblogic_domainUID":  wlDomain,
   416  							"kubernetes.labels.weblogic_serverName": "AdminServer",
   417  							"kubernetes.pod_name":                   wlsAdminServer,
   418  							"kubernetes.container_name":             "fluentd-stdout-sidecar",
   419  						})
   420  					}, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   421  				})
   422  			},
   423  
   424  			// GIVEN a WebLogic application with logging enabled
   425  			// WHEN the log records are retrieved from the Opensearch index
   426  			// THEN verify that a recent log record of hellodomain-managedserver1 log file is found
   427  			func() {
   428  				t.It("Verify recent hellodomain-managedserver1 log record exists", func() {
   429  					Eventually(func() bool {
   430  						return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{
   431  							"kubernetes.labels.weblogic_domainUID":  wlDomain,
   432  							"kubernetes.labels.weblogic_serverName": "ManagedServer1",
   433  							"kubernetes.pod_name":                   wlsManagedServer1,
   434  							"kubernetes.container_name":             "fluentd-stdout-sidecar",
   435  						})
   436  					}, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   437  				})
   438  			},
   439  
   440  			// GIVEN a WebLogic application with logging enabled
   441  			// WHEN the log records are retrieved from the Opensearch index
   442  			// THEN verify that a recent pattern-matched log record of hellodomain-adminserver stdout is found
   443  			func() {
   444  				t.It("Verify recent pattern-matched AdminServer log record exists", func() {
   445  					Eventually(func() bool {
   446  						return pkg.FindLog(indexName,
   447  							[]pkg.Match{
   448  								{Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"},
   449  								{Key: "subSystem.keyword", Value: "WorkManager"},
   450  								{Key: "serverName.keyword", Value: wlsAdminServer},
   451  								{Key: "serverName2.keyword", Value: "AdminServer"},
   452  								{Key: "message", Value: "standby threads"}},
   453  							[]pkg.Match{})
   454  					}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   455  				})
   456  			},
   457  
   458  			// GIVEN a WebLogic application with logging enabled
   459  			// WHEN the log records are retrieved from the Opensearch index
   460  			// THEN verify that a recent pattern-matched log record of hellodomain-managedserver1 stdout is found
   461  			func() {
   462  				t.It("Verify recent pattern-matched ManagedServer log record exists", func() {
   463  					Eventually(func() bool {
   464  						return pkg.FindLog(indexName,
   465  							[]pkg.Match{
   466  								{Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"},
   467  								{Key: "subSystem.keyword", Value: "WorkManager"},
   468  								{Key: "serverName.keyword", Value: wlsManagedServer1},
   469  								{Key: "serverName2.keyword", Value: "ManagedServer1"},
   470  								{Key: "message", Value: "standby threads"}},
   471  							[]pkg.Match{})
   472  					}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   473  				})
   474  			},
   475  
   476  			// GIVEN a WebLogic application with logging enabled
   477  			// WHEN the log records are retrieved from the Opensearch index
   478  			// THEN verify that a recent pattern-matched log record of hellodomain-adminserver stdout is found
   479  			func() {
   480  				t.It("Verify recent pattern-matched AdminServer log record exists", func() {
   481  					Eventually(func() bool {
   482  						return pkg.FindLog(indexName,
   483  							[]pkg.Match{
   484  								{Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"},
   485  								{Key: "subSystem", Value: "WorkManager"},
   486  								{Key: "serverName", Value: wlsAdminServer},
   487  								{Key: "serverName2", Value: "AdminServer"},
   488  								{Key: "message", Value: "Self-tuning"}},
   489  							[]pkg.Match{})
   490  					}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   491  				})
   492  			},
   493  			// GIVEN a WebLogic application with logging enabled
   494  			// WHEN the log records are retrieved from the Opensearch index
   495  			// THEN verify that a recent pattern-matched log record of hellodomain-managedserver1 stdout is found
   496  			func() {
   497  				t.It("Verify recent pattern-matched ManagedServer log record exists", func() {
   498  					Eventually(func() bool {
   499  						return pkg.FindLog(indexName,
   500  							[]pkg.Match{
   501  								{Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"},
   502  								{Key: "subSystem", Value: "WorkManager"},
   503  								{Key: "serverName", Value: wlsManagedServer1},
   504  								{Key: "serverName2", Value: "ManagedServer1"},
   505  								{Key: "message", Value: "Self-tuning"}},
   506  							[]pkg.Match{})
   507  					}, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record")
   508  				})
   509  			},
   510  		)
   511  	})
   512  })