github.com/verrazzano/verrazzano@v1.7.1/tests/e2e/workloads/weblogic-cluster/weblogic_workload_test.go (about) 1 // Copyright (c) 2023, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 package weblogiccluster 5 6 import ( 7 "fmt" 8 "net/http" 9 "time" 10 11 . "github.com/onsi/ginkgo/v2" 12 . "github.com/onsi/gomega" 13 "github.com/verrazzano/verrazzano/pkg/k8s/resource" 14 "github.com/verrazzano/verrazzano/pkg/k8sutil" 15 "github.com/verrazzano/verrazzano/tests/e2e/pkg" 16 dump "github.com/verrazzano/verrazzano/tests/e2e/pkg/test/clusterdump" 17 "github.com/verrazzano/verrazzano/tests/e2e/pkg/test/framework" 18 "github.com/verrazzano/verrazzano/tests/e2e/pkg/test/framework/metrics" 19 v1 "k8s.io/api/core/v1" 20 "k8s.io/apimachinery/pkg/api/errors" 21 ) 22 23 const ( 24 shortWaitTimeout = 10 * time.Minute 25 shortPollingInterval = 10 * time.Second 26 longPollingInterval = 20 * time.Second 27 imagePullWaitTimeout = 20 * time.Minute 28 imagePullPollingInterval = 30 * time.Second 29 30 appConfiguration = "tests/testdata/test-applications/weblogic/hello-weblogic/hello-wls-cluster-app.yaml" 31 compConfiguration = "tests/testdata/test-applications/weblogic/hello-weblogic/hello-wls-cluster-comp.yaml" 32 compUpdateConfiguration = "tests/testdata/test-applications/weblogic/hello-weblogic/hello-wls-cluster-comp-update.yaml" 33 34 appURL = "hello/weblogic/greetings/message" 35 welcomeMessage = "Hello WebLogic" 36 37 wlsUser = "weblogic" 38 wlDomain = "hellodomain" 39 wlsAdminServer = "hellodomain-adminserver" 40 wlsManagedServer1 = "hellodomain-managedserver1" 41 wlsManagedServer2 = "hellodomain-managedserver2" 42 trait = "hello-domain-trait" 43 44 helloDomainRepoCreds = "hellodomain-repo-credentials" 45 helloDomainWeblogicCreds = "hellodomain-weblogic-credentials" 46 ) 47 48 var ( 49 t = framework.NewTestFramework("weblogicworkload") 50 generatedNamespace = pkg.GenerateNamespace("hello-wls") 51 expectedPods = []string{wlsAdminServer, wlsManagedServer1} 52 host = "" 53 metricsTest pkg.MetricsTest 54 ) 55 56 var beforeSuite = t.BeforeSuiteFunc(func() { 57 if !skipDeploy { 58 start := time.Now() 59 deployWebLogicApp(namespace) 60 metrics.Emit(t.Metrics.With("deployment_elapsed_time", time.Since(start).Milliseconds())) 61 62 // Container image pull check 63 Eventually(func() bool { 64 return pkg.ContainerImagePullWait(namespace, expectedPods) 65 }, imagePullWaitTimeout, imagePullPollingInterval).Should(BeTrue()) 66 } 67 68 // check expected admin server pod is running 69 Eventually(func() bool { 70 result, err := pkg.PodsRunning(namespace, []string{wlsAdminServer}) 71 if err != nil { 72 AbortSuite(fmt.Sprintf("WebLogic admin server pod is not running in the namespace: %v, error: %v", namespace, err)) 73 } 74 return result 75 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Admin server pod is not ready") 76 77 // check expected managed server pod is running 78 Eventually(func() bool { 79 result, err := pkg.PodsRunning(namespace, []string{wlsManagedServer1}) 80 if err != nil { 81 AbortSuite(fmt.Sprintf("WebLogic managed server pod is not running in the namespace: %v, error: %v", namespace, err)) 82 } 83 return result 84 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed server pod is not ready") 85 86 // check expected admin service is running 87 Eventually(func() bool { 88 result, err := pkg.DoesServiceExist(namespace, wlsAdminServer) 89 if err != nil { 90 AbortSuite(fmt.Sprintf("Admin Service %s is not running in the namespace: %v, error: %v", wlsAdminServer, namespace, err)) 91 } 92 return result 93 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Admin service is not running") 94 95 // check expected managed service is running 96 Eventually(func() bool { 97 result, err := pkg.DoesServiceExist(namespace, wlsManagedServer1) 98 if err != nil { 99 AbortSuite(fmt.Sprintf("Managed Service %s is not running in the namespace: %v, error: %v", wlsManagedServer1, namespace, err)) 100 } 101 return result 102 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed service is not running") 103 104 // check expected VirtualService is ready 105 Eventually(func() bool { 106 result, err := pkg.DoesVirtualServiceExist(namespace, trait) 107 if err != nil { 108 AbortSuite(fmt.Sprintf("WebLogic VirtualService %s is not running in the namespace: %v, error: %v", trait, namespace, err)) 109 } 110 return result 111 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: VirtualService is not ready") 112 113 // check expected Secrets exist 114 Eventually(func() bool { 115 result, err := pkg.DoesSecretExist(namespace, helloDomainWeblogicCreds) 116 if err != nil { 117 AbortSuite(fmt.Sprintf("WebLogic Secret %s does not exist in the namespace: %v, error: %v", helloDomainWeblogicCreds, namespace, err)) 118 } 119 return result 120 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Secret does not exist") 121 122 Eventually(func() bool { 123 result, err := pkg.DoesSecretExist(namespace, helloDomainRepoCreds) 124 if err != nil { 125 AbortSuite(fmt.Sprintf("WebLogic Secret %s does not exist in the namespace: %v, error: %v", helloDomainRepoCreds, namespace, err)) 126 } 127 return result 128 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Secret does not exist") 129 130 // Update the component to specify replicas of 2 for clusters 131 Eventually(func() error { 132 file, err := pkg.FindTestDataFile(compUpdateConfiguration) 133 if err != nil { 134 return err 135 } 136 return resource.CreateOrUpdateResourceFromFileInGeneratedNamespace(file, namespace) 137 }, shortWaitTimeout, shortPollingInterval, "Failed to update component resources for WebLogic application").ShouldNot(HaveOccurred()) 138 139 // Container image pull check 140 Eventually(func() bool { 141 return pkg.ContainerImagePullWait(namespace, []string{wlsManagedServer2}) 142 }, imagePullWaitTimeout, imagePullPollingInterval).Should(BeTrue()) 143 144 // check expected managed server 2 pod is running after changing the cluster replicas to 2 145 Eventually(func() bool { 146 result, err := pkg.PodsRunning(namespace, []string{wlsManagedServer2}) 147 if err != nil { 148 AbortSuite(fmt.Sprintf("WebLogic managed server pod is not running in the namespace: %v, error: %v", namespace, err)) 149 } 150 return result 151 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed server pod is not ready") 152 153 // check expected managed service 2 is running after changing the cluster replicas to 2 154 Eventually(func() bool { 155 result, err := pkg.DoesServiceExist(namespace, wlsManagedServer2) 156 if err != nil { 157 AbortSuite(fmt.Sprintf("Managed Service %s is not running in the namespace: %v, error: %v", wlsManagedServer2, namespace, err)) 158 } 159 return result 160 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Failed to deploy the WebLogic Application: Managed service is not running") 161 162 var err error 163 // Get the host from the Istio gateway resource. 164 start := time.Now() 165 // check expected Gateway is ready 166 Eventually(func() (string, error) { 167 host, err = k8sutil.GetHostnameFromGateway(namespace, "") 168 return host, err 169 }, shortWaitTimeout, shortPollingInterval).Should(Not(BeEmpty()), "Failed to deploy the WebLogic Application: Gateway is not ready") 170 metrics.Emit(t.Metrics.With("get_host_name_elapsed_time", time.Since(start).Milliseconds())) 171 172 kubeconfigPath, err := k8sutil.GetKubeConfigLocation() 173 metricsTest, err = pkg.NewMetricsTest(kubeconfigPath, map[string]string{}) 174 if err != nil { 175 AbortSuite(fmt.Sprintf("Failed to create the Metrics test object: %v", err)) 176 } 177 178 beforeSuitePassed = true 179 }) 180 181 var _ = BeforeSuite(beforeSuite) 182 183 var failed = false 184 var beforeSuitePassed = false 185 var _ = t.AfterEach(func() { 186 failed = failed || CurrentSpecReport().Failed() 187 }) 188 189 var afterSuite = t.AfterSuiteFunc(func() { 190 if failed || !beforeSuitePassed { 191 dump.CaptureContainerLogs(namespace, wlsAdminServer, "weblogic-server", "/scratch/logs/hello-domain") 192 dump.ExecuteBugReport(namespace) 193 } 194 if !skipUndeploy { 195 undeployWebLogicApp() 196 } 197 }) 198 199 var _ = AfterSuite(afterSuite) 200 201 func deployWebLogicApp(namespace string) { 202 t.Logs.Info("Deploy WebLogic application") 203 wlsPass := pkg.GetRequiredEnvVarOrFail("WEBLOGIC_PSW") 204 regServ := pkg.GetRequiredEnvVarOrFail("OCR_REPO") 205 regUser := pkg.GetRequiredEnvVarOrFail("OCR_CREDS_USR") 206 regPass := pkg.GetRequiredEnvVarOrFail("OCR_CREDS_PSW") 207 208 t.Logs.Info("Create namespace") 209 Eventually(func() (*v1.Namespace, error) { 210 nsLabels := map[string]string{ 211 "verrazzano-managed": "true", 212 "istio-injection": istioInjection} 213 return pkg.CreateNamespace(namespace, nsLabels) 214 }, shortWaitTimeout, shortPollingInterval).ShouldNot(BeNil()) 215 216 // Create docker-registry secret to enable pulling image from the registry 217 Eventually(func() (*v1.Secret, error) { 218 return pkg.CreateDockerSecret(namespace, helloDomainRepoCreds, regServ, regUser, regPass) 219 }, shortWaitTimeout, shortPollingInterval).ShouldNot(BeNil()) 220 221 // Create secret for the WebLogic domain 222 Eventually(func() (*v1.Secret, error) { 223 return pkg.CreateCredentialsSecret(namespace, helloDomainWeblogicCreds, wlsUser, wlsPass, nil) 224 }, shortWaitTimeout, shortPollingInterval).ShouldNot(BeNil()) 225 226 // Note: creating the app config first to verify that default metrics traits are created properly if the app config exists before the components 227 t.Logs.Info("Create application resources") 228 Eventually(func() error { 229 file, err := pkg.FindTestDataFile(appConfiguration) 230 if err != nil { 231 return err 232 } 233 return resource.CreateOrUpdateResourceFromFileInGeneratedNamespace(file, namespace) 234 }, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred()) 235 236 t.Logs.Info("Create component resources") 237 Eventually(func() error { 238 file, err := pkg.FindTestDataFile(compConfiguration) 239 if err != nil { 240 return err 241 } 242 return resource.CreateOrUpdateResourceFromFileInGeneratedNamespace(file, namespace) 243 }, shortWaitTimeout, shortPollingInterval, "Failed to create component resources for WebLogic application").ShouldNot(HaveOccurred()) 244 } 245 246 func undeployWebLogicApp() { 247 t.Logs.Info("Undeploy WebLogic application") 248 t.Logs.Info("Delete application") 249 start := time.Now() 250 Eventually(func() error { 251 file, err := pkg.FindTestDataFile(appConfiguration) 252 if err != nil { 253 return err 254 } 255 return resource.DeleteResourceFromFileInGeneratedNamespace(file, namespace) 256 }, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred()) 257 258 t.Logs.Info("Delete component") 259 Eventually(func() error { 260 file, err := pkg.FindTestDataFile(compConfiguration) 261 if err != nil { 262 return err 263 } 264 return resource.DeleteResourceFromFileInGeneratedNamespace(file, namespace) 265 }, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred()) 266 267 // Wait for pod to terminate 268 Eventually(func() bool { 269 podsTerminated, _ := pkg.PodsNotRunning(namespace, append(expectedPods, wlsManagedServer2)) 270 return podsTerminated 271 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue()) 272 273 t.Logs.Info("Delete namespace") 274 Eventually(func() error { 275 return pkg.DeleteNamespace(namespace) 276 }, shortWaitTimeout, shortPollingInterval).ShouldNot(HaveOccurred()) 277 278 // Wait for namespace finalizer to be removed 279 Eventually(func() bool { 280 return pkg.CheckNamespaceFinalizerRemoved(namespace) 281 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue()) 282 283 // Wait for namespace deletion 284 Eventually(func() bool { 285 _, err := pkg.GetNamespace(namespace) 286 return err != nil && errors.IsNotFound(err) 287 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue()) 288 289 metrics.Emit(t.Metrics.With("undeployment_elapsed_time", time.Since(start).Milliseconds())) 290 } 291 292 var _ = t.Describe("Validate deployment of VerrazzanoWebLogicWorkload", Label("f:app-lcm.oam", "f:app-lcm.weblogic-workload"), func() { 293 294 t.Context("Ingress", Label("f:mesh.ingress"), FlakeAttempts(8), func() { 295 // Verify the application endpoint is working. 296 // GIVEN the sample WebLogic app is deployed 297 // WHEN the application endpoint is accessed 298 // THEN the expected returned page should contain an expected value. 299 t.It("Verify application endpoint is working", func() { 300 Eventually(func() (*pkg.HTTPResponse, error) { 301 url := fmt.Sprintf("https://%s/%s", host, appURL) 302 return pkg.GetWebPage(url, host) 303 }, shortWaitTimeout, shortPollingInterval).Should(And(pkg.HasStatus(http.StatusOK), pkg.BodyEquals(welcomeMessage))) 304 }) 305 }) 306 307 t.Context("Metrics", Label("f:observability.monitoring.prom"), FlakeAttempts(5), func() { 308 // Verify application Prometheus scraped metrics 309 // GIVEN the sample WebLogic app is deployed 310 // WHEN the application configuration uses a default metrics trait 311 // THEN confirm that metrics are being collected 312 t.It("Retrieve application Prometheus scraped metrics", func() { 313 pkg.Concurrently( 314 func() { 315 Eventually(func() bool { 316 return metricsTest.MetricsExist("wls_jvm_process_cpu_load", map[string]string{"weblogic_domainName": wlDomain}) 317 }, shortWaitTimeout, longPollingInterval).Should(BeTrue()) 318 }, 319 func() { 320 Eventually(func() bool { 321 return metricsTest.MetricsExist("wls_scrape_mbeans_count_total", map[string]string{"weblogic_domainName": wlDomain}) 322 }, shortWaitTimeout, longPollingInterval).Should(BeTrue()) 323 }, 324 func() { 325 Eventually(func() bool { 326 return metricsTest.MetricsExist("wls_server_state_val", map[string]string{"weblogic_domainName": wlDomain}) 327 }, shortWaitTimeout, longPollingInterval).Should(BeTrue()) 328 }, 329 ) 330 }) 331 332 // Verify Istio Prometheus scraped metrics 333 // GIVEN the sample WebLogic app is deployed 334 // WHEN the application configuration is deployed 335 // THEN confirm that Istio metrics are being collected 336 if istioInjection == "enabled" { 337 t.It("Retrieve Istio Prometheus scraped metrics", func() { 338 pkg.Concurrently( 339 func() { 340 Eventually(func() bool { 341 return metricsTest.MetricsExist("istio_tcp_received_bytes_total", map[string]string{"destination_canonical_service": "hello-domain"}) 342 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue()) 343 }, 344 func() { 345 Eventually(func() bool { 346 return metricsTest.MetricsExist("envoy_cluster_http2_pending_send_bytes", map[string]string{"pod_name": wlsAdminServer}) 347 }, shortWaitTimeout, longPollingInterval).Should(BeTrue()) 348 }, 349 func() { 350 Eventually(func() bool { 351 return metricsTest.MetricsExist("envoy_cluster_http2_pending_send_bytes", map[string]string{"pod_name": wlsManagedServer1}) 352 }, shortWaitTimeout, longPollingInterval).Should(BeTrue()) 353 }, 354 ) 355 }) 356 } 357 }) 358 359 t.Context("WebLogic logging", Label("f:observability.logging.es"), func() { 360 var indexName string 361 var err error 362 Eventually(func() error { 363 indexName, err = pkg.GetOpenSearchAppIndex(namespace) 364 return err 365 }, shortWaitTimeout, shortPollingInterval).Should(BeNil(), "Expected to get OpenSearch App Index") 366 367 // GIVEN a WebLogic application with logging enabled 368 // WHEN the Opensearch index is retrieved 369 // THEN verify that it is found 370 t.It("Verify Opensearch index exists", func() { 371 Eventually(func() bool { 372 return pkg.LogIndexFound(indexName) 373 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find log index "+indexName) 374 }) 375 pkg.Concurrently( 376 // GIVEN a WebLogic application with logging enabled 377 // WHEN the log records are retrieved from the Opensearch index 378 // THEN verify that a recent log record of hellodomain-adminserver stdout is found 379 func() { 380 t.It("Verify recent hellodomain-adminserver log record exists", func() { 381 Eventually(func() bool { 382 return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{ 383 "kubernetes.labels.weblogic_domainUID": wlDomain, 384 "kubernetes.labels.weblogic_serverName": "AdminServer", 385 "kubernetes.pod_name": wlsAdminServer, 386 "kubernetes.container_name": "weblogic-server", 387 }) 388 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 389 }) 390 }, 391 392 // GIVEN a WebLogic application with logging enabled 393 // WHEN the log records are retrieved from the Opensearch index 394 // THEN verify that a recent log record of hellodomain-managedserver1 stdout is found 395 func() { 396 t.It("Verify recent hellodomain-managedserver1 log record exists", func() { 397 Eventually(func() bool { 398 return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{ 399 "kubernetes.labels.weblogic_domainUID": wlDomain, 400 "kubernetes.labels.weblogic_serverName": "ManagedServer1", 401 "kubernetes.pod_name": wlsManagedServer1, 402 "kubernetes.container_name": "weblogic-server", 403 }) 404 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 405 }) 406 }, 407 408 // GIVEN a WebLogic application with logging enabled 409 // WHEN the log records are retrieved from the Opensearch index 410 // THEN verify that a recent log record of hellodomain-adminserver log file is found 411 func() { 412 t.It("Verify recent hellodomain-adminserver log record exists", func() { 413 Eventually(func() bool { 414 return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{ 415 "kubernetes.labels.weblogic_domainUID": wlDomain, 416 "kubernetes.labels.weblogic_serverName": "AdminServer", 417 "kubernetes.pod_name": wlsAdminServer, 418 "kubernetes.container_name": "fluentd-stdout-sidecar", 419 }) 420 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 421 }) 422 }, 423 424 // GIVEN a WebLogic application with logging enabled 425 // WHEN the log records are retrieved from the Opensearch index 426 // THEN verify that a recent log record of hellodomain-managedserver1 log file is found 427 func() { 428 t.It("Verify recent hellodomain-managedserver1 log record exists", func() { 429 Eventually(func() bool { 430 return pkg.LogRecordFound(indexName, time.Now().Add(-24*time.Hour), map[string]string{ 431 "kubernetes.labels.weblogic_domainUID": wlDomain, 432 "kubernetes.labels.weblogic_serverName": "ManagedServer1", 433 "kubernetes.pod_name": wlsManagedServer1, 434 "kubernetes.container_name": "fluentd-stdout-sidecar", 435 }) 436 }, shortWaitTimeout, shortPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 437 }) 438 }, 439 440 // GIVEN a WebLogic application with logging enabled 441 // WHEN the log records are retrieved from the Opensearch index 442 // THEN verify that a recent pattern-matched log record of hellodomain-adminserver stdout is found 443 func() { 444 t.It("Verify recent pattern-matched AdminServer log record exists", func() { 445 Eventually(func() bool { 446 return pkg.FindLog(indexName, 447 []pkg.Match{ 448 {Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"}, 449 {Key: "subSystem.keyword", Value: "WorkManager"}, 450 {Key: "serverName.keyword", Value: wlsAdminServer}, 451 {Key: "serverName2.keyword", Value: "AdminServer"}, 452 {Key: "message", Value: "standby threads"}}, 453 []pkg.Match{}) 454 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 455 }) 456 }, 457 458 // GIVEN a WebLogic application with logging enabled 459 // WHEN the log records are retrieved from the Opensearch index 460 // THEN verify that a recent pattern-matched log record of hellodomain-managedserver1 stdout is found 461 func() { 462 t.It("Verify recent pattern-matched ManagedServer log record exists", func() { 463 Eventually(func() bool { 464 return pkg.FindLog(indexName, 465 []pkg.Match{ 466 {Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"}, 467 {Key: "subSystem.keyword", Value: "WorkManager"}, 468 {Key: "serverName.keyword", Value: wlsManagedServer1}, 469 {Key: "serverName2.keyword", Value: "ManagedServer1"}, 470 {Key: "message", Value: "standby threads"}}, 471 []pkg.Match{}) 472 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 473 }) 474 }, 475 476 // GIVEN a WebLogic application with logging enabled 477 // WHEN the log records are retrieved from the Opensearch index 478 // THEN verify that a recent pattern-matched log record of hellodomain-adminserver stdout is found 479 func() { 480 t.It("Verify recent pattern-matched AdminServer log record exists", func() { 481 Eventually(func() bool { 482 return pkg.FindLog(indexName, 483 []pkg.Match{ 484 {Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"}, 485 {Key: "subSystem", Value: "WorkManager"}, 486 {Key: "serverName", Value: wlsAdminServer}, 487 {Key: "serverName2", Value: "AdminServer"}, 488 {Key: "message", Value: "Self-tuning"}}, 489 []pkg.Match{}) 490 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 491 }) 492 }, 493 // GIVEN a WebLogic application with logging enabled 494 // WHEN the log records are retrieved from the Opensearch index 495 // THEN verify that a recent pattern-matched log record of hellodomain-managedserver1 stdout is found 496 func() { 497 t.It("Verify recent pattern-matched ManagedServer log record exists", func() { 498 Eventually(func() bool { 499 return pkg.FindLog(indexName, 500 []pkg.Match{ 501 {Key: "kubernetes.container_name.keyword", Value: "fluentd-stdout-sidecar"}, 502 {Key: "subSystem", Value: "WorkManager"}, 503 {Key: "serverName", Value: wlsManagedServer1}, 504 {Key: "serverName2", Value: "ManagedServer1"}, 505 {Key: "message", Value: "Self-tuning"}}, 506 []pkg.Match{}) 507 }, shortWaitTimeout, longPollingInterval).Should(BeTrue(), "Expected to find a recent log record") 508 }) 509 }, 510 ) 511 }) 512 })