k8s.io/kubernetes@v1.29.3/test/e2e/framework/framework.go (about) 1 /* 2 Copyright 2015 The Kubernetes Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 // Package framework contains provider-independent helper code for 18 // building and running E2E tests with Ginkgo. The actual Ginkgo test 19 // suites gets assembled by combining this framework, the optional 20 // provider support code and specific tests via a separate .go file 21 // like Kubernetes' test/e2e.go. 22 package framework 23 24 import ( 25 "context" 26 "fmt" 27 "math/rand" 28 "os" 29 "path" 30 "reflect" 31 "strings" 32 "time" 33 34 "k8s.io/apimachinery/pkg/runtime" 35 36 v1 "k8s.io/api/core/v1" 37 apierrors "k8s.io/apimachinery/pkg/api/errors" 38 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 39 "k8s.io/apimachinery/pkg/labels" 40 "k8s.io/apimachinery/pkg/runtime/schema" 41 "k8s.io/apimachinery/pkg/util/wait" 42 v1svc "k8s.io/client-go/applyconfigurations/core/v1" 43 "k8s.io/client-go/discovery" 44 cacheddiscovery "k8s.io/client-go/discovery/cached/memory" 45 "k8s.io/client-go/dynamic" 46 clientset "k8s.io/client-go/kubernetes" 47 "k8s.io/client-go/kubernetes/scheme" 48 "k8s.io/client-go/rest" 49 "k8s.io/client-go/restmapper" 50 scaleclient "k8s.io/client-go/scale" 51 admissionapi "k8s.io/pod-security-admission/api" 52 53 "github.com/onsi/ginkgo/v2" 54 ) 55 56 const ( 57 // DefaultNamespaceDeletionTimeout is timeout duration for waiting for a namespace deletion. 58 DefaultNamespaceDeletionTimeout = 5 * time.Minute 59 defaultServiceAccountName = "default" 60 ) 61 62 var ( 63 // NewFrameworkExtensions lists functions that get called by 64 // NewFramework after constructing a new framework and after 65 // calling ginkgo.BeforeEach for the framework. 66 // 67 // This can be used by extensions of the core framework to modify 68 // settings in the framework instance or to add additional callbacks 69 // with gingko.BeforeEach/AfterEach/DeferCleanup. 70 // 71 // When a test runs, functions will be invoked in this order: 72 // - BeforeEaches defined by tests before f.NewDefaultFramework 73 // in the order in which they were defined (first-in-first-out) 74 // - f.BeforeEach 75 // - BeforeEaches defined by tests after f.NewDefaultFramework 76 // - It callback 77 // - all AfterEaches in the order in which they were defined 78 // - all DeferCleanups with the order reversed (first-in-last-out) 79 // - f.AfterEach 80 // 81 // Because a test might skip test execution in a BeforeEach that runs 82 // before f.BeforeEach, AfterEach callbacks that depend on the 83 // framework instance must check whether it was initialized. They can 84 // do that by checking f.ClientSet for nil. DeferCleanup callbacks 85 // don't need to do this because they get defined when the test 86 // runs. 87 NewFrameworkExtensions []func(f *Framework) 88 ) 89 90 // Framework supports common operations used by e2e tests; it will keep a client & a namespace for you. 91 // Eventual goal is to merge this with integration test framework. 92 // 93 // You can configure the pod security level for your test by setting the `NamespacePodSecurityLevel` 94 // which will set all three of pod security admission enforce, warn and audit labels on the namespace. 95 // The default pod security profile is "restricted". 96 // Each of the labels can be overridden by using more specific NamespacePodSecurity* attributes of this 97 // struct. 98 type Framework struct { 99 BaseName string 100 101 // Set together with creating the ClientSet and the namespace. 102 // Guaranteed to be unique in the cluster even when running the same 103 // test multiple times in parallel. 104 UniqueName string 105 106 clientConfig *rest.Config 107 ClientSet clientset.Interface 108 KubemarkExternalClusterClientSet clientset.Interface 109 110 DynamicClient dynamic.Interface 111 112 ScalesGetter scaleclient.ScalesGetter 113 114 SkipNamespaceCreation bool // Whether to skip creating a namespace 115 SkipSecretCreation bool // Whether to skip creating secret for a test 116 Namespace *v1.Namespace // Every test has at least one namespace unless creation is skipped 117 namespacesToDelete []*v1.Namespace // Some tests have more than one. 118 NamespaceDeletionTimeout time.Duration 119 NamespacePodSecurityEnforceLevel admissionapi.Level // The pod security enforcement level for namespaces to be applied. 120 NamespacePodSecurityWarnLevel admissionapi.Level // The pod security warn (client logging) level for namespaces to be applied. 121 NamespacePodSecurityAuditLevel admissionapi.Level // The pod security audit (server logging) level for namespaces to be applied. 122 NamespacePodSecurityLevel admissionapi.Level // The pod security level to be used for all of enforcement, warn and audit. Can be rewritten by more specific configuration attributes. 123 124 // Flaky operation failures in an e2e test can be captured through this. 125 flakeReport *FlakeReport 126 127 // configuration for framework's client 128 Options Options 129 130 // Place where various additional data is stored during test run to be printed to ReportDir, 131 // or stdout if ReportDir is not set once test ends. 132 TestSummaries []TestDataSummary 133 134 // Timeouts contains the custom timeouts used during the test execution. 135 Timeouts *TimeoutContext 136 137 // DumpAllNamespaceInfo is invoked by the framework to record 138 // information about a namespace after a test failure. 139 DumpAllNamespaceInfo DumpAllNamespaceInfoAction 140 } 141 142 // DumpAllNamespaceInfoAction is called after each failed test for namespaces 143 // created for the test. 144 type DumpAllNamespaceInfoAction func(ctx context.Context, f *Framework, namespace string) 145 146 // TestDataSummary is an interface for managing test data. 147 type TestDataSummary interface { 148 SummaryKind() string 149 PrintHumanReadable() string 150 PrintJSON() string 151 } 152 153 // Options is a struct for managing test framework options. 154 type Options struct { 155 ClientQPS float32 156 ClientBurst int 157 GroupVersion *schema.GroupVersion 158 } 159 160 // NewFrameworkWithCustomTimeouts makes a framework with custom timeouts. 161 // For timeout values that are zero the normal default value continues to 162 // be used. 163 func NewFrameworkWithCustomTimeouts(baseName string, timeouts *TimeoutContext) *Framework { 164 f := NewDefaultFramework(baseName) 165 in := reflect.ValueOf(timeouts).Elem() 166 out := reflect.ValueOf(f.Timeouts).Elem() 167 for i := 0; i < in.NumField(); i++ { 168 value := in.Field(i) 169 if !value.IsZero() { 170 out.Field(i).Set(value) 171 } 172 } 173 return f 174 } 175 176 // NewDefaultFramework makes a new framework and sets up a BeforeEach which 177 // initializes the framework instance. It cleans up with a DeferCleanup, 178 // which runs last, so a AfterEach in the test still has a valid framework 179 // instance. 180 func NewDefaultFramework(baseName string) *Framework { 181 options := Options{ 182 ClientQPS: 20, 183 ClientBurst: 50, 184 } 185 return NewFramework(baseName, options, nil) 186 } 187 188 // NewFramework creates a test framework. 189 func NewFramework(baseName string, options Options, client clientset.Interface) *Framework { 190 f := &Framework{ 191 BaseName: baseName, 192 Options: options, 193 ClientSet: client, 194 Timeouts: NewTimeoutContext(), 195 } 196 197 // The order is important here: if the extension calls ginkgo.BeforeEach 198 // itself, then it can be sure that f.BeforeEach already ran when its 199 // own callback gets invoked. 200 ginkgo.BeforeEach(f.BeforeEach, AnnotatedLocation("set up framework")) 201 for _, extension := range NewFrameworkExtensions { 202 extension(f) 203 } 204 205 return f 206 } 207 208 // BeforeEach gets a client and makes a namespace. 209 func (f *Framework) BeforeEach(ctx context.Context) { 210 // DeferCleanup, in contrast to AfterEach, triggers execution in 211 // first-in-last-out order. This ensures that the framework instance 212 // remains valid as long as possible. 213 // 214 // In addition, AfterEach will not be called if a test never gets here. 215 ginkgo.DeferCleanup(f.AfterEach, AnnotatedLocation("tear down framework")) 216 217 // Registered later and thus runs before deleting namespaces. 218 ginkgo.DeferCleanup(f.dumpNamespaceInfo, AnnotatedLocation("dump namespaces")) 219 220 ginkgo.By("Creating a kubernetes client") 221 config, err := LoadConfig() 222 ExpectNoError(err) 223 224 config.QPS = f.Options.ClientQPS 225 config.Burst = f.Options.ClientBurst 226 if f.Options.GroupVersion != nil { 227 config.GroupVersion = f.Options.GroupVersion 228 } 229 if TestContext.KubeAPIContentType != "" { 230 config.ContentType = TestContext.KubeAPIContentType 231 } 232 f.clientConfig = rest.CopyConfig(config) 233 f.ClientSet, err = clientset.NewForConfig(config) 234 ExpectNoError(err) 235 f.DynamicClient, err = dynamic.NewForConfig(config) 236 ExpectNoError(err) 237 238 // create scales getter, set GroupVersion and NegotiatedSerializer to default values 239 // as they are required when creating a REST client. 240 if config.GroupVersion == nil { 241 config.GroupVersion = &schema.GroupVersion{} 242 } 243 if config.NegotiatedSerializer == nil { 244 config.NegotiatedSerializer = scheme.Codecs 245 } 246 restClient, err := rest.RESTClientFor(config) 247 ExpectNoError(err) 248 discoClient, err := discovery.NewDiscoveryClientForConfig(config) 249 ExpectNoError(err) 250 cachedDiscoClient := cacheddiscovery.NewMemCacheClient(discoClient) 251 restMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoClient) 252 restMapper.Reset() 253 resolver := scaleclient.NewDiscoveryScaleKindResolver(cachedDiscoClient) 254 f.ScalesGetter = scaleclient.New(restClient, restMapper, dynamic.LegacyAPIPathResolverFunc, resolver) 255 256 TestContext.CloudConfig.Provider.FrameworkBeforeEach(f) 257 258 if !f.SkipNamespaceCreation { 259 ginkgo.By(fmt.Sprintf("Building a namespace api object, basename %s", f.BaseName)) 260 namespace, err := f.CreateNamespace(ctx, f.BaseName, map[string]string{ 261 "e2e-framework": f.BaseName, 262 }) 263 ExpectNoError(err) 264 265 f.Namespace = namespace 266 267 if TestContext.VerifyServiceAccount { 268 ginkgo.By("Waiting for a default service account to be provisioned in namespace") 269 err = WaitForDefaultServiceAccountInNamespace(ctx, f.ClientSet, namespace.Name) 270 ExpectNoError(err) 271 ginkgo.By("Waiting for kube-root-ca.crt to be provisioned in namespace") 272 err = WaitForKubeRootCAInNamespace(ctx, f.ClientSet, namespace.Name) 273 ExpectNoError(err) 274 } else { 275 Logf("Skipping waiting for service account") 276 } 277 278 f.UniqueName = f.Namespace.GetName() 279 } else { 280 // not guaranteed to be unique, but very likely 281 f.UniqueName = fmt.Sprintf("%s-%08x", f.BaseName, rand.Int31()) 282 } 283 284 f.flakeReport = NewFlakeReport() 285 } 286 287 func (f *Framework) dumpNamespaceInfo(ctx context.Context) { 288 if !ginkgo.CurrentSpecReport().Failed() { 289 return 290 } 291 if !TestContext.DumpLogsOnFailure { 292 return 293 } 294 if f.DumpAllNamespaceInfo == nil { 295 return 296 } 297 ginkgo.By("dump namespace information after failure", func() { 298 if !f.SkipNamespaceCreation { 299 for _, ns := range f.namespacesToDelete { 300 f.DumpAllNamespaceInfo(ctx, f, ns.Name) 301 } 302 } 303 }) 304 } 305 306 // printSummaries prints summaries of tests. 307 func printSummaries(summaries []TestDataSummary, testBaseName string) { 308 now := time.Now() 309 for i := range summaries { 310 Logf("Printing summary: %v", summaries[i].SummaryKind()) 311 switch TestContext.OutputPrintType { 312 case "hr": 313 if TestContext.ReportDir == "" { 314 Logf(summaries[i].PrintHumanReadable()) 315 } else { 316 // TODO: learn to extract test name and append it to the kind instead of timestamp. 317 filePath := path.Join(TestContext.ReportDir, summaries[i].SummaryKind()+"_"+testBaseName+"_"+now.Format(time.RFC3339)+".txt") 318 if err := os.WriteFile(filePath, []byte(summaries[i].PrintHumanReadable()), 0644); err != nil { 319 Logf("Failed to write file %v with test performance data: %v", filePath, err) 320 } 321 } 322 case "json": 323 fallthrough 324 default: 325 if TestContext.OutputPrintType != "json" { 326 Logf("Unknown output type: %v. Printing JSON", TestContext.OutputPrintType) 327 } 328 if TestContext.ReportDir == "" { 329 Logf("%v JSON\n%v", summaries[i].SummaryKind(), summaries[i].PrintJSON()) 330 Logf("Finished") 331 } else { 332 // TODO: learn to extract test name and append it to the kind instead of timestamp. 333 filePath := path.Join(TestContext.ReportDir, summaries[i].SummaryKind()+"_"+testBaseName+"_"+now.Format(time.RFC3339)+".json") 334 Logf("Writing to %s", filePath) 335 if err := os.WriteFile(filePath, []byte(summaries[i].PrintJSON()), 0644); err != nil { 336 Logf("Failed to write file %v with test performance data: %v", filePath, err) 337 } 338 } 339 } 340 } 341 } 342 343 // AfterEach deletes the namespace, after reading its events. 344 func (f *Framework) AfterEach(ctx context.Context) { 345 // This should not happen. Given ClientSet is a public field a test must have updated it! 346 // Error out early before any API calls during cleanup. 347 if f.ClientSet == nil { 348 Failf("The framework ClientSet must not be nil at this point") 349 } 350 351 // DeleteNamespace at the very end in defer, to avoid any 352 // expectation failures preventing deleting the namespace. 353 defer func() { 354 nsDeletionErrors := map[string]error{} 355 // Whether to delete namespace is determined by 3 factors: delete-namespace flag, delete-namespace-on-failure flag and the test result 356 // if delete-namespace set to false, namespace will always be preserved. 357 // if delete-namespace is true and delete-namespace-on-failure is false, namespace will be preserved if test failed. 358 if TestContext.DeleteNamespace && (TestContext.DeleteNamespaceOnFailure || !ginkgo.CurrentSpecReport().Failed()) { 359 for _, ns := range f.namespacesToDelete { 360 ginkgo.By(fmt.Sprintf("Destroying namespace %q for this suite.", ns.Name)) 361 if err := f.ClientSet.CoreV1().Namespaces().Delete(ctx, ns.Name, metav1.DeleteOptions{}); err != nil { 362 if !apierrors.IsNotFound(err) { 363 nsDeletionErrors[ns.Name] = err 364 365 // Dump namespace if we are unable to delete the namespace and the dump was not already performed. 366 if !ginkgo.CurrentSpecReport().Failed() && TestContext.DumpLogsOnFailure && f.DumpAllNamespaceInfo != nil { 367 f.DumpAllNamespaceInfo(ctx, f, ns.Name) 368 } 369 } else { 370 Logf("Namespace %v was already deleted", ns.Name) 371 } 372 } 373 } 374 } else { 375 if !TestContext.DeleteNamespace { 376 Logf("Found DeleteNamespace=false, skipping namespace deletion!") 377 } else { 378 Logf("Found DeleteNamespaceOnFailure=false and current test failed, skipping namespace deletion!") 379 } 380 } 381 382 // Unsetting this is relevant for a following test that uses 383 // the same instance because it might not reach f.BeforeEach 384 // when some other BeforeEach skips the test first. 385 f.Namespace = nil 386 f.clientConfig = nil 387 f.ClientSet = nil 388 f.namespacesToDelete = nil 389 390 // if we had errors deleting, report them now. 391 if len(nsDeletionErrors) != 0 { 392 messages := []string{} 393 for namespaceKey, namespaceErr := range nsDeletionErrors { 394 messages = append(messages, fmt.Sprintf("Couldn't delete ns: %q: %s (%#v)", namespaceKey, namespaceErr, namespaceErr)) 395 } 396 Failf(strings.Join(messages, ",")) 397 } 398 }() 399 400 TestContext.CloudConfig.Provider.FrameworkAfterEach(f) 401 402 // Report any flakes that were observed in the e2e test and reset. 403 if f.flakeReport != nil && f.flakeReport.GetFlakeCount() > 0 { 404 f.TestSummaries = append(f.TestSummaries, f.flakeReport) 405 f.flakeReport = nil 406 } 407 408 printSummaries(f.TestSummaries, f.BaseName) 409 } 410 411 // DeleteNamespace can be used to delete a namespace. Additionally it can be used to 412 // dump namespace information so as it can be used as an alternative of framework 413 // deleting the namespace towards the end. 414 func (f *Framework) DeleteNamespace(ctx context.Context, name string) { 415 defer func() { 416 err := f.ClientSet.CoreV1().Namespaces().Delete(ctx, name, metav1.DeleteOptions{}) 417 if err != nil && !apierrors.IsNotFound(err) { 418 Logf("error deleting namespace %s: %v", name, err) 419 return 420 } 421 err = WaitForNamespacesDeleted(ctx, f.ClientSet, []string{name}, DefaultNamespaceDeletionTimeout) 422 if err != nil { 423 Logf("error deleting namespace %s: %v", name, err) 424 return 425 } 426 // remove deleted namespace from namespacesToDelete map 427 for i, ns := range f.namespacesToDelete { 428 if ns == nil { 429 continue 430 } 431 if ns.Name == name { 432 f.namespacesToDelete = append(f.namespacesToDelete[:i], f.namespacesToDelete[i+1:]...) 433 } 434 } 435 }() 436 // if current test failed then we should dump namespace information 437 if !f.SkipNamespaceCreation && ginkgo.CurrentSpecReport().Failed() && TestContext.DumpLogsOnFailure && f.DumpAllNamespaceInfo != nil { 438 f.DumpAllNamespaceInfo(ctx, f, name) 439 } 440 441 } 442 443 // CreateNamespace creates a namespace for e2e testing. 444 func (f *Framework) CreateNamespace(ctx context.Context, baseName string, labels map[string]string) (*v1.Namespace, error) { 445 createTestingNS := TestContext.CreateTestingNS 446 if createTestingNS == nil { 447 createTestingNS = CreateTestingNS 448 } 449 450 if labels == nil { 451 labels = make(map[string]string) 452 } else { 453 labelsCopy := make(map[string]string) 454 for k, v := range labels { 455 labelsCopy[k] = v 456 } 457 labels = labelsCopy 458 } 459 460 labels[admissionapi.EnforceLevelLabel] = firstNonEmptyPSaLevelOrRestricted(f.NamespacePodSecurityEnforceLevel, f.NamespacePodSecurityLevel) 461 labels[admissionapi.WarnLevelLabel] = firstNonEmptyPSaLevelOrRestricted(f.NamespacePodSecurityWarnLevel, f.NamespacePodSecurityLevel) 462 labels[admissionapi.AuditLevelLabel] = firstNonEmptyPSaLevelOrRestricted(f.NamespacePodSecurityAuditLevel, f.NamespacePodSecurityLevel) 463 464 ns, err := createTestingNS(ctx, baseName, f.ClientSet, labels) 465 // check ns instead of err to see if it's nil as we may 466 // fail to create serviceAccount in it. 467 f.AddNamespacesToDelete(ns) 468 469 if TestContext.E2EDockerConfigFile != "" && !f.SkipSecretCreation { 470 // With the Secret created, the default service account (in the new namespace) 471 // is patched with the secret and can then be referenced by all the pods spawned by E2E process, and repository authentication should be successful. 472 secret, err := f.createSecretFromDockerConfig(ctx, ns.Name) 473 if err != nil { 474 return ns, fmt.Errorf("failed to create secret from docker config file: %v", err) 475 } 476 477 serviceAccountClient := f.ClientSet.CoreV1().ServiceAccounts(ns.Name) 478 serviceAccountConfig := v1svc.ServiceAccount(defaultServiceAccountName, ns.Name) 479 serviceAccountConfig.ImagePullSecrets = append(serviceAccountConfig.ImagePullSecrets, v1svc.LocalObjectReferenceApplyConfiguration{Name: &secret.Name}) 480 481 svc, err := serviceAccountClient.Apply(ctx, serviceAccountConfig, metav1.ApplyOptions{FieldManager: "e2e-framework"}) 482 if err != nil { 483 return ns, fmt.Errorf("failed to patch imagePullSecret [%s] to service account [%s]: %v", secret.Name, svc.Name, err) 484 } 485 486 } 487 488 return ns, err 489 } 490 491 func firstNonEmptyPSaLevelOrRestricted(levelConfig ...admissionapi.Level) string { 492 for _, l := range levelConfig { 493 if len(l) > 0 { 494 return string(l) 495 } 496 } 497 return string(admissionapi.LevelRestricted) 498 } 499 500 // createSecretFromDockerConfig creates a secret using the private image registry credentials. 501 // The credentials are provided by --e2e-docker-config-file flag. 502 func (f *Framework) createSecretFromDockerConfig(ctx context.Context, namespace string) (*v1.Secret, error) { 503 contents, err := os.ReadFile(TestContext.E2EDockerConfigFile) 504 if err != nil { 505 return nil, fmt.Errorf("error reading docker config file: %v", err) 506 } 507 508 secretObject := &v1.Secret{ 509 Data: map[string][]byte{v1.DockerConfigJsonKey: contents}, 510 Type: v1.SecretTypeDockerConfigJson, 511 } 512 secretObject.GenerateName = "registry-cred" 513 Logf("create image pull secret %s", secretObject.Name) 514 515 secret, err := f.ClientSet.CoreV1().Secrets(namespace).Create(ctx, secretObject, metav1.CreateOptions{}) 516 517 return secret, err 518 } 519 520 // RecordFlakeIfError records flakeness info if error happens. 521 // NOTE: This function is not used at any places yet, but we are in progress for https://github.com/kubernetes/kubernetes/issues/66239 which requires this. Please don't remove this. 522 func (f *Framework) RecordFlakeIfError(err error, optionalDescription ...interface{}) { 523 f.flakeReport.RecordFlakeIfError(err, optionalDescription...) 524 } 525 526 // AddNamespacesToDelete adds one or more namespaces to be deleted when the test 527 // completes. 528 func (f *Framework) AddNamespacesToDelete(namespaces ...*v1.Namespace) { 529 for _, ns := range namespaces { 530 if ns == nil { 531 continue 532 } 533 f.namespacesToDelete = append(f.namespacesToDelete, ns) 534 535 } 536 } 537 538 // ClientConfig an externally accessible method for reading the kube client config. 539 func (f *Framework) ClientConfig() *rest.Config { 540 ret := rest.CopyConfig(f.clientConfig) 541 // json is least common denominator 542 ret.ContentType = runtime.ContentTypeJSON 543 ret.AcceptContentTypes = runtime.ContentTypeJSON 544 return ret 545 } 546 547 // KubeUser is a struct for managing kubernetes user info. 548 type KubeUser struct { 549 Name string `yaml:"name"` 550 User struct { 551 Username string `yaml:"username"` 552 Password string `yaml:"password" datapolicy:"password"` 553 Token string `yaml:"token" datapolicy:"token"` 554 } `yaml:"user"` 555 } 556 557 // KubeCluster is a struct for managing kubernetes cluster info. 558 type KubeCluster struct { 559 Name string `yaml:"name"` 560 Cluster struct { 561 CertificateAuthorityData string `yaml:"certificate-authority-data"` 562 Server string `yaml:"server"` 563 } `yaml:"cluster"` 564 } 565 566 // KubeConfig is a struct for managing kubernetes config. 567 type KubeConfig struct { 568 Contexts []struct { 569 Name string `yaml:"name"` 570 Context struct { 571 Cluster string `yaml:"cluster"` 572 User string 573 } `yaml:"context"` 574 } `yaml:"contexts"` 575 576 Clusters []KubeCluster `yaml:"clusters"` 577 578 Users []KubeUser `yaml:"users"` 579 } 580 581 // FindUser returns user info which is the specified user name. 582 func (kc *KubeConfig) FindUser(name string) *KubeUser { 583 for _, user := range kc.Users { 584 if user.Name == name { 585 return &user 586 } 587 } 588 return nil 589 } 590 591 // FindCluster returns cluster info which is the specified cluster name. 592 func (kc *KubeConfig) FindCluster(name string) *KubeCluster { 593 for _, cluster := range kc.Clusters { 594 if cluster.Name == name { 595 return &cluster 596 } 597 } 598 return nil 599 } 600 601 // PodStateVerification represents a verification of pod state. 602 // Any time you have a set of pods that you want to operate against or query, 603 // this struct can be used to declaratively identify those pods. 604 type PodStateVerification struct { 605 // Optional: only pods that have k=v labels will pass this filter. 606 Selectors map[string]string 607 608 // Required: The phases which are valid for your pod. 609 ValidPhases []v1.PodPhase 610 611 // Optional: only pods passing this function will pass the filter 612 // Verify a pod. 613 // As an optimization, in addition to specifying filter (boolean), 614 // this function allows specifying an error as well. 615 // The error indicates that the polling of the pod spectrum should stop. 616 Verify func(v1.Pod) (bool, error) 617 618 // Optional: only pods with this name will pass the filter. 619 PodName string 620 } 621 622 // ClusterVerification is a struct for a verification of cluster state. 623 type ClusterVerification struct { 624 client clientset.Interface 625 namespace *v1.Namespace // pointer rather than string, since ns isn't created until before each. 626 podState PodStateVerification 627 } 628 629 // NewClusterVerification creates a new cluster verification. 630 func (f *Framework) NewClusterVerification(namespace *v1.Namespace, filter PodStateVerification) *ClusterVerification { 631 return &ClusterVerification{ 632 f.ClientSet, 633 namespace, 634 filter, 635 } 636 } 637 638 func passesPodNameFilter(pod v1.Pod, name string) bool { 639 return name == "" || strings.Contains(pod.Name, name) 640 } 641 642 func passesVerifyFilter(pod v1.Pod, verify func(p v1.Pod) (bool, error)) (bool, error) { 643 if verify == nil { 644 return true, nil 645 } 646 647 verified, err := verify(pod) 648 // If an error is returned, by definition, pod verification fails 649 if err != nil { 650 return false, err 651 } 652 return verified, nil 653 } 654 655 func passesPhasesFilter(pod v1.Pod, validPhases []v1.PodPhase) bool { 656 passesPhaseFilter := false 657 for _, phase := range validPhases { 658 if pod.Status.Phase == phase { 659 passesPhaseFilter = true 660 } 661 } 662 return passesPhaseFilter 663 } 664 665 // filterLabels returns a list of pods which have labels. 666 func filterLabels(ctx context.Context, selectors map[string]string, cli clientset.Interface, ns string) (*v1.PodList, error) { 667 var err error 668 var selector labels.Selector 669 var pl *v1.PodList 670 // List pods based on selectors. This might be a tiny optimization rather then filtering 671 // everything manually. 672 if len(selectors) > 0 { 673 selector = labels.SelectorFromSet(labels.Set(selectors)) 674 options := metav1.ListOptions{LabelSelector: selector.String()} 675 pl, err = cli.CoreV1().Pods(ns).List(ctx, options) 676 } else { 677 pl, err = cli.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{}) 678 } 679 return pl, err 680 } 681 682 // filter filters pods which pass a filter. It can be used to compose 683 // the more useful abstractions like ForEach, WaitFor, and so on, which 684 // can be used directly by tests. 685 func (p *PodStateVerification) filter(ctx context.Context, c clientset.Interface, namespace *v1.Namespace) ([]v1.Pod, error) { 686 if len(p.ValidPhases) == 0 || namespace == nil { 687 panic(fmt.Errorf("Need to specify a valid pod phases (%v) and namespace (%v). ", p.ValidPhases, namespace)) 688 } 689 690 ns := namespace.Name 691 pl, err := filterLabels(ctx, p.Selectors, c, ns) // Build an v1.PodList to operate against. 692 Logf("Selector matched %v pods for %v", len(pl.Items), p.Selectors) 693 if len(pl.Items) == 0 || err != nil { 694 return pl.Items, err 695 } 696 697 unfilteredPods := pl.Items 698 filteredPods := []v1.Pod{} 699 ReturnPodsSoFar: 700 // Next: Pod must match at least one of the states that the user specified 701 for _, pod := range unfilteredPods { 702 if !(passesPhasesFilter(pod, p.ValidPhases) && passesPodNameFilter(pod, p.PodName)) { 703 continue 704 } 705 passesVerify, err := passesVerifyFilter(pod, p.Verify) 706 if err != nil { 707 Logf("Error detected on %v : %v !", pod.Name, err) 708 break ReturnPodsSoFar 709 } 710 if passesVerify { 711 filteredPods = append(filteredPods, pod) 712 } 713 } 714 return filteredPods, err 715 } 716 717 // WaitFor waits for some minimum number of pods to be verified, according to the PodStateVerification 718 // definition. 719 func (cl *ClusterVerification) WaitFor(ctx context.Context, atLeast int, timeout time.Duration) ([]v1.Pod, error) { 720 pods := []v1.Pod{} 721 var returnedErr error 722 723 err := wait.PollWithContext(ctx, 1*time.Second, timeout, func(ctx context.Context) (bool, error) { 724 pods, returnedErr = cl.podState.filter(ctx, cl.client, cl.namespace) 725 726 // Failure 727 if returnedErr != nil { 728 Logf("Cutting polling short: We got an error from the pod filtering layer.") 729 // stop polling if the pod filtering returns an error. that should never happen. 730 // it indicates, for example, that the client is broken or something non-pod related. 731 return false, returnedErr 732 } 733 Logf("Found %v / %v", len(pods), atLeast) 734 735 // Success 736 if len(pods) >= atLeast { 737 return true, nil 738 } 739 // Keep trying... 740 return false, nil 741 }) 742 Logf("WaitFor completed with timeout %v. Pods found = %v out of %v", timeout, len(pods), atLeast) 743 return pods, err 744 } 745 746 // WaitForOrFail provides a shorthand WaitFor with failure as an option if anything goes wrong. 747 func (cl *ClusterVerification) WaitForOrFail(ctx context.Context, atLeast int, timeout time.Duration) { 748 pods, err := cl.WaitFor(ctx, atLeast, timeout) 749 if err != nil || len(pods) < atLeast { 750 Failf("Verified %v of %v pods , error : %v", len(pods), atLeast, err) 751 } 752 } 753 754 // ForEach runs a function against every verifiable pod. Be warned that this doesn't wait for "n" pods to verify, 755 // so it may return very quickly if you have strict pod state requirements. 756 // 757 // For example, if you require at least 5 pods to be running before your test will pass, 758 // its smart to first call "clusterVerification.WaitFor(5)" before you call clusterVerification.ForEach. 759 func (cl *ClusterVerification) ForEach(ctx context.Context, podFunc func(v1.Pod)) error { 760 pods, err := cl.podState.filter(ctx, cl.client, cl.namespace) 761 if err == nil { 762 if len(pods) == 0 { 763 Failf("No pods matched the filter.") 764 } 765 Logf("ForEach: Found %v pods from the filter. Now looping through them.", len(pods)) 766 for _, p := range pods { 767 podFunc(p) 768 } 769 } else { 770 Logf("ForEach: Something went wrong when filtering pods to execute against: %v", err) 771 } 772 773 return err 774 }