github.com/aloknnikhil/operator-sdk@v0.8.2/test/e2e/memcached_test.go (about) 1 // Copyright 2018 The Operator-SDK Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package e2e 16 17 import ( 18 "bytes" 19 "context" 20 "fmt" 21 "io/ioutil" 22 "os" 23 "os/exec" 24 "path/filepath" 25 "regexp" 26 "strings" 27 "testing" 28 "time" 29 30 "github.com/operator-framework/operator-sdk/internal/pkg/scaffold" 31 "github.com/operator-framework/operator-sdk/internal/util/fileutil" 32 "github.com/operator-framework/operator-sdk/internal/util/projutil" 33 "github.com/operator-framework/operator-sdk/internal/util/yamlutil" 34 framework "github.com/operator-framework/operator-sdk/pkg/test" 35 "github.com/operator-framework/operator-sdk/pkg/test/e2eutil" 36 37 "github.com/ghodss/yaml" 38 "github.com/pkg/errors" 39 "github.com/prometheus/prometheus/util/promlint" 40 v1 "k8s.io/api/core/v1" 41 apierrors "k8s.io/apimachinery/pkg/api/errors" 42 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 43 "k8s.io/apimachinery/pkg/types" 44 "k8s.io/apimachinery/pkg/util/wait" 45 "k8s.io/client-go/kubernetes" 46 "k8s.io/client-go/rest" 47 "sigs.k8s.io/controller-runtime/pkg/client" 48 ) 49 50 const ( 51 crYAML string = "apiVersion: \"cache.example.com/v1alpha1\"\nkind: \"Memcached\"\nmetadata:\n name: \"example-memcached\"\nspec:\n size: 3" 52 retryInterval = time.Second * 5 53 timeout = time.Second * 120 54 cleanupRetryInterval = time.Second * 1 55 cleanupTimeout = time.Second * 10 56 operatorName = "memcached-operator" 57 ) 58 59 func TestMemcached(t *testing.T) { 60 // get global framework variables 61 ctx := framework.NewTestCtx(t) 62 defer ctx.Cleanup() 63 gopath, ok := os.LookupEnv(projutil.GoPathEnv) 64 if !ok || gopath == "" { 65 t.Fatalf("$GOPATH not set") 66 } 67 cd, err := os.Getwd() 68 if err != nil { 69 t.Fatal(err) 70 } 71 defer func() { 72 if err := os.Chdir(cd); err != nil { 73 t.Errorf("Failed to change back to original working directory: (%v)", err) 74 } 75 }() 76 // For go commands in operator projects. 77 if err = os.Setenv("GO111MODULE", "on"); err != nil { 78 t.Fatal(err) 79 } 80 81 // Setup 82 absProjectPath, err := ioutil.TempDir(filepath.Join(gopath, "src"), "tmp.") 83 if err != nil { 84 t.Fatal(err) 85 } 86 ctx.AddCleanupFn(func() error { return os.RemoveAll(absProjectPath) }) 87 88 if err := os.MkdirAll(absProjectPath, fileutil.DefaultDirFileMode); err != nil { 89 t.Fatal(err) 90 } 91 if err := os.Chdir(absProjectPath); err != nil { 92 t.Fatal(err) 93 } 94 95 t.Log("Creating new operator project") 96 cmdOut, err := exec.Command("operator-sdk", 97 "new", 98 operatorName).CombinedOutput() 99 if err != nil { 100 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 101 } 102 103 if err := os.Chdir(operatorName); err != nil { 104 t.Fatalf("Failed to change to %s directory: (%v)", operatorName, err) 105 } 106 107 sdkRepo := "github.com/operator-framework/operator-sdk" 108 localSDKPath := filepath.Join(gopath, "src", sdkRepo) 109 110 replace := getGoModReplace(t, localSDKPath) 111 if replace.repo != sdkRepo { 112 if replace.isLocal { 113 // A hacky way to get local module substitution to work is to write a 114 // stub go.mod into the local SDK repo referred to in 115 // memcached-operator's go.mod, which allows go to recognize 116 // the local SDK repo as a module. 117 sdkModPath := filepath.Join(filepath.FromSlash(replace.repo), "go.mod") 118 if _, err = os.Stat(sdkModPath); err != nil && os.IsNotExist(err) { 119 err = ioutil.WriteFile(sdkModPath, []byte("module "+sdkRepo), fileutil.DefaultFileMode) 120 if err != nil { 121 t.Fatalf("Failed to write main repo go.mod file: %v", err) 122 } 123 defer func() { 124 if err = os.RemoveAll(sdkModPath); err != nil { 125 t.Fatalf("Failed to remove %s: %v", sdkModPath, err) 126 } 127 }() 128 } 129 } 130 modBytes, err := insertGoModReplace(sdkRepo, replace.repo, replace.ref) 131 if err != nil { 132 t.Fatalf("Failed to insert go.mod replace: %v", err) 133 } 134 t.Logf("go.mod: %v", string(modBytes)) 135 } 136 137 cmdOut, err = exec.Command("go", "mod", "vendor").CombinedOutput() 138 if err != nil { 139 t.Fatalf("Error after modifying go.mod: %v\nCommand Output: %s\n", err, string(cmdOut)) 140 } 141 142 // Set replicas to 2 to test leader election. In production, this should 143 // almost always be set to 1, because there isn't generally value in having 144 // a hot spare operator process. 145 opYaml, err := ioutil.ReadFile("deploy/operator.yaml") 146 if err != nil { 147 t.Fatalf("Could not read deploy/operator.yaml: %v", err) 148 } 149 newOpYaml := bytes.Replace(opYaml, []byte("replicas: 1"), []byte("replicas: 2"), 1) 150 err = ioutil.WriteFile("deploy/operator.yaml", newOpYaml, 0644) 151 if err != nil { 152 t.Fatalf("Could not write deploy/operator.yaml: %v", err) 153 } 154 155 cmd := exec.Command("operator-sdk", 156 "add", 157 "api", 158 "--api-version=cache.example.com/v1alpha1", 159 "--kind=Memcached") 160 cmd.Env = os.Environ() 161 cmdOut, err = cmd.CombinedOutput() 162 if err != nil { 163 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 164 } 165 cmdOut, err = exec.Command("operator-sdk", 166 "add", 167 "controller", 168 "--api-version=cache.example.com/v1alpha1", 169 "--kind=Memcached").CombinedOutput() 170 if err != nil { 171 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 172 } 173 174 tmplFiles := map[string]string{ 175 filepath.Join(localSDKPath, "example/memcached-operator/memcached_controller.go.tmpl"): "pkg/controller/memcached/memcached_controller.go", 176 filepath.Join(localSDKPath, "test/e2e/incluster-test-code/main_test.go.tmpl"): "test/e2e/main_test.go", 177 filepath.Join(localSDKPath, "test/e2e/incluster-test-code/memcached_test.go.tmpl"): "test/e2e/memcached_test.go", 178 } 179 for src, dst := range tmplFiles { 180 if err := os.MkdirAll(filepath.Dir(dst), fileutil.DefaultDirFileMode); err != nil { 181 t.Fatalf("Could not create template destination directory: %s", err) 182 } 183 srcTmpl, err := ioutil.ReadFile(src) 184 if err != nil { 185 t.Fatalf("Could not read template from %s: %s", src, err) 186 } 187 dstData := strings.Replace(string(srcTmpl), "github.com/example-inc", filepath.Base(absProjectPath), -1) 188 if err := ioutil.WriteFile(dst, []byte(dstData), fileutil.DefaultFileMode); err != nil { 189 t.Fatalf("Could not write template output to %s: %s", dst, err) 190 } 191 } 192 193 memcachedTypesFile, err := ioutil.ReadFile("pkg/apis/cache/v1alpha1/memcached_types.go") 194 if err != nil { 195 t.Fatal(err) 196 } 197 memcachedTypesFileLines := bytes.Split(memcachedTypesFile, []byte("\n")) 198 for lineNum, line := range memcachedTypesFileLines { 199 if strings.Contains(string(line), "type MemcachedSpec struct {") { 200 memcachedTypesFileLinesIntermediate := append(memcachedTypesFileLines[:lineNum+1], []byte("\tSize int32 `json:\"size\"`")) 201 memcachedTypesFileLines = append(memcachedTypesFileLinesIntermediate, memcachedTypesFileLines[lineNum+3:]...) 202 break 203 } 204 } 205 for lineNum, line := range memcachedTypesFileLines { 206 if strings.Contains(string(line), "type MemcachedStatus struct {") { 207 memcachedTypesFileLinesIntermediate := append(memcachedTypesFileLines[:lineNum+1], []byte("\tNodes []string `json:\"nodes\"`")) 208 memcachedTypesFileLines = append(memcachedTypesFileLinesIntermediate, memcachedTypesFileLines[lineNum+3:]...) 209 break 210 } 211 } 212 if err := os.Remove("pkg/apis/cache/v1alpha1/memcached_types.go"); err != nil { 213 t.Fatalf("Failed to remove old memcached_type.go file: (%v)", err) 214 } 215 err = ioutil.WriteFile("pkg/apis/cache/v1alpha1/memcached_types.go", bytes.Join(memcachedTypesFileLines, []byte("\n")), fileutil.DefaultFileMode) 216 if err != nil { 217 t.Fatal(err) 218 } 219 220 t.Log("Generating k8s") 221 cmdOut, err = exec.Command("operator-sdk", "generate", "k8s").CombinedOutput() 222 if err != nil { 223 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 224 } 225 226 t.Log("Pulling new dependencies with go mod") 227 cmdOut, err = exec.Command("go", "mod", "vendor").CombinedOutput() 228 if err != nil { 229 t.Fatalf("Command 'go mod vendor' failed: %v\nCommand Output:\n%v", err, string(cmdOut)) 230 } 231 232 file, err := yamlutil.GenerateCombinedGlobalManifest(scaffold.CRDsDir) 233 if err != nil { 234 t.Fatal(err) 235 } 236 ctx.AddCleanupFn(func() error { return os.Remove(file.Name()) }) 237 238 // hacky way to use createFromYAML without exposing the method 239 // create crd 240 filename := file.Name() 241 framework.Global.NamespacedManPath = &filename 242 err = ctx.InitializeClusterResources(&framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) 243 if err != nil { 244 t.Fatal(err) 245 } 246 t.Log("Created global resources") 247 248 // run subtests 249 t.Run("memcached-group", func(t *testing.T) { 250 t.Run("Cluster", MemcachedCluster) 251 t.Run("Local", MemcachedLocal) 252 }) 253 } 254 255 type goModReplace struct { 256 repo string 257 ref string 258 isLocal bool 259 } 260 261 // getGoModReplace returns a go.mod replacement that is appropriate based on the build's 262 // environment to support PR, fork/branch, and local builds. 263 // 264 // PR: 265 // 1. Activate when TRAVIS_PULL_REQUEST_SLUG and TRAVIS_PULL_REQUEST_SHA are set 266 // 2. Modify go.mod to replace osdk import with github.com/${TRAVIS_PULL_REQUEST_SLUG} ${TRAVIS_PULL_REQUEST_SHA} 267 // 268 // Fork/branch: 269 // 1. Activate when TRAVIS_REPO_SLUG and TRAVIS_COMMIT are set 270 // 2. Modify go.mod to replace osdk import with github.com/${TRAVIS_REPO_SLUG} ${TRAVIS_COMMIT} 271 // 272 // Local: 273 // 1. Activate when none of the above TRAVIS_* variables are set. 274 // 2. Modify go.mod to replace osdk import with local filesystem path. 275 // 276 func getGoModReplace(t *testing.T, localSDKPath string) goModReplace { 277 // PR environment 278 prSlug, prSlugOk := os.LookupEnv("TRAVIS_PULL_REQUEST_SLUG") 279 prSha, prShaOk := os.LookupEnv("TRAVIS_PULL_REQUEST_SHA") 280 if prSlugOk && prSlug != "" && prShaOk && prSha != "" { 281 return goModReplace{ 282 repo: fmt.Sprintf("github.com/%s", prSlug), 283 ref: prSha, 284 } 285 } 286 287 // Fork/branch environment 288 slug, slugOk := os.LookupEnv("TRAVIS_REPO_SLUG") 289 sha, shaOk := os.LookupEnv("TRAVIS_COMMIT") 290 if slugOk && slug != "" && shaOk && sha != "" { 291 return goModReplace{ 292 repo: fmt.Sprintf("github.com/%s", slug), 293 ref: sha, 294 } 295 } 296 297 // If neither of the above cases is applicable, but one of the TRAVIS_* 298 // variables is nonetheless set, something unexpected is going on. Log 299 // the vars and exit. 300 if prSlugOk || prShaOk || slugOk || shaOk { 301 t.Logf("TRAVIS_PULL_REQUEST_SLUG='%s', set: %t", prSlug, prSlugOk) 302 t.Logf("TRAVIS_PULL_REQUEST_SHA='%s', set: %t", prSha, prShaOk) 303 t.Logf("TRAVIS_REPO_SLUG='%s', set: %t", slug, slugOk) 304 t.Logf("TRAVIS_COMMIT='%s', set: %t", sha, shaOk) 305 t.Fatal("Invalid travis environment") 306 } 307 308 // Local environment 309 return goModReplace{ 310 repo: localSDKPath, 311 isLocal: true, 312 } 313 } 314 315 func insertGoModReplace(repo, path, sha string) ([]byte, error) { 316 modBytes, err := ioutil.ReadFile("go.mod") 317 if err != nil { 318 return nil, errors.Wrap(err, "failed to read go.mod") 319 } 320 // Remove all replace lines in go.mod. 321 replaceRe := regexp.MustCompile(fmt.Sprintf("(replace )?%s =>.+", repo)) 322 modBytes = replaceRe.ReplaceAll(modBytes, nil) 323 // Append the desired replace to the end of go.mod's bytes. 324 sdkReplace := fmt.Sprintf("replace %s => %s", repo, path) 325 if sha != "" { 326 sdkReplace = fmt.Sprintf("%s %s", sdkReplace, sha) 327 } 328 modBytes = append(modBytes, []byte("\n"+sdkReplace)...) 329 err = ioutil.WriteFile("go.mod", modBytes, fileutil.DefaultFileMode) 330 if err != nil { 331 return nil, errors.Wrap(err, "failed to write go.mod before replacing SDK repo") 332 } 333 return modBytes, nil 334 } 335 336 func memcachedLeaderTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error { 337 namespace, err := ctx.GetNamespace() 338 if err != nil { 339 return err 340 } 341 342 err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, operatorName, 2, retryInterval, timeout) 343 if err != nil { 344 return err 345 } 346 347 label := map[string]string{"name": operatorName} 348 349 leader, err := verifyLeader(t, namespace, f, label) 350 if err != nil { 351 return err 352 } 353 354 // delete the leader's pod so a new leader will get elected 355 err = f.Client.Delete(context.TODO(), leader) 356 if err != nil { 357 return err 358 } 359 360 err = e2eutil.WaitForDeletion(t, f.Client.Client, leader, retryInterval, timeout) 361 if err != nil { 362 return err 363 } 364 365 err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, operatorName, 2, retryInterval, timeout) 366 if err != nil { 367 return err 368 } 369 370 newLeader, err := verifyLeader(t, namespace, f, label) 371 if err != nil { 372 return err 373 } 374 if newLeader.Name == leader.Name { 375 return fmt.Errorf("leader pod name did not change across pod delete") 376 } 377 378 return nil 379 } 380 381 func verifyLeader(t *testing.T, namespace string, f *framework.Framework, labels map[string]string) (*v1.Pod, error) { 382 // get configmap, which is the lock 383 lockName := "memcached-operator-lock" 384 lock := v1.ConfigMap{} 385 err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { 386 err = f.Client.Get(context.TODO(), types.NamespacedName{Name: lockName, Namespace: namespace}, &lock) 387 if err != nil { 388 if apierrors.IsNotFound(err) { 389 t.Logf("Waiting for availability of leader lock configmap %s\n", lockName) 390 return false, nil 391 } 392 return false, err 393 } 394 return true, nil 395 }) 396 if err != nil { 397 return nil, fmt.Errorf("error getting leader lock configmap: %v\n", err) 398 } 399 t.Logf("Found leader lock configmap %s\n", lockName) 400 401 owners := lock.GetOwnerReferences() 402 if len(owners) != 1 { 403 return nil, fmt.Errorf("leader lock has %d owner refs, expected 1", len(owners)) 404 } 405 owner := owners[0] 406 407 // get operator pods 408 pods := v1.PodList{} 409 opts := client.ListOptions{Namespace: namespace} 410 for k, v := range labels { 411 if err := opts.SetLabelSelector(fmt.Sprintf("%s=%s", k, v)); err != nil { 412 return nil, fmt.Errorf("failed to set list label selector: (%v)", err) 413 } 414 } 415 if err := opts.SetFieldSelector("status.phase=Running"); err != nil { 416 t.Fatalf("Failed to set list field selector: (%v)", err) 417 } 418 err = f.Client.List(context.TODO(), &opts, &pods) 419 if err != nil { 420 return nil, err 421 } 422 if len(pods.Items) != 2 { 423 return nil, fmt.Errorf("expected 2 pods, found %d", len(pods.Items)) 424 } 425 426 // find and return the leader 427 for _, pod := range pods.Items { 428 if pod.Name == owner.Name { 429 return &pod, nil 430 } 431 } 432 return nil, fmt.Errorf("did not find operator pod that was referenced by configmap") 433 } 434 435 func memcachedScaleTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error { 436 // create example-memcached yaml file 437 filename := "deploy/cr.yaml" 438 err := ioutil.WriteFile(filename, 439 []byte(crYAML), 440 fileutil.DefaultFileMode) 441 if err != nil { 442 return err 443 } 444 445 // create memcached custom resource 446 framework.Global.NamespacedManPath = &filename 447 err = ctx.InitializeClusterResources(&framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) 448 if err != nil { 449 return err 450 } 451 t.Log("Created cr") 452 453 namespace, err := ctx.GetNamespace() 454 if err != nil { 455 return err 456 } 457 // wait for example-memcached to reach 3 replicas 458 err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 3, retryInterval, timeout) 459 if err != nil { 460 return err 461 } 462 463 // get fresh copy of memcached object as unstructured 464 obj := unstructured.Unstructured{} 465 jsonSpec, err := yaml.YAMLToJSON([]byte(crYAML)) 466 if err != nil { 467 return fmt.Errorf("could not convert yaml file to json: %v", err) 468 } 469 if err := obj.UnmarshalJSON(jsonSpec); err != nil { 470 t.Fatalf("Failed to unmarshal memcached CR: (%v)", err) 471 } 472 obj.SetNamespace(namespace) 473 err = f.Client.Get(context.TODO(), types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, &obj) 474 if err != nil { 475 return fmt.Errorf("failed to get memcached object: %s", err) 476 } 477 // update memcached CR size to 4 478 spec, ok := obj.Object["spec"].(map[string]interface{}) 479 if !ok { 480 return errors.New("memcached object missing spec field") 481 } 482 spec["size"] = 4 483 err = f.Client.Update(context.TODO(), &obj) 484 if err != nil { 485 return err 486 } 487 488 // wait for example-memcached to reach 4 replicas 489 return e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 4, retryInterval, timeout) 490 } 491 492 func MemcachedLocal(t *testing.T) { 493 // get global framework variables 494 ctx := framework.NewTestCtx(t) 495 defer ctx.Cleanup() 496 namespace, err := ctx.GetNamespace() 497 if err != nil { 498 t.Fatal(err) 499 } 500 cmd := exec.Command("operator-sdk", "up", "local", "--namespace="+namespace) 501 stderr, err := os.Create("stderr.txt") 502 if err != nil { 503 t.Fatalf("Failed to create stderr.txt: %v", err) 504 } 505 cmd.Stderr = stderr 506 defer func() { 507 if err := stderr.Close(); err != nil && !fileutil.IsClosedError(err) { 508 t.Errorf("Failed to close stderr: (%v)", err) 509 } 510 }() 511 512 err = cmd.Start() 513 if err != nil { 514 t.Fatalf("Error: %v", err) 515 } 516 ctx.AddCleanupFn(func() error { return cmd.Process.Signal(os.Interrupt) }) 517 518 // wait for operator to start (may take a minute to compile the command...) 519 err = wait.Poll(time.Second*5, time.Second*100, func() (done bool, err error) { 520 file, err := ioutil.ReadFile("stderr.txt") 521 if err != nil { 522 return false, err 523 } 524 if len(file) == 0 { 525 return false, nil 526 } 527 return true, nil 528 }) 529 if err != nil { 530 t.Fatalf("Local operator not ready after 100 seconds: %v\n", err) 531 } 532 533 if err = memcachedScaleTest(t, framework.Global, ctx); err != nil { 534 t.Fatal(err) 535 } 536 } 537 538 func MemcachedCluster(t *testing.T) { 539 // get global framework variables 540 ctx := framework.NewTestCtx(t) 541 defer ctx.Cleanup() 542 operatorYAML, err := ioutil.ReadFile("deploy/operator.yaml") 543 if err != nil { 544 t.Fatalf("Could not read deploy/operator.yaml: %v", err) 545 } 546 local := *e2eImageName == "" 547 if local { 548 *e2eImageName = "quay.io/example/memcached-operator:v0.0.1" 549 if err != nil { 550 t.Fatal(err) 551 } 552 operatorYAML = bytes.Replace(operatorYAML, []byte("imagePullPolicy: Always"), []byte("imagePullPolicy: Never"), 1) 553 err = ioutil.WriteFile("deploy/operator.yaml", operatorYAML, fileutil.DefaultFileMode) 554 if err != nil { 555 t.Fatal(err) 556 } 557 } 558 operatorYAML = bytes.Replace(operatorYAML, []byte("REPLACE_IMAGE"), []byte(*e2eImageName), 1) 559 err = ioutil.WriteFile("deploy/operator.yaml", operatorYAML, os.FileMode(0644)) 560 if err != nil { 561 t.Fatalf("Failed to write deploy/operator.yaml: %v", err) 562 } 563 t.Log("Building operator docker image") 564 cmdOut, err := exec.Command("operator-sdk", "build", *e2eImageName).CombinedOutput() 565 if err != nil { 566 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 567 } 568 569 if !local { 570 t.Log("Pushing docker image to repo") 571 cmdOut, err = exec.Command("docker", "push", *e2eImageName).CombinedOutput() 572 if err != nil { 573 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 574 } 575 } 576 577 file, err := yamlutil.GenerateCombinedNamespacedManifest(scaffold.DeployDir) 578 if err != nil { 579 t.Fatal(err) 580 } 581 ctx.AddCleanupFn(func() error { return os.Remove(file.Name()) }) 582 583 // create namespaced resources 584 filename := file.Name() 585 framework.Global.NamespacedManPath = &filename 586 err = ctx.InitializeClusterResources(&framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) 587 if err != nil { 588 t.Fatal(err) 589 } 590 t.Log("Created namespaced resources") 591 592 namespace, err := ctx.GetNamespace() 593 if err != nil { 594 t.Fatal(err) 595 } 596 // wait for memcached-operator to be ready 597 err = e2eutil.WaitForOperatorDeployment(t, framework.Global.KubeClient, namespace, operatorName, 2, retryInterval, timeout) 598 if err != nil { 599 t.Fatal(err) 600 } 601 602 if err = memcachedLeaderTest(t, framework.Global, ctx); err != nil { 603 t.Fatal(err) 604 } 605 606 if err = memcachedScaleTest(t, framework.Global, ctx); err != nil { 607 t.Fatal(err) 608 } 609 610 if err = memcachedMetricsTest(t, framework.Global, ctx); err != nil { 611 t.Fatal(err) 612 } 613 } 614 615 func memcachedMetricsTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error { 616 namespace, err := ctx.GetNamespace() 617 if err != nil { 618 return err 619 } 620 621 // Make sure metrics Service exists 622 s := v1.Service{} 623 err = f.Client.Get(context.TODO(), types.NamespacedName{Name: operatorName, Namespace: namespace}, &s) 624 if err != nil { 625 return fmt.Errorf("could not get metrics Service: (%v)", err) 626 } 627 628 // Get operator pod 629 pods := v1.PodList{} 630 opts := client.InNamespace(namespace) 631 if len(s.Spec.Selector) == 0 { 632 return fmt.Errorf("no labels found in metrics Service") 633 } 634 635 for k, v := range s.Spec.Selector { 636 if err := opts.SetLabelSelector(fmt.Sprintf("%s=%s", k, v)); err != nil { 637 return fmt.Errorf("failed to set list label selector: (%v)", err) 638 } 639 } 640 641 if err := opts.SetFieldSelector("status.phase=Running"); err != nil { 642 return fmt.Errorf("failed to set list field selector: (%v)", err) 643 } 644 err = f.Client.List(context.TODO(), opts, &pods) 645 if err != nil { 646 return fmt.Errorf("failed to get pods: (%v)", err) 647 } 648 649 podName := "" 650 numPods := len(pods.Items) 651 // TODO(lili): Remove below logic when we enable exposing metrics in all pods. 652 if numPods == 0 { 653 podName = pods.Items[0].Name 654 } else if numPods > 1 { 655 // If we got more than one pod, get leader pod name. 656 leader, err := verifyLeader(t, namespace, f, s.Spec.Selector) 657 if err != nil { 658 return err 659 } 660 podName = leader.Name 661 } else { 662 return fmt.Errorf("failed to get operator pod: could not select any pods with Service selector %v", s.Spec.Selector) 663 } 664 // Pod name must be there, otherwise we cannot read metrics data via pod proxy. 665 if podName == "" { 666 return fmt.Errorf("failed to get pod name") 667 } 668 669 // Get metrics data 670 request := proxyViaPod(f.KubeClient, namespace, podName, "8383", "/metrics") 671 response, err := request.DoRaw() 672 if err != nil { 673 return fmt.Errorf("failed to get response from metrics: %v", err) 674 } 675 676 // Make sure metrics are present 677 if len(response) == 0 { 678 return fmt.Errorf("metrics body is empty") 679 } 680 681 // Perform prometheus metrics lint checks 682 l := promlint.New(bytes.NewReader(response)) 683 problems, err := l.Lint() 684 if err != nil { 685 return fmt.Errorf("failed to lint metrics: %v", err) 686 } 687 // TODO(lili): Change to 0, when we upgrade to 1.14. 688 // currently there is a problem with one of the metrics in upstream Kubernetes: 689 // `workqueue_longest_running_processor_microseconds`. 690 // This has been fixed in 1.14 release. 691 if len(problems) > 1 { 692 return fmt.Errorf("found problems with metrics: %#+v", problems) 693 } 694 695 return nil 696 } 697 698 func proxyViaPod(kubeClient kubernetes.Interface, namespace, podName, podPortName, path string) *rest.Request { 699 return kubeClient. 700 CoreV1(). 701 RESTClient(). 702 Get(). 703 Namespace(namespace). 704 Resource("pods"). 705 SubResource("proxy"). 706 Name(fmt.Sprintf("%s:%s", podName, podPortName)). 707 Suffix(path) 708 }