github.com/kinvolk/operator-sdk@v0.8.1/test/e2e/memcached_test.go (about) 1 // Copyright 2018 The Operator-SDK Authors 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package e2e 16 17 import ( 18 "bytes" 19 "context" 20 "errors" 21 "fmt" 22 "io/ioutil" 23 "os" 24 "os/exec" 25 "path/filepath" 26 "strings" 27 "testing" 28 "time" 29 30 "github.com/operator-framework/operator-sdk/internal/pkg/scaffold" 31 "github.com/operator-framework/operator-sdk/internal/util/fileutil" 32 "github.com/operator-framework/operator-sdk/internal/util/projutil" 33 "github.com/operator-framework/operator-sdk/internal/util/yamlutil" 34 framework "github.com/operator-framework/operator-sdk/pkg/test" 35 "github.com/operator-framework/operator-sdk/pkg/test/e2eutil" 36 37 "github.com/ghodss/yaml" 38 "github.com/prometheus/prometheus/util/promlint" 39 "github.com/rogpeppe/go-internal/modfile" 40 v1 "k8s.io/api/core/v1" 41 apierrors "k8s.io/apimachinery/pkg/api/errors" 42 "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" 43 "k8s.io/apimachinery/pkg/types" 44 "k8s.io/apimachinery/pkg/util/wait" 45 "k8s.io/client-go/kubernetes" 46 "k8s.io/client-go/rest" 47 "sigs.k8s.io/controller-runtime/pkg/client" 48 ) 49 50 const ( 51 crYAML string = "apiVersion: \"cache.example.com/v1alpha1\"\nkind: \"Memcached\"\nmetadata:\n name: \"example-memcached\"\nspec:\n size: 3" 52 retryInterval = time.Second * 5 53 timeout = time.Second * 120 54 cleanupRetryInterval = time.Second * 1 55 cleanupTimeout = time.Second * 10 56 operatorName = "memcached-operator" 57 ) 58 59 func TestMemcached(t *testing.T) { 60 // get global framework variables 61 ctx := framework.NewTestCtx(t) 62 defer ctx.Cleanup() 63 gopath, ok := os.LookupEnv(projutil.GoPathEnv) 64 if !ok || gopath == "" { 65 t.Fatalf("$GOPATH not set") 66 } 67 cd, err := os.Getwd() 68 if err != nil { 69 t.Fatal(err) 70 } 71 defer func() { 72 if err := os.Chdir(cd); err != nil { 73 t.Errorf("Failed to change back to original working directory: (%v)", err) 74 } 75 }() 76 // For go commands in operator projects. 77 if err = os.Setenv("GO111MODULE", "on"); err != nil { 78 t.Fatal(err) 79 } 80 81 // Setup 82 absProjectPath, err := ioutil.TempDir(filepath.Join(gopath, "src"), "tmp.") 83 if err != nil { 84 t.Fatal(err) 85 } 86 ctx.AddCleanupFn(func() error { return os.RemoveAll(absProjectPath) }) 87 88 if err := os.MkdirAll(absProjectPath, fileutil.DefaultDirFileMode); err != nil { 89 t.Fatal(err) 90 } 91 if err := os.Chdir(absProjectPath); err != nil { 92 t.Fatal(err) 93 } 94 95 t.Log("Creating new operator project") 96 cmdOut, err := exec.Command("operator-sdk", 97 "new", 98 operatorName).CombinedOutput() 99 if err != nil { 100 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 101 } 102 103 if err := os.Chdir(operatorName); err != nil { 104 t.Fatalf("Failed to change to %s directory: (%v)", operatorName, err) 105 } 106 107 sdkRepo := "github.com/operator-framework/operator-sdk" 108 localSDKPath := filepath.Join(gopath, "src", sdkRepo) 109 110 replace := getGoModReplace(t, localSDKPath) 111 if replace.repo != sdkRepo { 112 if replace.isLocal { 113 // A hacky way to get local module substitution to work is to write a 114 // stub go.mod into the local SDK repo referred to in 115 // memcached-operator's go.mod, which allows go to recognize 116 // the local SDK repo as a module. 117 sdkModPath := filepath.Join(replace.repo, "go.mod") 118 err = ioutil.WriteFile(sdkModPath, []byte("module "+sdkRepo), fileutil.DefaultFileMode) 119 if err != nil { 120 t.Fatalf("Failed to write main repo go.mod file: %v", err) 121 } 122 defer func() { 123 if err = os.RemoveAll(sdkModPath); err != nil { 124 t.Fatalf("Failed to remove %s: %v", sdkModPath, err) 125 } 126 }() 127 } 128 writeGoModReplace(t, sdkRepo, replace.repo, replace.ref) 129 } 130 131 cmdOut, err = exec.Command("go", "mod", "vendor").CombinedOutput() 132 if err != nil { 133 t.Fatalf("Error after modifying go.mod: %v\nCommand Output: %s\n", err, string(cmdOut)) 134 } 135 136 // Set replicas to 2 to test leader election. In production, this should 137 // almost always be set to 1, because there isn't generally value in having 138 // a hot spare operator process. 139 opYaml, err := ioutil.ReadFile("deploy/operator.yaml") 140 if err != nil { 141 t.Fatalf("Could not read deploy/operator.yaml: %v", err) 142 } 143 newOpYaml := bytes.Replace(opYaml, []byte("replicas: 1"), []byte("replicas: 2"), 1) 144 err = ioutil.WriteFile("deploy/operator.yaml", newOpYaml, 0644) 145 if err != nil { 146 t.Fatalf("Could not write deploy/operator.yaml: %v", err) 147 } 148 149 cmd := exec.Command("operator-sdk", 150 "add", 151 "api", 152 "--api-version=cache.example.com/v1alpha1", 153 "--kind=Memcached") 154 cmd.Env = os.Environ() 155 cmdOut, err = cmd.CombinedOutput() 156 if err != nil { 157 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 158 } 159 cmdOut, err = exec.Command("operator-sdk", 160 "add", 161 "controller", 162 "--api-version=cache.example.com/v1alpha1", 163 "--kind=Memcached").CombinedOutput() 164 if err != nil { 165 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 166 } 167 168 tmplFiles := map[string]string{ 169 filepath.Join(localSDKPath, "example/memcached-operator/memcached_controller.go.tmpl"): "pkg/controller/memcached/memcached_controller.go", 170 filepath.Join(localSDKPath, "test/e2e/incluster-test-code/main_test.go.tmpl"): "test/e2e/main_test.go", 171 filepath.Join(localSDKPath, "test/e2e/incluster-test-code/memcached_test.go.tmpl"): "test/e2e/memcached_test.go", 172 } 173 for src, dst := range tmplFiles { 174 if err := os.MkdirAll(filepath.Dir(dst), fileutil.DefaultDirFileMode); err != nil { 175 t.Fatalf("Could not create template destination directory: %s", err) 176 } 177 srcTmpl, err := ioutil.ReadFile(src) 178 if err != nil { 179 t.Fatalf("Could not read template from %s: %s", src, err) 180 } 181 dstData := strings.Replace(string(srcTmpl), "github.com/example-inc", filepath.Base(absProjectPath), -1) 182 if err := ioutil.WriteFile(dst, []byte(dstData), fileutil.DefaultFileMode); err != nil { 183 t.Fatalf("Could not write template output to %s: %s", dst, err) 184 } 185 } 186 187 memcachedTypesFile, err := ioutil.ReadFile("pkg/apis/cache/v1alpha1/memcached_types.go") 188 if err != nil { 189 t.Fatal(err) 190 } 191 memcachedTypesFileLines := bytes.Split(memcachedTypesFile, []byte("\n")) 192 for lineNum, line := range memcachedTypesFileLines { 193 if strings.Contains(string(line), "type MemcachedSpec struct {") { 194 memcachedTypesFileLinesIntermediate := append(memcachedTypesFileLines[:lineNum+1], []byte("\tSize int32 `json:\"size\"`")) 195 memcachedTypesFileLines = append(memcachedTypesFileLinesIntermediate, memcachedTypesFileLines[lineNum+3:]...) 196 break 197 } 198 } 199 for lineNum, line := range memcachedTypesFileLines { 200 if strings.Contains(string(line), "type MemcachedStatus struct {") { 201 memcachedTypesFileLinesIntermediate := append(memcachedTypesFileLines[:lineNum+1], []byte("\tNodes []string `json:\"nodes\"`")) 202 memcachedTypesFileLines = append(memcachedTypesFileLinesIntermediate, memcachedTypesFileLines[lineNum+3:]...) 203 break 204 } 205 } 206 if err := os.Remove("pkg/apis/cache/v1alpha1/memcached_types.go"); err != nil { 207 t.Fatalf("Failed to remove old memcached_type.go file: (%v)", err) 208 } 209 err = ioutil.WriteFile("pkg/apis/cache/v1alpha1/memcached_types.go", bytes.Join(memcachedTypesFileLines, []byte("\n")), fileutil.DefaultFileMode) 210 if err != nil { 211 t.Fatal(err) 212 } 213 214 t.Log("Generating k8s") 215 cmdOut, err = exec.Command("operator-sdk", "generate", "k8s").CombinedOutput() 216 if err != nil { 217 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 218 } 219 220 t.Log("Pulling new dependencies with go mod") 221 cmdOut, err = exec.Command("go", "mod", "vendor").CombinedOutput() 222 if err != nil { 223 t.Fatalf("Command 'go mod vendor' failed: %v\nCommand Output:\n%v", err, string(cmdOut)) 224 } 225 226 file, err := yamlutil.GenerateCombinedGlobalManifest(scaffold.CRDsDir) 227 if err != nil { 228 t.Fatal(err) 229 } 230 ctx.AddCleanupFn(func() error { return os.Remove(file.Name()) }) 231 232 // hacky way to use createFromYAML without exposing the method 233 // create crd 234 filename := file.Name() 235 framework.Global.NamespacedManPath = &filename 236 err = ctx.InitializeClusterResources(&framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) 237 if err != nil { 238 t.Fatal(err) 239 } 240 t.Log("Created global resources") 241 242 // run subtests 243 t.Run("memcached-group", func(t *testing.T) { 244 t.Run("Cluster", MemcachedCluster) 245 t.Run("Local", MemcachedLocal) 246 }) 247 } 248 249 type goModReplace struct { 250 repo string 251 ref string 252 isLocal bool 253 } 254 255 // getGoModReplace returns a go.mod replacement that is appropriate based on the build's 256 // environment to support PR, fork/branch, and local builds. 257 // 258 // PR: 259 // 1. Activate when TRAVIS_PULL_REQUEST_SLUG and TRAVIS_PULL_REQUEST_SHA are set 260 // 2. Modify go.mod to replace osdk import with github.com/${TRAVIS_PULL_REQUEST_SLUG} ${TRAVIS_PULL_REQUEST_SHA} 261 // 262 // Fork/branch: 263 // 1. Activate when TRAVIS_REPO_SLUG and TRAVIS_COMMIT are set 264 // 2. Modify go.mod to replace osdk import with github.com/${TRAVIS_REPO_SLUG} ${TRAVIS_COMMIT} 265 // 266 // Local: 267 // 1. Activate when none of the above TRAVIS_* variables are set. 268 // 2. Modify go.mod to replace osdk import with local filesystem path. 269 // 270 func getGoModReplace(t *testing.T, localSDKPath string) goModReplace { 271 // PR environment 272 prSlug, prSlugOk := os.LookupEnv("TRAVIS_PULL_REQUEST_SLUG") 273 prSha, prShaOk := os.LookupEnv("TRAVIS_PULL_REQUEST_SHA") 274 if prSlugOk && prSlug != "" && prShaOk && prSha != "" { 275 return goModReplace{ 276 repo: fmt.Sprintf("github.com/%s", prSlug), 277 ref: prSha, 278 } 279 } 280 281 // Fork/branch environment 282 slug, slugOk := os.LookupEnv("TRAVIS_REPO_SLUG") 283 sha, shaOk := os.LookupEnv("TRAVIS_COMMIT") 284 if slugOk && slug != "" && shaOk && sha != "" { 285 return goModReplace{ 286 repo: fmt.Sprintf("github.com/%s", slug), 287 ref: sha, 288 } 289 } 290 291 // If neither of the above cases is applicable, but one of the TRAVIS_* 292 // variables is nonetheless set, something unexpected is going on. Log 293 // the vars and exit. 294 if prSlugOk || prShaOk || slugOk || shaOk { 295 t.Logf("TRAVIS_PULL_REQUEST_SLUG='%s', set: %t", prSlug, prSlugOk) 296 t.Logf("TRAVIS_PULL_REQUEST_SHA='%s', set: %t", prSha, prShaOk) 297 t.Logf("TRAVIS_REPO_SLUG='%s', set: %t", slug, slugOk) 298 t.Logf("TRAVIS_COMMIT='%s', set: %t", sha, shaOk) 299 t.Fatal("Invalid travis environment") 300 } 301 302 // Local environment 303 return goModReplace{ 304 repo: localSDKPath, 305 isLocal: true, 306 } 307 } 308 309 func writeGoModReplace(t *testing.T, repo, path, sha string) { 310 modBytes, err := ioutil.ReadFile("go.mod") 311 if err != nil { 312 t.Fatalf("Failed to read go.mod: %v", err) 313 } 314 modFile, err := modfile.Parse("go.mod", modBytes, nil) 315 if err != nil { 316 t.Fatalf("Failed to parse go.mod: %v", err) 317 } 318 if err = modFile.AddReplace(repo, "", path, sha); err != nil { 319 s := "" 320 if sha != "" { 321 s = " " + sha 322 } 323 t.Fatalf(`Failed to add "replace %s => %s%s: %v"`, repo, path, s, err) 324 } 325 if modBytes, err = modFile.Format(); err != nil { 326 t.Fatalf("Failed to format go.mod: %v", err) 327 } 328 err = ioutil.WriteFile("go.mod", modBytes, fileutil.DefaultFileMode) 329 if err != nil { 330 t.Fatalf("Failed to write updated go.mod: %v", err) 331 } 332 t.Logf("go.mod: %v", string(modBytes)) 333 } 334 335 func memcachedLeaderTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error { 336 namespace, err := ctx.GetNamespace() 337 if err != nil { 338 return err 339 } 340 341 err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, operatorName, 2, retryInterval, timeout) 342 if err != nil { 343 return err 344 } 345 346 label := map[string]string{"name": operatorName} 347 348 leader, err := verifyLeader(t, namespace, f, label) 349 if err != nil { 350 return err 351 } 352 353 // delete the leader's pod so a new leader will get elected 354 err = f.Client.Delete(context.TODO(), leader) 355 if err != nil { 356 return err 357 } 358 359 err = e2eutil.WaitForDeletion(t, f.Client.Client, leader, retryInterval, timeout) 360 if err != nil { 361 return err 362 } 363 364 err = e2eutil.WaitForOperatorDeployment(t, f.KubeClient, namespace, operatorName, 2, retryInterval, timeout) 365 if err != nil { 366 return err 367 } 368 369 newLeader, err := verifyLeader(t, namespace, f, label) 370 if err != nil { 371 return err 372 } 373 if newLeader.Name == leader.Name { 374 return fmt.Errorf("leader pod name did not change across pod delete") 375 } 376 377 return nil 378 } 379 380 func verifyLeader(t *testing.T, namespace string, f *framework.Framework, labels map[string]string) (*v1.Pod, error) { 381 // get configmap, which is the lock 382 lockName := "memcached-operator-lock" 383 lock := v1.ConfigMap{} 384 err := wait.Poll(retryInterval, timeout, func() (done bool, err error) { 385 err = f.Client.Get(context.TODO(), types.NamespacedName{Name: lockName, Namespace: namespace}, &lock) 386 if err != nil { 387 if apierrors.IsNotFound(err) { 388 t.Logf("Waiting for availability of leader lock configmap %s\n", lockName) 389 return false, nil 390 } 391 return false, err 392 } 393 return true, nil 394 }) 395 if err != nil { 396 return nil, fmt.Errorf("error getting leader lock configmap: %v\n", err) 397 } 398 t.Logf("Found leader lock configmap %s\n", lockName) 399 400 owners := lock.GetOwnerReferences() 401 if len(owners) != 1 { 402 return nil, fmt.Errorf("leader lock has %d owner refs, expected 1", len(owners)) 403 } 404 owner := owners[0] 405 406 // get operator pods 407 pods := v1.PodList{} 408 opts := client.ListOptions{Namespace: namespace} 409 for k, v := range labels { 410 if err := opts.SetLabelSelector(fmt.Sprintf("%s=%s", k, v)); err != nil { 411 return nil, fmt.Errorf("failed to set list label selector: (%v)", err) 412 } 413 } 414 if err := opts.SetFieldSelector("status.phase=Running"); err != nil { 415 t.Fatalf("Failed to set list field selector: (%v)", err) 416 } 417 err = f.Client.List(context.TODO(), &opts, &pods) 418 if err != nil { 419 return nil, err 420 } 421 if len(pods.Items) != 2 { 422 return nil, fmt.Errorf("expected 2 pods, found %d", len(pods.Items)) 423 } 424 425 // find and return the leader 426 for _, pod := range pods.Items { 427 if pod.Name == owner.Name { 428 return &pod, nil 429 } 430 } 431 return nil, fmt.Errorf("did not find operator pod that was referenced by configmap") 432 } 433 434 func memcachedScaleTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error { 435 // create example-memcached yaml file 436 filename := "deploy/cr.yaml" 437 err := ioutil.WriteFile(filename, 438 []byte(crYAML), 439 fileutil.DefaultFileMode) 440 if err != nil { 441 return err 442 } 443 444 // create memcached custom resource 445 framework.Global.NamespacedManPath = &filename 446 err = ctx.InitializeClusterResources(&framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) 447 if err != nil { 448 return err 449 } 450 t.Log("Created cr") 451 452 namespace, err := ctx.GetNamespace() 453 if err != nil { 454 return err 455 } 456 // wait for example-memcached to reach 3 replicas 457 err = e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 3, retryInterval, timeout) 458 if err != nil { 459 return err 460 } 461 462 // get fresh copy of memcached object as unstructured 463 obj := unstructured.Unstructured{} 464 jsonSpec, err := yaml.YAMLToJSON([]byte(crYAML)) 465 if err != nil { 466 return fmt.Errorf("could not convert yaml file to json: %v", err) 467 } 468 if err := obj.UnmarshalJSON(jsonSpec); err != nil { 469 t.Fatalf("Failed to unmarshal memcached CR: (%v)", err) 470 } 471 obj.SetNamespace(namespace) 472 err = f.Client.Get(context.TODO(), types.NamespacedName{Name: obj.GetName(), Namespace: obj.GetNamespace()}, &obj) 473 if err != nil { 474 return fmt.Errorf("failed to get memcached object: %s", err) 475 } 476 // update memcached CR size to 4 477 spec, ok := obj.Object["spec"].(map[string]interface{}) 478 if !ok { 479 return errors.New("memcached object missing spec field") 480 } 481 spec["size"] = 4 482 err = f.Client.Update(context.TODO(), &obj) 483 if err != nil { 484 return err 485 } 486 487 // wait for example-memcached to reach 4 replicas 488 return e2eutil.WaitForDeployment(t, f.KubeClient, namespace, "example-memcached", 4, retryInterval, timeout) 489 } 490 491 func MemcachedLocal(t *testing.T) { 492 // get global framework variables 493 ctx := framework.NewTestCtx(t) 494 defer ctx.Cleanup() 495 namespace, err := ctx.GetNamespace() 496 if err != nil { 497 t.Fatal(err) 498 } 499 cmd := exec.Command("operator-sdk", "up", "local", "--namespace="+namespace) 500 stderr, err := os.Create("stderr.txt") 501 if err != nil { 502 t.Fatalf("Failed to create stderr.txt: %v", err) 503 } 504 cmd.Stderr = stderr 505 defer func() { 506 if err := stderr.Close(); err != nil && !fileutil.IsClosedError(err) { 507 t.Errorf("Failed to close stderr: (%v)", err) 508 } 509 }() 510 511 err = cmd.Start() 512 if err != nil { 513 t.Fatalf("Error: %v", err) 514 } 515 ctx.AddCleanupFn(func() error { return cmd.Process.Signal(os.Interrupt) }) 516 517 // wait for operator to start (may take a minute to compile the command...) 518 err = wait.Poll(time.Second*5, time.Second*100, func() (done bool, err error) { 519 file, err := ioutil.ReadFile("stderr.txt") 520 if err != nil { 521 return false, err 522 } 523 if len(file) == 0 { 524 return false, nil 525 } 526 return true, nil 527 }) 528 if err != nil { 529 t.Fatalf("Local operator not ready after 100 seconds: %v\n", err) 530 } 531 532 if err = memcachedScaleTest(t, framework.Global, ctx); err != nil { 533 t.Fatal(err) 534 } 535 } 536 537 func MemcachedCluster(t *testing.T) { 538 // get global framework variables 539 ctx := framework.NewTestCtx(t) 540 defer ctx.Cleanup() 541 operatorYAML, err := ioutil.ReadFile("deploy/operator.yaml") 542 if err != nil { 543 t.Fatalf("Could not read deploy/operator.yaml: %v", err) 544 } 545 local := *e2eImageName == "" 546 if local { 547 *e2eImageName = "quay.io/example/memcached-operator:v0.0.1" 548 if err != nil { 549 t.Fatal(err) 550 } 551 operatorYAML = bytes.Replace(operatorYAML, []byte("imagePullPolicy: Always"), []byte("imagePullPolicy: Never"), 1) 552 err = ioutil.WriteFile("deploy/operator.yaml", operatorYAML, fileutil.DefaultFileMode) 553 if err != nil { 554 t.Fatal(err) 555 } 556 } 557 operatorYAML = bytes.Replace(operatorYAML, []byte("REPLACE_IMAGE"), []byte(*e2eImageName), 1) 558 err = ioutil.WriteFile("deploy/operator.yaml", operatorYAML, os.FileMode(0644)) 559 if err != nil { 560 t.Fatalf("Failed to write deploy/operator.yaml: %v", err) 561 } 562 t.Log("Building operator docker image") 563 cmdOut, err := exec.Command("operator-sdk", "build", *e2eImageName).CombinedOutput() 564 if err != nil { 565 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 566 } 567 568 if !local { 569 t.Log("Pushing docker image to repo") 570 cmdOut, err = exec.Command("docker", "push", *e2eImageName).CombinedOutput() 571 if err != nil { 572 t.Fatalf("Error: %v\nCommand Output: %s\n", err, string(cmdOut)) 573 } 574 } 575 576 file, err := yamlutil.GenerateCombinedNamespacedManifest(scaffold.DeployDir) 577 if err != nil { 578 t.Fatal(err) 579 } 580 ctx.AddCleanupFn(func() error { return os.Remove(file.Name()) }) 581 582 // create namespaced resources 583 filename := file.Name() 584 framework.Global.NamespacedManPath = &filename 585 err = ctx.InitializeClusterResources(&framework.CleanupOptions{TestContext: ctx, Timeout: cleanupTimeout, RetryInterval: cleanupRetryInterval}) 586 if err != nil { 587 t.Fatal(err) 588 } 589 t.Log("Created namespaced resources") 590 591 namespace, err := ctx.GetNamespace() 592 if err != nil { 593 t.Fatal(err) 594 } 595 // wait for memcached-operator to be ready 596 err = e2eutil.WaitForOperatorDeployment(t, framework.Global.KubeClient, namespace, operatorName, 2, retryInterval, timeout) 597 if err != nil { 598 t.Fatal(err) 599 } 600 601 if err = memcachedLeaderTest(t, framework.Global, ctx); err != nil { 602 t.Fatal(err) 603 } 604 605 if err = memcachedScaleTest(t, framework.Global, ctx); err != nil { 606 t.Fatal(err) 607 } 608 609 if err = memcachedMetricsTest(t, framework.Global, ctx); err != nil { 610 t.Fatal(err) 611 } 612 } 613 614 func memcachedMetricsTest(t *testing.T, f *framework.Framework, ctx *framework.TestCtx) error { 615 namespace, err := ctx.GetNamespace() 616 if err != nil { 617 return err 618 } 619 620 // Make sure metrics Service exists 621 s := v1.Service{} 622 err = f.Client.Get(context.TODO(), types.NamespacedName{Name: operatorName, Namespace: namespace}, &s) 623 if err != nil { 624 return fmt.Errorf("could not get metrics Service: (%v)", err) 625 } 626 627 // Get operator pod 628 pods := v1.PodList{} 629 opts := client.InNamespace(namespace) 630 if len(s.Spec.Selector) == 0 { 631 return fmt.Errorf("no labels found in metrics Service") 632 } 633 634 for k, v := range s.Spec.Selector { 635 if err := opts.SetLabelSelector(fmt.Sprintf("%s=%s", k, v)); err != nil { 636 return fmt.Errorf("failed to set list label selector: (%v)", err) 637 } 638 } 639 640 if err := opts.SetFieldSelector("status.phase=Running"); err != nil { 641 return fmt.Errorf("failed to set list field selector: (%v)", err) 642 } 643 err = f.Client.List(context.TODO(), opts, &pods) 644 if err != nil { 645 return fmt.Errorf("failed to get pods: (%v)", err) 646 } 647 648 podName := "" 649 numPods := len(pods.Items) 650 // TODO(lili): Remove below logic when we enable exposing metrics in all pods. 651 if numPods == 0 { 652 podName = pods.Items[0].Name 653 } else if numPods > 1 { 654 // If we got more than one pod, get leader pod name. 655 leader, err := verifyLeader(t, namespace, f, s.Spec.Selector) 656 if err != nil { 657 return err 658 } 659 podName = leader.Name 660 } else { 661 return fmt.Errorf("failed to get operator pod: could not select any pods with Service selector %v", s.Spec.Selector) 662 } 663 // Pod name must be there, otherwise we cannot read metrics data via pod proxy. 664 if podName == "" { 665 return fmt.Errorf("failed to get pod name") 666 } 667 668 // Get metrics data 669 request := proxyViaPod(f.KubeClient, namespace, podName, "8383", "/metrics") 670 response, err := request.DoRaw() 671 if err != nil { 672 return fmt.Errorf("failed to get response from metrics: %v", err) 673 } 674 675 // Make sure metrics are present 676 if len(response) == 0 { 677 return fmt.Errorf("metrics body is empty") 678 } 679 680 // Perform prometheus metrics lint checks 681 l := promlint.New(bytes.NewReader(response)) 682 problems, err := l.Lint() 683 if err != nil { 684 return fmt.Errorf("failed to lint metrics: %v", err) 685 } 686 // TODO(lili): Change to 0, when we upgrade to 1.14. 687 // currently there is a problem with one of the metrics in upstream Kubernetes: 688 // `workqueue_longest_running_processor_microseconds`. 689 // This has been fixed in 1.14 release. 690 if len(problems) > 1 { 691 return fmt.Errorf("found problems with metrics: %#+v", problems) 692 } 693 694 return nil 695 } 696 697 func proxyViaPod(kubeClient kubernetes.Interface, namespace, podName, podPortName, path string) *rest.Request { 698 return kubeClient. 699 CoreV1(). 700 RESTClient(). 701 Get(). 702 Namespace(namespace). 703 Resource("pods"). 704 SubResource("proxy"). 705 Name(fmt.Sprintf("%s:%s", podName, podPortName)). 706 Suffix(path) 707 }