github.com/verrazzano/verrazzano@v1.7.0/application-operator/controllers/webhooks/verrazzanoproject_webhook_test.go (about) 1 // Copyright (c) 2021, 2022, Oracle and/or its affiliates. 2 // Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl. 3 4 package webhooks 5 6 import ( 7 "context" 8 "fmt" 9 "testing" 10 11 "github.com/prometheus/client_golang/prometheus/testutil" 12 "github.com/stretchr/testify/assert" 13 v1alpha12 "github.com/verrazzano/verrazzano/application-operator/apis/clusters/v1alpha1" 14 "github.com/verrazzano/verrazzano/application-operator/constants" 15 "github.com/verrazzano/verrazzano/application-operator/metricsexporter" 16 "github.com/verrazzano/verrazzano/cluster-operator/apis/clusters/v1alpha1" 17 admissionv1 "k8s.io/api/admission/v1" 18 corev1 "k8s.io/api/core/v1" 19 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 20 "sigs.k8s.io/controller-runtime/pkg/client/fake" 21 "sigs.k8s.io/controller-runtime/pkg/webhook/admission" 22 ) 23 24 // newVerrazzanoProjectValidator creates a new VerrazzanoProjectValidator 25 func newVerrazzanoProjectValidator() VerrazzanoProjectValidator { 26 scheme := newScheme() 27 decoder, _ := admission.NewDecoder(scheme) 28 cli := fake.NewClientBuilder().WithScheme(scheme).Build() 29 v := VerrazzanoProjectValidator{client: cli, decoder: decoder} 30 return v 31 } 32 33 // TestVerrazzanoProject tests the validation of VerrazzanoProject resource 34 // GIVEN a call validate VerrazzanoProject on create or update 35 // WHEN the VerrazzanoProject is properly formed 36 // THEN the validation should succeed 37 func TestVerrazzanoProject(t *testing.T) { 38 39 asrt := assert.New(t) 40 v := newVerrazzanoProjectValidator() 41 42 // Test data 43 testVP := testProject 44 testMC := testManagedCluster 45 asrt.NoError(v.client.Create(context.TODO(), &testMC)) 46 47 req := newAdmissionRequest(admissionv1.Create, testVP) 48 res := v.Handle(context.TODO(), req) 49 asrt.True(res.Allowed, "Expected project validation to succeed.") 50 51 req = newAdmissionRequest(admissionv1.Update, testVP) 52 res = v.Handle(context.TODO(), req) 53 asrt.True(res.Allowed, "Expected project validation to succeed.") 54 } 55 56 // TestInvalidNamespace tests the validation of VerrazzanoProject resource 57 // GIVEN a call validate VerrazzanoProject on create or update 58 // WHEN the VerrazzanoProject contains an invalid namespace 59 // THEN the validation should fail 60 func TestInvalidNamespace(t *testing.T) { 61 62 asrt := assert.New(t) 63 v := newVerrazzanoProjectValidator() 64 65 // Test data 66 testVP := testProject 67 testVP.Namespace = "invalid-namespace" 68 testMC := testManagedCluster 69 asrt.NoError(v.client.Create(context.TODO(), &testMC)) 70 71 // Test create 72 req := newAdmissionRequest(admissionv1.Create, testVP) 73 res := v.Handle(context.TODO(), req) 74 asrt.False(res.Allowed, "Expected project create validation to fail.") 75 asrt.Containsf(res.Result.Reason, fmt.Sprintf("resource must be %q", constants.VerrazzanoMultiClusterNamespace), "unexpected failure string") 76 77 // Test update 78 req = newAdmissionRequest(admissionv1.Update, testVP) 79 res = v.Handle(context.TODO(), req) 80 asrt.False(res.Allowed, "Expected project update validation to fail.") 81 asrt.Containsf(res.Result.Reason, fmt.Sprintf("resource must be %q", constants.VerrazzanoMultiClusterNamespace), "unexpected failure string") 82 } 83 84 // TestInvalidNamespaces tests the validation of VerrazzanoProject resource 85 // GIVEN a call validate VerrazzanoProject on create or update 86 // WHEN the VerrazzanoProject contains an invalid namespace list 87 // THEN the validation should fail 88 func TestInvalidNamespaces(t *testing.T) { 89 90 asrt := assert.New(t) 91 v := newVerrazzanoProjectValidator() 92 93 // Test data 94 testVP := testProject 95 testVP.Spec.Template.Namespaces = []v1alpha12.NamespaceTemplate{} 96 testMC := testManagedCluster 97 asrt.NoError(v.client.Create(context.TODO(), &testMC)) 98 99 // Test create 100 req := newAdmissionRequest(admissionv1.Create, testVP) 101 res := v.Handle(context.TODO(), req) 102 asrt.False(res.Allowed, "Expected project create validation to fail for invalid namespace list") 103 asrt.Containsf(res.Result.Reason, "One or more namespaces must be provided", "unexpected failure string") 104 105 // Test update 106 req = newAdmissionRequest(admissionv1.Update, testVP) 107 res = v.Handle(context.TODO(), req) 108 asrt.False(res.Allowed, "Expected project create validation to fail for invalid namespace list") 109 asrt.Containsf(res.Result.Reason, "One or more namespaces must be provided", "unexpected failure string") 110 } 111 112 // TestNetworkPolicyNamespace tests the validation of VerrazzanoProject NetworkPolicyTemplate 113 // GIVEN a call validate VerrazzanoProject on create or update 114 // WHEN the VerrazzanoProject has a NetworkPolicyTemplate with a namespace that exists in the project 115 // THEN the validation should succeed 116 func TestNetworkPolicyNamespace(t *testing.T) { 117 118 asrt := assert.New(t) 119 v := newVerrazzanoProjectValidator() 120 121 // Test data 122 testVP := testNetworkPolicy 123 testMC := testManagedCluster 124 asrt.NoError(v.client.Create(context.TODO(), &testMC)) 125 126 // Test create 127 req := newAdmissionRequest(admissionv1.Create, testVP) 128 res := v.Handle(context.TODO(), req) 129 asrt.True(res.Allowed, "Error validating VerrazzanProject with NetworkPolicyTemplate") 130 131 // Test update 132 req = newAdmissionRequest(admissionv1.Update, testVP) 133 res = v.Handle(context.TODO(), req) 134 asrt.True(res.Allowed, "Error validating VerrazzanProject with NetworkPolicyTemplate") 135 } 136 137 // TestNetworkPolicyMissingNamespace tests the validation of VerrazzanoProject NetworkPolicyTemplate 138 // GIVEN a call validate VerrazzanoProject on create or update 139 // WHEN the VerrazzanoProject has a NetworkPolicyTemplate with a namespace that does not exist in the project 140 // THEN the validation should fail 141 func TestNetworkPolicyMissingNamespace(t *testing.T) { 142 143 asrt := assert.New(t) 144 v := newVerrazzanoProjectValidator() 145 146 // Test data 147 testVP := testNetworkPolicy 148 testVP.Spec.Template.Namespaces[0].Metadata.Name = "ns2" 149 testMC := testManagedCluster 150 asrt.NoError(v.client.Create(context.TODO(), &testMC)) 151 152 // Test create 153 req := newAdmissionRequest(admissionv1.Create, testVP) 154 res := v.Handle(context.TODO(), req) 155 asrt.False(res.Allowed, "Expected project validation to fail due to missing network policy") 156 asrt.Containsf(res.Result.Reason, "namespace ns1 used in NetworkPolicy net1 does not exist in project", "Error validating VerrazzanProject with NetworkPolicyTemplate") 157 158 // Test update 159 req = newAdmissionRequest(admissionv1.Update, testVP) 160 res = v.Handle(context.TODO(), req) 161 asrt.False(res.Allowed, "Expected project validation to fail due to missing network policy") 162 asrt.Containsf(res.Result.Reason, "namespace ns1 used in NetworkPolicy net1 does not exist in project", "Error validating VerrazzanProject with NetworkPolicyTemplate") 163 } 164 165 // TestNamespaceUniquenessForProjects tests that the namespace of a VerrazzanoProject N does not conflict with a preexisting project 166 // GIVEN a call validate VerrazzanoProject on create or update 167 // WHEN the VerrazzanoProject has a a namespace that conflicts with any pre-existing projects 168 // THEN the validation should fail 169 func TestNamespaceUniquenessForProjects(t *testing.T) { 170 171 v := newVerrazzanoProjectValidator() 172 173 // When creating the fake client, prepopulate it with 2 Verrazzano projects 174 // existingVP1 has namespaces project1 and project2 175 // existingVP2 has namespaces project3 and project4 176 // Adding any new Verrazzano projects with these namespaces will fail validation 177 existingVP1 := &v1alpha12.VerrazzanoProject{ 178 ObjectMeta: metav1.ObjectMeta{ 179 Name: "existing-project-1", 180 Namespace: constants.VerrazzanoMultiClusterNamespace, 181 }, 182 Spec: v1alpha12.VerrazzanoProjectSpec{ 183 Template: v1alpha12.ProjectTemplate{ 184 Namespaces: []v1alpha12.NamespaceTemplate{ 185 { 186 Metadata: metav1.ObjectMeta{ 187 Name: "project1", 188 }, 189 }, 190 { 191 Metadata: metav1.ObjectMeta{ 192 Name: "project2", 193 }, 194 }, 195 }, 196 }, 197 }, 198 } 199 assert.NoError(t, v.client.Create(context.TODO(), existingVP1)) 200 201 existingVP2 := &v1alpha12.VerrazzanoProject{ 202 ObjectMeta: metav1.ObjectMeta{ 203 Name: "existing-project-2", 204 Namespace: constants.VerrazzanoMultiClusterNamespace, 205 }, 206 Spec: v1alpha12.VerrazzanoProjectSpec{ 207 Template: v1alpha12.ProjectTemplate{ 208 Namespaces: []v1alpha12.NamespaceTemplate{ 209 { 210 Metadata: metav1.ObjectMeta{ 211 Name: "project3", 212 }, 213 }, 214 { 215 Metadata: metav1.ObjectMeta{ 216 Name: "project4", 217 }, 218 }, 219 }, 220 }, 221 }, 222 } 223 assert.NoError(t, v.client.Create(context.TODO(), existingVP2)) 224 225 currentVP := v1alpha12.VerrazzanoProject{ 226 ObjectMeta: metav1.ObjectMeta{ 227 Name: "test-project", 228 Namespace: constants.VerrazzanoMultiClusterNamespace, 229 }, 230 Spec: v1alpha12.VerrazzanoProjectSpec{ 231 Template: v1alpha12.ProjectTemplate{ 232 Namespaces: []v1alpha12.NamespaceTemplate{ 233 { 234 Metadata: metav1.ObjectMeta{ 235 Name: "project", 236 }, 237 }, 238 }, 239 }, 240 }, 241 } 242 // This test will succeed because Verrazzano project test-project has unique namespace project 243 err := validateNamespaceCanBeUsed(v.client, ¤tVP) 244 assert.Nil(t, err) 245 246 currentVP = v1alpha12.VerrazzanoProject{ 247 ObjectMeta: metav1.ObjectMeta{ 248 Name: "test-project1", 249 Namespace: constants.VerrazzanoMultiClusterNamespace, 250 }, 251 Spec: v1alpha12.VerrazzanoProjectSpec{ 252 Template: v1alpha12.ProjectTemplate{ 253 Namespaces: []v1alpha12.NamespaceTemplate{ 254 { 255 Metadata: metav1.ObjectMeta{ 256 Name: "project2", 257 }, 258 }, 259 }, 260 }, 261 }, 262 } 263 264 // This test will fail because Verrazzano project test-project1 has conflicting namespace project2 265 err = validateNamespaceCanBeUsed(v.client, ¤tVP) 266 assert.NotNil(t, err) 267 // This test will fail same as above but this time coming in through parent validator 268 err = validateVerrazzanoProject(v.client, ¤tVP) 269 assert.NotNil(t, err) 270 271 currentVP = v1alpha12.VerrazzanoProject{ 272 ObjectMeta: metav1.ObjectMeta{ 273 Name: "test-project2", 274 Namespace: constants.VerrazzanoMultiClusterNamespace, 275 }, 276 Spec: v1alpha12.VerrazzanoProjectSpec{ 277 Template: v1alpha12.ProjectTemplate{ 278 Namespaces: []v1alpha12.NamespaceTemplate{ 279 { 280 Metadata: metav1.ObjectMeta{ 281 Name: "project", 282 }, 283 }, 284 { 285 Metadata: metav1.ObjectMeta{ 286 Name: "project4", 287 }, 288 }, 289 }, 290 }, 291 }, 292 } 293 // UPDATE FAIL This test will fail because Verrazzano project test-project2 has conflicting namespace project4 294 err = validateNamespaceCanBeUsed(v.client, ¤tVP) 295 assert.NotNil(t, err) 296 297 currentVP = v1alpha12.VerrazzanoProject{ 298 ObjectMeta: metav1.ObjectMeta{ 299 Name: "existing-project-1", 300 Namespace: constants.VerrazzanoMultiClusterNamespace, 301 }, 302 Spec: v1alpha12.VerrazzanoProjectSpec{ 303 Template: v1alpha12.ProjectTemplate{ 304 Namespaces: []v1alpha12.NamespaceTemplate{ 305 { 306 Metadata: metav1.ObjectMeta{ 307 Name: "project", 308 }, 309 }, 310 { 311 Metadata: metav1.ObjectMeta{ 312 Name: "project4", 313 }, 314 }, 315 }, 316 }, 317 }, 318 } 319 // This test will fail because Verrazzano project name, existing-project-1, is using a namespace in existing-project-2 320 err = validateNamespaceCanBeUsed(v.client, ¤tVP) 321 assert.NotNil(t, err) 322 323 currentVP = v1alpha12.VerrazzanoProject{ 324 ObjectMeta: metav1.ObjectMeta{ 325 Name: "existing-project-1", 326 Namespace: constants.VerrazzanoMultiClusterNamespace, 327 }, 328 Spec: v1alpha12.VerrazzanoProjectSpec{ 329 Template: v1alpha12.ProjectTemplate{ 330 Namespaces: []v1alpha12.NamespaceTemplate{ 331 { 332 Metadata: metav1.ObjectMeta{ 333 Name: "project", 334 }, 335 }, 336 { 337 Metadata: metav1.ObjectMeta{ 338 Name: "project2", 339 }, 340 }, 341 }, 342 }, 343 }, 344 } 345 // UPDATE PASS This test will pass because Verrazzano project name, existing-project-1, is not using a namespace associated with any existing projects 346 err = validateNamespaceCanBeUsed(v.client, ¤tVP) 347 assert.Nil(t, err) 348 } 349 350 // TestValidationFailureForProjectCreationWithoutTargetClusters tests preventing the creation 351 // of a VerrazzanoProject resources that is missing Placement information. 352 // GIVEN a call to validate a VerrazzanoProject resource 353 // WHEN the VerrazzanoProject resource is missing Placement information 354 // THEN the validation should fail. 355 func TestValidationFailureForProjectCreationWithoutTargetClusters(t *testing.T) { 356 asrt := assert.New(t) 357 v := newVerrazzanoProjectValidator() 358 p := v1alpha12.VerrazzanoProject{ 359 ObjectMeta: metav1.ObjectMeta{ 360 Name: "test-project-name", 361 Namespace: constants.VerrazzanoMultiClusterNamespace, 362 }, 363 Spec: v1alpha12.VerrazzanoProjectSpec{ 364 Template: v1alpha12.ProjectTemplate{ 365 Namespaces: []v1alpha12.NamespaceTemplate{ 366 { 367 Metadata: metav1.ObjectMeta{ 368 Name: "test-target-namespace", 369 }, 370 }, 371 }, 372 }, 373 }, 374 } 375 376 req := newAdmissionRequest(admissionv1.Create, p) 377 res := v.Handle(context.TODO(), req) 378 asrt.False(res.Allowed, "Expected project validation to fail due to missing placement information.") 379 asrt.Contains(res.Result.Reason, "target cluster") 380 381 req = newAdmissionRequest(admissionv1.Update, p) 382 res = v.Handle(context.TODO(), req) 383 asrt.False(res.Allowed, "Expected project validation to fail due to missing placement information.") 384 asrt.Contains(res.Result.Reason, "target cluster") 385 } 386 387 // TestValidationFailureForProjectCreationTargetingMissingManagedCluster tests preventing the creation 388 // of a VerrazzanoProject resources that references a non-existent managed cluster. 389 // GIVEN a call to validate a VerrazzanoProject resource 390 // WHEN the VerrazzanoProject resource references a VerrazzanoManagedCluster that does not exist 391 // THEN the validation should fail. 392 func TestValidationFailureForProjectCreationTargetingMissingManagedCluster(t *testing.T) { 393 394 asrt := assert.New(t) 395 v := newVerrazzanoProjectValidator() 396 p := v1alpha12.VerrazzanoProject{ 397 ObjectMeta: metav1.ObjectMeta{ 398 Name: "test-project-name", 399 Namespace: constants.VerrazzanoMultiClusterNamespace, 400 }, 401 Spec: v1alpha12.VerrazzanoProjectSpec{ 402 Placement: v1alpha12.Placement{ 403 Clusters: []v1alpha12.Cluster{{Name: "invalid-cluster-name"}}, 404 }, 405 Template: v1alpha12.ProjectTemplate{ 406 Namespaces: []v1alpha12.NamespaceTemplate{ 407 { 408 Metadata: metav1.ObjectMeta{ 409 Name: "test-target-namespace", 410 }, 411 }, 412 }, 413 }, 414 }, 415 } 416 417 req := newAdmissionRequest(admissionv1.Create, p) 418 res := v.Handle(context.TODO(), req) 419 asrt.False(res.Allowed, "Expected project validation to fail due to missing placement information.") 420 asrt.Contains(res.Result.Reason, "invalid-cluster-name") 421 422 req = newAdmissionRequest(admissionv1.Update, p) 423 res = v.Handle(context.TODO(), req) 424 asrt.False(res.Allowed, "Expected project validation to fail due to missing placement information.") 425 asrt.Contains(res.Result.Reason, "invalid-cluster-name") 426 } 427 428 // TestValidationSuccessForProjectCreationTargetingExistingManagedCluster tests allowing the creation 429 // of a VerrazzanoProject resources that references an existent managed cluster. 430 // GIVEN a call to validate a VerrazzanoProject resource 431 // WHEN the VerrazzanoProject resource references a VerrazzanoManagedCluster that does exist 432 // THEN the validation should pass. 433 func TestValidationSuccessForProjectCreationTargetingExistingManagedCluster(t *testing.T) { 434 435 asrt := assert.New(t) 436 v := newVerrazzanoProjectValidator() 437 c := v1alpha1.VerrazzanoManagedCluster{ 438 ObjectMeta: metav1.ObjectMeta{ 439 Name: "valid-cluster-name", 440 Namespace: constants.VerrazzanoMultiClusterNamespace, 441 }, 442 Spec: v1alpha1.VerrazzanoManagedClusterSpec{ 443 CASecret: "test-secret", 444 ManagedClusterManifestSecret: "test-cluster-manifest-secret", 445 ServiceAccount: "test-service-account", 446 }, 447 } 448 p := v1alpha12.VerrazzanoProject{ 449 ObjectMeta: metav1.ObjectMeta{ 450 Name: "test-project-name", 451 Namespace: constants.VerrazzanoMultiClusterNamespace, 452 }, 453 Spec: v1alpha12.VerrazzanoProjectSpec{ 454 Placement: v1alpha12.Placement{ 455 Clusters: []v1alpha12.Cluster{{Name: "valid-cluster-name"}}, 456 }, 457 Template: v1alpha12.ProjectTemplate{ 458 Namespaces: []v1alpha12.NamespaceTemplate{ 459 { 460 Metadata: metav1.ObjectMeta{ 461 Name: "test-target-namespace", 462 }, 463 }, 464 }, 465 }, 466 }, 467 } 468 asrt.NoError(v.client.Create(context.TODO(), &c)) 469 470 req := newAdmissionRequest(admissionv1.Create, p) 471 res := v.Handle(context.TODO(), req) 472 asrt.True(res.Allowed, "Expected project create validation to succeed.") 473 474 req = newAdmissionRequest(admissionv1.Update, p) 475 res = v.Handle(context.TODO(), req) 476 asrt.True(res.Allowed, "Expected project update validation to succeed.") 477 } 478 479 // TestValidationSuccessForProjectCreationWithoutTargetClustersOnManagedCluster tests allowing the creation 480 // of a VerrazzanoProject resources that is missing target cluster information when on managed cluster. 481 // GIVEN a call to validate a VerrazzanoProject resource 482 // WHEN the VerrazzanoProject resource is missing Placement information 483 // AND the validation is being done on a managed cluster 484 // THEN the validation should succeed. 485 func TestValidationSuccessForProjectCreationWithoutTargetClustersOnManagedCluster(t *testing.T) { 486 487 asrt := assert.New(t) 488 v := newVerrazzanoProjectValidator() 489 s := corev1.Secret{ 490 ObjectMeta: metav1.ObjectMeta{ 491 Name: constants.MCRegistrationSecret, 492 Namespace: constants.VerrazzanoSystemNamespace, 493 }, 494 } 495 p := v1alpha12.VerrazzanoProject{ 496 ObjectMeta: metav1.ObjectMeta{ 497 Name: "test-project-name", 498 Namespace: constants.VerrazzanoMultiClusterNamespace, 499 }, 500 Spec: v1alpha12.VerrazzanoProjectSpec{ 501 Placement: v1alpha12.Placement{ 502 Clusters: []v1alpha12.Cluster{{Name: "invalid-cluster-name"}}, 503 }, 504 Template: v1alpha12.ProjectTemplate{ 505 Namespaces: []v1alpha12.NamespaceTemplate{ 506 { 507 Metadata: metav1.ObjectMeta{ 508 Name: "test-target-namespace", 509 }, 510 }, 511 }, 512 }, 513 }, 514 } 515 asrt.NoError(v.client.Create(context.TODO(), &s)) 516 517 req := newAdmissionRequest(admissionv1.Create, p) 518 res := v.Handle(context.TODO(), req) 519 asrt.True(res.Allowed, "Expected project validation to succeed with missing placement information on managed cluster.") 520 521 req = newAdmissionRequest(admissionv1.Update, p) 522 res = v.Handle(context.TODO(), req) 523 asrt.True(res.Allowed, "Expected project validation to succeed with missing placement information on managed cluster.") 524 } 525 526 // TestValidationSuccessForProjectCreationTargetingLocalCluster tests allowing the creation 527 // of a VerrazzanoProject resources on the local admin cluster. 528 // GIVEN a call to validate a VerrazzanoProject resource 529 // WHEN the VerrazzanoProject resource targets the local cluster via Placement information 530 // AND the validation is being done on the admin cluster 531 // THEN the validation should succeed. 532 func TestValidationSuccessForProjectCreationTargetingLocalCluster(t *testing.T) { 533 534 asrt := assert.New(t) 535 v := newVerrazzanoProjectValidator() 536 p := v1alpha12.VerrazzanoProject{ 537 ObjectMeta: metav1.ObjectMeta{ 538 Name: "test-project-name", 539 Namespace: constants.VerrazzanoMultiClusterNamespace, 540 }, 541 Spec: v1alpha12.VerrazzanoProjectSpec{ 542 Placement: v1alpha12.Placement{ 543 Clusters: []v1alpha12.Cluster{{Name: constants.DefaultClusterName}}, 544 }, 545 Template: v1alpha12.ProjectTemplate{ 546 Namespaces: []v1alpha12.NamespaceTemplate{ 547 { 548 Metadata: metav1.ObjectMeta{ 549 Name: "test-target-namespace", 550 }, 551 }, 552 }, 553 }, 554 }, 555 } 556 557 req := newAdmissionRequest(admissionv1.Create, p) 558 res := v.Handle(context.TODO(), req) 559 asrt.True(res.Allowed, "Expected project validation to succeed with placement targeting local cluster.") 560 561 req = newAdmissionRequest(admissionv1.Update, p) 562 res = v.Handle(context.TODO(), req) 563 asrt.True(res.Allowed, "Expected project validation to succeed with placement targeting local cluster.") 564 } 565 566 // TestVzProjHandleFailed tests to make sure the failure metric is being exposed 567 // GIVEN a call to validate a VerrazzanoProject resource 568 // WHEN the VerrazzanoProject resource is failing 569 // THEN the validation should fail. 570 func TestVzProjHandleFailed(t *testing.T) { 571 572 assert := assert.New(t) 573 // Create a request and Handle 574 v := newVerrazzanoProjectValidator() 575 // Test data 576 testVP := testProject 577 req := newAdmissionRequest(admissionv1.Create, testVP) 578 v.Handle(context.TODO(), req) 579 reconcileerrorCounterObject, err := metricsexporter.GetSimpleCounterMetric(metricsexporter.VzProjHandleError) 580 assert.NoError(err) 581 // Expect a call to fetch the error 582 reconcileFailedCounterBefore := testutil.ToFloat64(reconcileerrorCounterObject.Get()) 583 reconcileerrorCounterObject.Get().Inc() 584 reconcileFailedCounterAfter := testutil.ToFloat64(reconcileerrorCounterObject.Get()) 585 assert.Equal(reconcileFailedCounterBefore, reconcileFailedCounterAfter-1) 586 }