github.com/kubewharf/katalyst-core@v0.5.3/pkg/scheduler/plugins/noderesourcetopology/score_test.go (about) 1 /* 2 Copyright 2022 The Katalyst Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package noderesourcetopology 18 19 import ( 20 "context" 21 "reflect" 22 "testing" 23 24 "github.com/stretchr/testify/assert" 25 v1 "k8s.io/api/core/v1" 26 "k8s.io/apimachinery/pkg/api/resource" 27 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 28 "k8s.io/kubernetes/pkg/scheduler/apis/config" 29 "k8s.io/kubernetes/pkg/scheduler/framework" 30 "k8s.io/kubernetes/pkg/scheduler/framework/runtime" 31 32 "github.com/kubewharf/katalyst-api/pkg/apis/node/v1alpha1" 33 "github.com/kubewharf/katalyst-api/pkg/consts" 34 "github.com/kubewharf/katalyst-core/pkg/config/generic" 35 "github.com/kubewharf/katalyst-core/pkg/scheduler/cache" 36 "github.com/kubewharf/katalyst-core/pkg/scheduler/util" 37 ) 38 39 func makeTestScoreNodes(policy v1alpha1.TopologyPolicy) ([]*v1alpha1.CustomNodeResource, []string, []*v1.Pod) { 40 cnrs := []*v1alpha1.CustomNodeResource{ 41 { 42 ObjectMeta: metav1.ObjectMeta{Name: "node-2numa-8c16g"}, 43 Status: v1alpha1.CustomNodeResourceStatus{ 44 TopologyPolicy: policy, 45 TopologyZone: []*v1alpha1.TopologyZone{ 46 { 47 Name: "0", 48 Type: v1alpha1.TopologyTypeSocket, 49 Children: []*v1alpha1.TopologyZone{ 50 { 51 Name: "0", 52 Type: v1alpha1.TopologyTypeNuma, 53 Resources: v1alpha1.Resources{ 54 Capacity: &v1.ResourceList{ 55 v1.ResourceCPU: resource.MustParse("4"), 56 v1.ResourceMemory: resource.MustParse("8Gi"), 57 "Gpu": resource.MustParse("4"), 58 }, 59 Allocatable: &v1.ResourceList{ 60 v1.ResourceCPU: resource.MustParse("4"), 61 v1.ResourceMemory: resource.MustParse("8Gi"), 62 "Gpu": resource.MustParse("4"), 63 }, 64 }, 65 }, 66 { 67 Name: "1", 68 Type: v1alpha1.TopologyTypeNuma, 69 Resources: v1alpha1.Resources{ 70 Capacity: &v1.ResourceList{ 71 v1.ResourceCPU: resource.MustParse("4"), 72 v1.ResourceMemory: resource.MustParse("8Gi"), 73 "Gpu": resource.MustParse("4"), 74 }, 75 Allocatable: &v1.ResourceList{ 76 v1.ResourceCPU: resource.MustParse("4"), 77 v1.ResourceMemory: resource.MustParse("8Gi"), 78 "Gpu": resource.MustParse("4"), 79 }, 80 }, 81 }, 82 }, 83 }, 84 }, 85 }, 86 }, 87 { 88 ObjectMeta: metav1.ObjectMeta{Name: "node-2numa-4c8g"}, 89 Status: v1alpha1.CustomNodeResourceStatus{ 90 TopologyPolicy: policy, 91 TopologyZone: []*v1alpha1.TopologyZone{ 92 { 93 Name: "0", 94 Type: v1alpha1.TopologyTypeSocket, 95 Children: []*v1alpha1.TopologyZone{ 96 { 97 Name: "0", 98 Type: v1alpha1.TopologyTypeNuma, 99 Resources: v1alpha1.Resources{ 100 Capacity: &v1.ResourceList{ 101 v1.ResourceCPU: resource.MustParse("2"), 102 v1.ResourceMemory: resource.MustParse("4Gi"), 103 "Gpu": resource.MustParse("2"), 104 }, 105 Allocatable: &v1.ResourceList{ 106 v1.ResourceCPU: resource.MustParse("2"), 107 v1.ResourceMemory: resource.MustParse("4Gi"), 108 "Gpu": resource.MustParse("2"), 109 }, 110 }, 111 }, 112 { 113 Name: "1", 114 Type: v1alpha1.TopologyTypeNuma, 115 Resources: v1alpha1.Resources{ 116 Capacity: &v1.ResourceList{ 117 v1.ResourceCPU: resource.MustParse("2"), 118 v1.ResourceMemory: resource.MustParse("4Gi"), 119 "Gpu": resource.MustParse("2"), 120 }, 121 Allocatable: &v1.ResourceList{ 122 v1.ResourceCPU: resource.MustParse("2"), 123 v1.ResourceMemory: resource.MustParse("4Gi"), 124 "Gpu": resource.MustParse("2"), 125 }, 126 }, 127 }, 128 }, 129 }, 130 }, 131 }, 132 }, 133 { 134 ObjectMeta: metav1.ObjectMeta{Name: "node-2numa-8c16g-with-allocation"}, 135 Status: v1alpha1.CustomNodeResourceStatus{ 136 TopologyPolicy: policy, 137 TopologyZone: []*v1alpha1.TopologyZone{ 138 { 139 Name: "0", 140 Type: v1alpha1.TopologyTypeSocket, 141 Children: []*v1alpha1.TopologyZone{ 142 { 143 Name: "0", 144 Type: v1alpha1.TopologyTypeNuma, 145 Resources: v1alpha1.Resources{ 146 Capacity: &v1.ResourceList{ 147 v1.ResourceCPU: resource.MustParse("4"), 148 v1.ResourceMemory: resource.MustParse("8Gi"), 149 "Gpu": resource.MustParse("4"), 150 }, 151 Allocatable: &v1.ResourceList{ 152 v1.ResourceCPU: resource.MustParse("4"), 153 v1.ResourceMemory: resource.MustParse("8Gi"), 154 "Gpu": resource.MustParse("4"), 155 }, 156 }, 157 Allocations: []*v1alpha1.Allocation{ 158 { 159 Consumer: "testNamespace/testPod1/uid", 160 Requests: &v1.ResourceList{ 161 v1.ResourceCPU: resource.MustParse("1"), 162 v1.ResourceMemory: resource.MustParse("2Gi"), 163 }, 164 }, 165 }, 166 }, 167 { 168 Name: "1", 169 Type: v1alpha1.TopologyTypeNuma, 170 Resources: v1alpha1.Resources{ 171 Capacity: &v1.ResourceList{ 172 v1.ResourceCPU: resource.MustParse("4"), 173 v1.ResourceMemory: resource.MustParse("8Gi"), 174 "Gpu": resource.MustParse("4"), 175 }, 176 Allocatable: &v1.ResourceList{ 177 v1.ResourceCPU: resource.MustParse("4"), 178 v1.ResourceMemory: resource.MustParse("8Gi"), 179 "Gpu": resource.MustParse("4"), 180 }, 181 }, 182 Allocations: []*v1alpha1.Allocation{ 183 { 184 Consumer: "testNamespace/testPod2/uid", 185 Requests: &v1.ResourceList{ 186 v1.ResourceCPU: resource.MustParse("3"), 187 v1.ResourceMemory: resource.MustParse("6Gi"), 188 }, 189 }, 190 }, 191 }, 192 }, 193 }, 194 }, 195 }, 196 }, 197 { 198 ObjectMeta: metav1.ObjectMeta{Name: "node-4numa-8c16g"}, 199 Status: v1alpha1.CustomNodeResourceStatus{ 200 TopologyPolicy: policy, 201 TopologyZone: []*v1alpha1.TopologyZone{ 202 { 203 Name: "0", 204 Type: v1alpha1.TopologyTypeSocket, 205 Children: []*v1alpha1.TopologyZone{ 206 { 207 Name: "0", 208 Type: v1alpha1.TopologyTypeNuma, 209 Resources: v1alpha1.Resources{ 210 Capacity: &v1.ResourceList{ 211 v1.ResourceCPU: resource.MustParse("2"), 212 v1.ResourceMemory: resource.MustParse("4Gi"), 213 }, 214 Allocatable: &v1.ResourceList{ 215 v1.ResourceCPU: resource.MustParse("2"), 216 v1.ResourceMemory: resource.MustParse("4Gi"), 217 }, 218 }, 219 }, 220 { 221 Name: "1", 222 Type: v1alpha1.TopologyTypeNuma, 223 Resources: v1alpha1.Resources{ 224 Capacity: &v1.ResourceList{ 225 v1.ResourceCPU: resource.MustParse("2"), 226 v1.ResourceMemory: resource.MustParse("4Gi"), 227 }, 228 Allocatable: &v1.ResourceList{ 229 v1.ResourceCPU: resource.MustParse("2"), 230 v1.ResourceMemory: resource.MustParse("4Gi"), 231 }, 232 }, 233 }, 234 }, 235 }, 236 { 237 Name: "1", 238 Type: v1alpha1.TopologyTypeSocket, 239 Children: []*v1alpha1.TopologyZone{ 240 { 241 Name: "2", 242 Type: v1alpha1.TopologyTypeNuma, 243 Resources: v1alpha1.Resources{ 244 Capacity: &v1.ResourceList{ 245 v1.ResourceCPU: resource.MustParse("2"), 246 v1.ResourceMemory: resource.MustParse("4Gi"), 247 }, 248 Allocatable: &v1.ResourceList{ 249 v1.ResourceCPU: resource.MustParse("2"), 250 v1.ResourceMemory: resource.MustParse("4Gi"), 251 }, 252 }, 253 }, 254 { 255 Name: "3", 256 Type: v1alpha1.TopologyTypeNuma, 257 Resources: v1alpha1.Resources{ 258 Capacity: &v1.ResourceList{ 259 v1.ResourceCPU: resource.MustParse("2"), 260 v1.ResourceMemory: resource.MustParse("4Gi"), 261 }, 262 Allocatable: &v1.ResourceList{ 263 v1.ResourceCPU: resource.MustParse("2"), 264 v1.ResourceMemory: resource.MustParse("4Gi"), 265 }, 266 }, 267 }, 268 }, 269 }, 270 }, 271 }, 272 }, 273 } 274 275 pods := []*v1.Pod{ 276 { 277 ObjectMeta: metav1.ObjectMeta{ 278 Namespace: "testNamespace", 279 Name: "testPod1", 280 Annotations: map[string]string{ 281 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 282 }, 283 }, 284 Spec: v1.PodSpec{ 285 NodeName: "node-2numa-8c16g-with-allocation", 286 }, 287 }, 288 { 289 ObjectMeta: metav1.ObjectMeta{ 290 Namespace: "testNamespace", 291 Name: "testPod2", 292 Annotations: map[string]string{ 293 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 294 }, 295 }, 296 Spec: v1.PodSpec{ 297 NodeName: "node-2numa-8c16g-with-allocation", 298 }, 299 }, 300 } 301 return cnrs, []string{"node-2numa-8c16g", "node-2numa-4c8g", "node-2numa-8c16g-with-allocation", "node-4numa-8c16g"}, pods 302 } 303 304 func TestScore(t *testing.T) { 305 type testCase struct { 306 name string 307 policy v1alpha1.TopologyPolicy 308 strategy config.ScoringStrategyType 309 alignedResource []string 310 pod *v1.Pod 311 wantStatus *framework.Status 312 wantRes map[string]int64 313 } 314 315 nativeTestCases := []testCase{ 316 { 317 name: "native pod + single numa + MostAllocated strategy", 318 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 319 strategy: config.MostAllocated, 320 alignedResource: []string{"cpu", "memory"}, 321 wantRes: map[string]int64{ 322 "node-2numa-8c16g": 100, // just binpack a numanode full 323 "node-2numa-4c8g": 0, // not satisfy 324 "node-2numa-8c16g-with-allocation": 0, 325 "node-4numa-8c16g": 0, // not satisfy 326 }, 327 wantStatus: nil, 328 pod: makePodByResourceList(&v1.ResourceList{ 329 v1.ResourceCPU: resource.MustParse("4"), 330 v1.ResourceMemory: resource.MustParse("8Gi"), 331 }, map[string]string{}), 332 }, 333 { 334 name: "native pod + single numa + LeastAllocated strategy", 335 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 336 strategy: config.LeastAllocated, 337 alignedResource: []string{"cpu", "memory"}, 338 wantRes: map[string]int64{ 339 "node-2numa-8c16g": 50, 340 "node-2numa-4c8g": 0, 341 "node-2numa-8c16g-with-allocation": 33, 342 "node-4numa-8c16g": 0, 343 }, 344 wantStatus: nil, 345 pod: makePodByResourceList(&v1.ResourceList{ 346 v1.ResourceCPU: resource.MustParse("2"), 347 v1.ResourceMemory: resource.MustParse("4Gi"), 348 }, map[string]string{}), 349 }, 350 { 351 name: "native pod + single numa + BalancedAllocation strategy", 352 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 353 strategy: consts.BalancedAllocation, 354 alignedResource: []string{"cpu", "memory"}, 355 wantRes: map[string]int64{ 356 "node-2numa-8c16g": 100, 357 "node-2numa-4c8g": 100, 358 "node-2numa-8c16g-with-allocation": 100, 359 "node-4numa-8c16g": 100, 360 }, 361 wantStatus: nil, 362 pod: makePodByResourceList(&v1.ResourceList{ 363 v1.ResourceCPU: resource.MustParse("2"), 364 v1.ResourceMemory: resource.MustParse("4Gi"), 365 }, map[string]string{}), 366 }, 367 { 368 name: "native pod + no single numa nodes", 369 policy: v1alpha1.TopologyPolicyNumericContainerLevel, 370 strategy: config.MostAllocated, 371 alignedResource: []string{"cpu", "memory"}, 372 wantRes: map[string]int64{ 373 // not best node 374 "node-2numa-8c16g": 0, 375 "node-2numa-4c8g": 0, 376 "node-2numa-8c16g-with-allocation": 0, 377 "node-4numa-8c16g": 0, 378 }, 379 wantStatus: nil, 380 pod: makePodByResourceList(&v1.ResourceList{ 381 v1.ResourceCPU: resource.MustParse("4"), 382 v1.ResourceMemory: resource.MustParse("8Gi"), 383 }, map[string]string{}), 384 }, 385 { 386 name: "native pod + not Guaranteed", 387 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 388 strategy: config.MostAllocated, 389 alignedResource: []string{"cpu", "memory"}, 390 wantRes: map[string]int64{ 391 // skip score 392 "node-2numa-8c16g": 100, 393 "node-2numa-4c8g": 100, 394 "node-2numa-8c16g-with-allocation": 100, 395 "node-4numa-8c16g": 100, 396 }, 397 wantStatus: nil, 398 pod: makePodByResourceList(&v1.ResourceList{ 399 v1.ResourceCPU: resource.MustParse("350m"), 400 v1.ResourceMemory: resource.MustParse("8Gi"), 401 }, map[string]string{}), 402 }, 403 } 404 405 dedicatedTestCases := []testCase{ 406 { 407 name: "dedicated_cores with numabinding + single numa + MostAllocated strategy", 408 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 409 strategy: config.MostAllocated, 410 alignedResource: []string{"cpu", "memory"}, 411 wantRes: map[string]int64{ 412 "node-2numa-8c16g": 50, 413 "node-2numa-4c8g": 100, 414 "node-2numa-8c16g-with-allocation": 66, 415 "node-4numa-8c16g": 100, 416 }, 417 pod: makePodByResourceList(&v1.ResourceList{ 418 v1.ResourceCPU: resource.MustParse("2"), 419 v1.ResourceMemory: resource.MustParse("4Gi"), 420 "Gpu": resource.MustParse("1"), 421 }, map[string]string{ 422 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 423 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding":"true"}`, 424 }), 425 }, 426 { 427 name: "dedicated_cores with numabinding + single numa + LeastAllocated strategy", 428 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 429 strategy: config.LeastAllocated, 430 alignedResource: []string{"cpu", "memory"}, 431 wantRes: map[string]int64{ 432 "node-2numa-8c16g": 50, 433 "node-2numa-4c8g": 0, 434 "node-2numa-8c16g-with-allocation": 33, 435 "node-4numa-8c16g": 0, 436 }, 437 pod: makePodByResourceList(&v1.ResourceList{ 438 v1.ResourceCPU: resource.MustParse("2"), 439 v1.ResourceMemory: resource.MustParse("4Gi"), 440 "Gpu": resource.MustParse("1"), 441 }, map[string]string{ 442 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 443 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding":"true"}`, 444 }), 445 }, 446 { 447 name: "dedicated_cores without numabinding", 448 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 449 strategy: config.LeastAllocated, 450 alignedResource: []string{"cpu", "memory"}, 451 pod: makePodByResourceList(&v1.ResourceList{ 452 v1.ResourceCPU: resource.MustParse("2"), 453 v1.ResourceMemory: resource.MustParse("4Gi"), 454 "Gpu": resource.MustParse("1"), 455 }, map[string]string{ 456 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 457 }), 458 // ignore score 459 wantRes: map[string]int64{ 460 "node-2numa-8c16g": 100, 461 "node-2numa-4c8g": 100, 462 "node-2numa-8c16g-with-allocation": 100, 463 "node-4numa-8c16g": 100, 464 }, 465 }, 466 { 467 name: "dedicated_cores with numa_exclusive + single numa + MostAllocated", 468 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 469 strategy: config.MostAllocated, 470 alignedResource: []string{"cpu", "memory"}, 471 wantRes: map[string]int64{ 472 "node-2numa-8c16g": 50, 473 "node-2numa-4c8g": 100, 474 "node-2numa-8c16g-with-allocation": 0, 475 "node-4numa-8c16g": 100, 476 }, 477 pod: makePodByResourceList(&v1.ResourceList{ 478 v1.ResourceCPU: resource.MustParse("2"), 479 v1.ResourceMemory: resource.MustParse("4Gi"), 480 "Gpu": resource.MustParse("1"), 481 }, map[string]string{ 482 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 483 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding":"true","numa_exclusive":"true"}`, 484 }), 485 }, 486 { 487 name: "dedicaetd_cores with numa_exclusive + single numa + LeastAllocated", 488 policy: v1alpha1.TopologyPolicySingleNUMANodeContainerLevel, 489 strategy: config.LeastAllocated, 490 alignedResource: []string{"cpu", "memory"}, 491 wantRes: map[string]int64{ 492 "node-2numa-8c16g": 50, 493 "node-2numa-4c8g": 0, 494 "node-2numa-8c16g-with-allocation": 0, 495 "node-4numa-8c16g": 0, 496 }, 497 pod: makePodByResourceList(&v1.ResourceList{ 498 v1.ResourceCPU: resource.MustParse("2"), 499 v1.ResourceMemory: resource.MustParse("4Gi"), 500 "Gpu": resource.MustParse("1"), 501 }, map[string]string{ 502 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 503 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding":"true","numa_exclusive":"true"}`, 504 }), 505 }, 506 { 507 name: "dedicated_cores with numa_exclusive + numeric + MostAllocated", 508 policy: v1alpha1.TopologyPolicyNumericContainerLevel, 509 strategy: config.MostAllocated, 510 alignedResource: []string{"cpu", "memory"}, 511 wantRes: map[string]int64{ 512 "node-2numa-8c16g": 75, // one numaNode 513 "node-2numa-4c8g": 75, // two numaNodes 514 "node-2numa-8c16g-with-allocation": 0, // no numaNode idle 515 "node-4numa-8c16g": 75, // two numaNodes 516 }, 517 pod: makePodByResourceList(&v1.ResourceList{ 518 v1.ResourceCPU: resource.MustParse("3"), 519 v1.ResourceMemory: resource.MustParse("6Gi"), 520 "Gpu": resource.MustParse("1"), 521 }, map[string]string{ 522 consts.PodAnnotationQoSLevelKey: consts.PodAnnotationQoSLevelDedicatedCores, 523 consts.PodAnnotationMemoryEnhancementKey: `{"numa_binding":"true","numa_exclusive":"true"}`, 524 }), 525 }, 526 } 527 528 c := cache.GetCache() 529 for _, tc := range nativeTestCases { 530 util.SetQoSConfig(generic.NewQoSConfiguration()) 531 cnrs, nodeNames, pods := makeTestScoreNodes(tc.policy) 532 for _, cnr := range cnrs { 533 c.AddOrUpdateCNR(cnr) 534 } 535 536 nodes := make([]*v1.Node, 0) 537 for _, node := range nodeNames { 538 n := &v1.Node{} 539 n.SetName(node) 540 nodes = append(nodes, n) 541 } 542 f, err := runtime.NewFramework(nil, nil, 543 runtime.WithSnapshotSharedLister(newTestSharedLister(pods, nodes))) 544 assert.NoError(t, err) 545 tm, err := MakeTestTm(MakeTestArgs(tc.strategy, tc.alignedResource, "native"), f) 546 assert.NoError(t, err) 547 548 nodeToScore := make(map[string]int64) 549 for _, node := range nodeNames { 550 score, status := tm.(*TopologyMatch).Score(context.TODO(), nil, tc.pod, node) 551 assert.True(t, reflect.DeepEqual(status, tc.wantStatus), tc.name) 552 553 nodeToScore[node] = score 554 } 555 for wantN, wantS := range tc.wantRes { 556 assert.Equal(t, wantS, nodeToScore[wantN]) 557 } 558 559 // clean cache 560 for _, cnr := range cnrs { 561 c.RemoveCNR(cnr) 562 } 563 } 564 565 for _, tc := range dedicatedTestCases { 566 util.SetQoSConfig(generic.NewQoSConfiguration()) 567 cnrs, nodeNames, pods := makeTestScoreNodes(tc.policy) 568 for _, cnr := range cnrs { 569 c.AddOrUpdateCNR(cnr) 570 } 571 572 nodes := make([]*v1.Node, 0) 573 for _, node := range nodeNames { 574 n := &v1.Node{} 575 n.SetName(node) 576 nodes = append(nodes, n) 577 } 578 f, err := runtime.NewFramework(nil, nil, 579 runtime.WithSnapshotSharedLister(newTestSharedLister(pods, nodes))) 580 assert.NoError(t, err) 581 tm, err := MakeTestTm(MakeTestArgs(tc.strategy, tc.alignedResource, "dynamic"), f) 582 assert.NoError(t, err) 583 nodeToScore := make(map[string]int64) 584 for _, node := range nodeNames { 585 score, status := tm.(*TopologyMatch).Score(context.TODO(), nil, tc.pod, node) 586 assert.True(t, reflect.DeepEqual(status, tc.wantStatus), tc.name) 587 588 nodeToScore[node] = score 589 } 590 for wantN, wantS := range tc.wantRes { 591 assert.Equal(t, wantS, nodeToScore[wantN]) 592 } 593 594 // clean cache 595 for _, cnr := range cnrs { 596 c.RemoveCNR(cnr) 597 } 598 } 599 }