github.com/kubewharf/katalyst-core@v0.5.3/pkg/util/machine/topology.go (about) 1 /* 2 Copyright 2022 The Katalyst Authors. 3 4 Licensed under the Apache License, Version 2.0 (the "License"); 5 you may not use this file except in compliance with the License. 6 You may obtain a copy of the License at 7 8 http://www.apache.org/licenses/LICENSE-2.0 9 10 Unless required by applicable law or agreed to in writing, software 11 distributed under the License is distributed on an "AS IS" BASIS, 12 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 See the License for the specific language governing permissions and 14 limitations under the License. 15 */ 16 17 package machine 18 19 import ( 20 "fmt" 21 22 info "github.com/google/cadvisor/info/v1" 23 "k8s.io/apimachinery/pkg/api/resource" 24 "k8s.io/apimachinery/pkg/util/sets" 25 "k8s.io/klog/v2" 26 27 "github.com/kubewharf/katalyst-core/pkg/config/agent/global" 28 ) 29 30 // NUMANodeInfo is a map from NUMANode ID to a list of 31 // CPU IDs associated with that NUMANode. 32 type NUMANodeInfo map[int]CPUSet 33 34 // CPUDetails is a map from CPU ID to Core ID, Socket ID, and NUMA ID. 35 type CPUDetails map[int]CPUInfo 36 37 // CPUTopology contains details of node cpu, where : 38 // CPU - logical CPU, cadvisor - thread 39 // Core - physical CPU, cadvisor - Core 40 // Socket - socket, cadvisor - Socket 41 // NUMA Node - NUMA cell, cadvisor - Node 42 type CPUTopology struct { 43 NumCPUs int 44 NumCores int 45 NumSockets int 46 NumNUMANodes int 47 CPUDetails CPUDetails 48 } 49 50 type MemoryDetails map[int]uint64 51 52 // Equal returns true if the MemoryDetails map is equal to the supplied MemoryDetails 53 func (d MemoryDetails) Equal(want MemoryDetails) bool { 54 if len(d) != len(want) { 55 return false 56 } 57 58 for k, v := range d { 59 if v != want[k] { 60 return false 61 } 62 } 63 64 return true 65 } 66 67 // Clone creates a new MemoryDetails instance with the same content. 68 func (d MemoryDetails) Clone() MemoryDetails { 69 if d == nil { 70 return nil 71 } 72 73 clone := make(MemoryDetails) 74 for key, value := range d { 75 clone[key] = value 76 } 77 78 return clone 79 } 80 81 // FillNUMANodesWithZero takes a CPUSet containing NUMA node IDs and ensures that each ID is present in MemoryDetails. 82 // If a NUMA node ID from the CPUSet is not present in the MemoryDetails map, it is added with a value of 0. 83 // The method returns an updated MemoryDetails map with these changes. 84 func (d MemoryDetails) FillNUMANodesWithZero(allNUMAs CPUSet) MemoryDetails { 85 // Clone the original MemoryDetails map 86 updatedDetails := d.Clone() 87 88 // Iterate through all NUMA IDs and ensure they are in the map 89 for numaID := range allNUMAs.ToSliceInt() { 90 if _, exists := updatedDetails[numaID]; !exists { 91 // Add the NUMA ID with a value of 0 if it doesn't exist in the map 92 updatedDetails[numaID] = 0 93 } 94 } 95 96 // Return the updated MemoryDetails map 97 return updatedDetails 98 } 99 100 type MemoryTopology struct { 101 MemoryDetails MemoryDetails 102 } 103 104 // CPUsPerCore returns the number of logical CPUs 105 // associated with each core. 106 func (topo *CPUTopology) CPUsPerCore() int { 107 if topo.NumCores == 0 { 108 return 0 109 } 110 return topo.NumCPUs / topo.NumCores 111 } 112 113 // CPUsPerSocket returns the number of logical CPUs 114 // associated with each socket. 115 func (topo *CPUTopology) CPUsPerSocket() int { 116 if topo.NumSockets == 0 { 117 return 0 118 } 119 return topo.NumCPUs / topo.NumSockets 120 } 121 122 // CPUsPerNuma returns the number of logical CPUs 123 // associated with each numa node. 124 func (topo *CPUTopology) CPUsPerNuma() int { 125 if topo.NumNUMANodes == 0 { 126 return 0 127 } 128 return topo.NumCPUs / topo.NumNUMANodes 129 } 130 131 // NUMAsPerSocket returns the the number of NUMA 132 // are associated with each socket. 133 func (topo *CPUTopology) NUMAsPerSocket() (int, error) { 134 numasCount := topo.CPUDetails.NUMANodes().Size() 135 136 if numasCount%topo.NumSockets != 0 { 137 return 0, fmt.Errorf("invalid numasCount: %d and socketsCount: %d", numasCount, topo.NumSockets) 138 } 139 140 return numasCount / topo.NumSockets, nil 141 } 142 143 // GetSocketTopology parses the given CPUTopology to a mapping 144 // from socket id to cpu id lists 145 func (topo *CPUTopology) GetSocketTopology() map[int]string { 146 if topo == nil { 147 return nil 148 } 149 150 socketTopology := make(map[int]string) 151 for _, socketID := range topo.CPUDetails.Sockets().ToSliceInt() { 152 socketTopology[socketID] = topo.CPUDetails.NUMANodesInSockets(socketID).String() 153 } 154 155 return socketTopology 156 } 157 158 func GenerateDummyMachineInfo(numaNum int, memoryCapacityGB int) (*info.MachineInfo, error) { 159 machineInfo := &info.MachineInfo{} 160 161 if memoryCapacityGB%numaNum != 0 { 162 return nil, fmt.Errorf("invalid memoryCapacityGB: %d and NUMA number: %d", memoryCapacityGB, numaNum) 163 } 164 165 perNumaCapacityGB := uint64(memoryCapacityGB / numaNum) 166 perNumaCapacityQuantity := resource.MustParse(fmt.Sprintf("%dGi", perNumaCapacityGB)) 167 168 machineInfo.Topology = make([]info.Node, 0, numaNum) 169 for i := 0; i < numaNum; i++ { 170 machineInfo.Topology = append(machineInfo.Topology, info.Node{ 171 Id: i, 172 Memory: uint64(perNumaCapacityQuantity.Value()), 173 }) 174 } 175 176 return machineInfo, nil 177 } 178 179 func GenerateDummyCPUTopology(cpuNum, socketNum, numaNum int) (*CPUTopology, error) { 180 if numaNum%socketNum != 0 { 181 return nil, fmt.Errorf("invalid NUMA number: %d and socket number: %d", numaNum, socketNum) 182 } else if cpuNum%numaNum != 0 { 183 return nil, fmt.Errorf("invalid cpu number: %d and NUMA number: %d", cpuNum, numaNum) 184 } else if cpuNum%2 != 0 { 185 // assume that we should use hyper-threads 186 return nil, fmt.Errorf("invalid cpu number: %d and NUMA number: %d", cpuNum, numaNum) 187 } 188 189 cpuTopology := new(CPUTopology) 190 cpuTopology.CPUDetails = make(map[int]CPUInfo) 191 cpuTopology.NumCPUs = cpuNum 192 cpuTopology.NumCores = cpuNum / 2 193 cpuTopology.NumSockets = socketNum 194 cpuTopology.NumNUMANodes = numaNum 195 196 numaPerSocket := numaNum / socketNum 197 cpusPerNUMA := cpuNum / numaNum 198 199 for i := 0; i < socketNum; i++ { 200 for j := i * numaPerSocket; j < (i+1)*numaPerSocket; j++ { 201 for k := j * (cpusPerNUMA / 2); k < (j+1)*(cpusPerNUMA/2); k++ { 202 cpuTopology.CPUDetails[k] = CPUInfo{ 203 NUMANodeID: j, 204 SocketID: i, 205 CoreID: k, 206 } 207 208 cpuTopology.CPUDetails[k+cpuNum/2] = CPUInfo{ 209 NUMANodeID: j, 210 SocketID: i, 211 CoreID: k, 212 } 213 } 214 } 215 } 216 217 return cpuTopology, nil 218 } 219 220 func GenerateDummyMemoryTopology(numaNum int, memoryCapacity uint64) (*MemoryTopology, error) { 221 memoryTopology := &MemoryTopology{map[int]uint64{}} 222 for i := 0; i < numaNum; i++ { 223 memoryTopology.MemoryDetails[i] = memoryCapacity / uint64(numaNum) 224 } 225 return memoryTopology, nil 226 } 227 228 func GenerateDummyExtraTopology(numaNum int) (*ExtraTopologyInfo, error) { 229 var ( 230 socketNum = 2 231 distanceNumaInSameSocket = 11 232 distanceNumaInOtherSocket = 21 233 ) 234 235 extraTopology := &ExtraTopologyInfo{ 236 NumaDistanceMap: make(map[int][]NumaDistanceInfo), 237 SiblingNumaInfo: &SiblingNumaInfo{ 238 SiblingNumaMap: make(map[int]sets.Int), 239 SiblingNumaAvgMBWAllocatableMap: make(map[int]int64), 240 SiblingNumaAvgMBWCapacityMap: make(map[int]int64), 241 }, 242 } 243 244 for i := 0; i < numaNum; i++ { 245 numaDistanceInfos := make([]NumaDistanceInfo, 0) 246 for j := 0; j < numaNum; j++ { 247 if i == j { 248 continue 249 } else if i/socketNum == j/socketNum { 250 numaDistanceInfos = append(numaDistanceInfos, NumaDistanceInfo{ 251 Distance: distanceNumaInSameSocket, 252 NumaID: j, 253 }) 254 } else { 255 numaDistanceInfos = append(numaDistanceInfos, NumaDistanceInfo{ 256 Distance: distanceNumaInOtherSocket, 257 NumaID: j, 258 }) 259 } 260 } 261 262 extraTopology.NumaDistanceMap[i] = numaDistanceInfos 263 extraTopology.SiblingNumaMap[i] = make(sets.Int) 264 } 265 return extraTopology, nil 266 } 267 268 // CPUInfo contains the NUMA, socket, and core IDs associated with a CPU. 269 type CPUInfo struct { 270 NUMANodeID int 271 SocketID int 272 CoreID int 273 } 274 275 // KeepOnly returns a new CPUDetails object with only the supplied cpus. 276 func (d CPUDetails) KeepOnly(cpus CPUSet) CPUDetails { 277 result := CPUDetails{} 278 for cpu, info := range d { 279 if cpus.Contains(cpu) { 280 result[cpu] = info 281 } 282 } 283 return result 284 } 285 286 // NUMANodes returns all NUMANode IDs associated with the CPUs in this CPUDetails. 287 func (d CPUDetails) NUMANodes() CPUSet { 288 b := NewCPUSet() 289 for _, info := range d { 290 b.Add(info.NUMANodeID) 291 } 292 return b 293 } 294 295 // NUMANodesInSockets returns all logical NUMANode IDs associated with 296 // the given socket IDs in this CPUDetails. 297 func (d CPUDetails) NUMANodesInSockets(ids ...int) CPUSet { 298 b := NewCPUSet() 299 for _, id := range ids { 300 for _, info := range d { 301 if info.SocketID == id { 302 b.Add(info.NUMANodeID) 303 } 304 } 305 } 306 return b 307 } 308 309 // Sockets returns all socket IDs associated with the CPUs in this CPUDetails. 310 func (d CPUDetails) Sockets() CPUSet { 311 b := NewCPUSet() 312 for _, info := range d { 313 b.Add(info.SocketID) 314 } 315 return b 316 } 317 318 // CPUsInSockets returns all logical CPU IDs associated with the given 319 // socket IDs in this CPUDetails. 320 func (d CPUDetails) CPUsInSockets(ids ...int) CPUSet { 321 b := NewCPUSet() 322 for _, id := range ids { 323 for cpu, info := range d { 324 if info.SocketID == id { 325 b.Add(cpu) 326 } 327 } 328 } 329 return b 330 } 331 332 // SocketsInNUMANodes returns all logical Socket IDs associated with the 333 // given NUMANode IDs in this CPUDetails. 334 func (d CPUDetails) SocketsInNUMANodes(ids ...int) CPUSet { 335 b := NewCPUSet() 336 for _, id := range ids { 337 for _, info := range d { 338 if info.NUMANodeID == id { 339 b.Add(info.SocketID) 340 } 341 } 342 } 343 return b 344 } 345 346 // Cores returns all core IDs associated with the CPUs in this CPUDetails. 347 func (d CPUDetails) Cores() CPUSet { 348 b := NewCPUSet() 349 for _, info := range d { 350 b.Add(info.CoreID) 351 } 352 return b 353 } 354 355 // CoresInNUMANodes returns all core IDs associated with the given 356 // NUMANode IDs in this CPUDetails. 357 func (d CPUDetails) CoresInNUMANodes(ids ...int) CPUSet { 358 b := NewCPUSet() 359 for _, id := range ids { 360 for _, info := range d { 361 if info.NUMANodeID == id { 362 b.Add(info.CoreID) 363 } 364 } 365 } 366 return b 367 } 368 369 // CoresInSockets returns all core IDs associated with the given socket 370 // IDs in this CPUDetails. 371 func (d CPUDetails) CoresInSockets(ids ...int) CPUSet { 372 b := NewCPUSet() 373 for _, id := range ids { 374 for _, info := range d { 375 if info.SocketID == id { 376 b.Add(info.CoreID) 377 } 378 } 379 } 380 return b 381 } 382 383 // CPUs returns all logical CPU IDs in this CPUDetails. 384 func (d CPUDetails) CPUs() CPUSet { 385 b := NewCPUSet() 386 for cpuID := range d { 387 b.Add(cpuID) 388 } 389 return b 390 } 391 392 // CPUsInNUMANodes returns all logical CPU IDs associated with the given 393 // NUMANode IDs in this CPUDetails. 394 func (d CPUDetails) CPUsInNUMANodes(ids ...int) CPUSet { 395 b := NewCPUSet() 396 for _, id := range ids { 397 for cpu, info := range d { 398 if info.NUMANodeID == id { 399 b.Add(cpu) 400 } 401 } 402 } 403 return b 404 } 405 406 // CPUsInCores returns all logical CPU IDs associated with the given 407 // core IDs in this CPUDetails. 408 func (d CPUDetails) CPUsInCores(ids ...int) CPUSet { 409 b := NewCPUSet() 410 for _, id := range ids { 411 for cpu, info := range d { 412 if info.CoreID == id { 413 b.Add(cpu) 414 } 415 } 416 } 417 return b 418 } 419 420 // Discover returns CPUTopology based on cadvisor node info 421 func Discover(machineInfo *info.MachineInfo) (*CPUTopology, *MemoryTopology, error) { 422 if machineInfo.NumCores == 0 { 423 return nil, nil, fmt.Errorf("could not detect number of cpus") 424 } 425 426 CPUDetails := CPUDetails{} 427 numPhysicalCores := 0 428 429 memoryTopology := MemoryTopology{MemoryDetails: map[int]uint64{}} 430 431 for _, node := range machineInfo.Topology { 432 memoryTopology.MemoryDetails[node.Id] = node.Memory 433 434 numPhysicalCores += len(node.Cores) 435 for _, core := range node.Cores { 436 if coreID, err := getUniqueCoreID(core.Threads); err == nil { 437 for _, cpu := range core.Threads { 438 CPUDetails[cpu] = CPUInfo{ 439 CoreID: coreID, 440 SocketID: core.SocketID, 441 NUMANodeID: node.Id, 442 } 443 } 444 } else { 445 klog.ErrorS(nil, "Could not get unique coreID for socket", 446 "socket", core.SocketID, "core", core.Id, "threads", core.Threads) 447 return nil, nil, err 448 } 449 } 450 } 451 452 return &CPUTopology{ 453 NumCPUs: machineInfo.NumCores, 454 NumSockets: machineInfo.NumSockets, 455 NumCores: numPhysicalCores, 456 NumNUMANodes: CPUDetails.NUMANodes().Size(), 457 CPUDetails: CPUDetails, 458 }, &memoryTopology, nil 459 } 460 461 // getUniqueCoreID computes coreId as the lowest cpuID 462 // for a given Threads []int slice. This will assure that coreID's are 463 // platform unique (opposite to what cAdvisor reports) 464 func getUniqueCoreID(threads []int) (coreID int, err error) { 465 if len(threads) == 0 { 466 return 0, fmt.Errorf("no cpus provided") 467 } 468 469 if len(threads) != NewCPUSet(threads...).Size() { 470 return 0, fmt.Errorf("cpus provided are not unique") 471 } 472 473 min := threads[0] 474 for _, thread := range threads[1:] { 475 if thread < min { 476 min = thread 477 } 478 } 479 480 return min, nil 481 } 482 483 // GetNumaAwareAssignments returns a mapping from NUMA id to cpu core 484 func GetNumaAwareAssignments(topology *CPUTopology, cset CPUSet) (map[int]CPUSet, error) { 485 if topology == nil { 486 return nil, fmt.Errorf("GetTopologyAwareAssignmentsByCPUSet got nil cpuset") 487 } 488 489 topologyAwareAssignments := make(map[int]CPUSet) 490 numaNodes := topology.CPUDetails.NUMANodes() 491 for _, numaNode := range numaNodes.ToSliceNoSortInt() { 492 cs := cset.Intersection(topology.CPUDetails.CPUsInNUMANodes(numaNode).Clone()) 493 if cs.Size() > 0 { 494 topologyAwareAssignments[numaNode] = cs 495 } 496 } 497 498 return topologyAwareAssignments, nil 499 } 500 501 // CheckNUMACrossSockets judges whether the given NUMA nodes are located 502 // in different sockets 503 func CheckNUMACrossSockets(numaNodes []int, cpuTopology *CPUTopology) (bool, error) { 504 if cpuTopology == nil { 505 return false, fmt.Errorf("CheckNUMACrossSockets got nil cpuTopology") 506 } 507 508 if len(numaNodes) <= 1 { 509 return false, nil 510 } 511 return cpuTopology.CPUDetails.SocketsInNUMANodes(numaNodes...).Size() > 1, nil 512 } 513 514 func GetSiblingNumaInfo(conf *global.MachineInfoConfiguration, 515 numaDistanceMap map[int][]NumaDistanceInfo, 516 ) *SiblingNumaInfo { 517 siblingNumaMap := make(map[int]sets.Int) 518 siblingNumaAvgMBWAllocatableMap := make(map[int]int64) 519 siblingNumaAvgMBWCapacityMap := make(map[int]int64) 520 521 // calculate the sibling NUMA allocatable memory bandwidth by the capacity multiplying the allocatable rate. 522 // Now, all the NUMAs have the same memory bandwidth capacity and allocatable 523 siblingNumaMBWCapacity := conf.SiblingNumaMemoryBandwidthCapacity 524 siblingNumaMBWAllocatable := int64(float64(siblingNumaMBWCapacity) * conf.SiblingNumaMemoryBandwidthAllocatableRate) 525 526 for numaID, distanceMap := range numaDistanceMap { 527 var selfNumaDistance int 528 for _, distance := range distanceMap { 529 if distance.NumaID == numaID { 530 selfNumaDistance = distance.Distance 531 break 532 } 533 } 534 535 siblingSet := sets.NewInt() 536 for _, distance := range distanceMap { 537 if distance.NumaID == numaID { 538 continue 539 } 540 541 // the distance between two different NUMAs is equal to the distance between 542 // it and itself are siblings each other 543 if distance.Distance == selfNumaDistance { 544 siblingSet.Insert(distance.NumaID) 545 } 546 } 547 548 siblingNumaMap[numaID] = siblingSet 549 siblingNumaAvgMBWAllocatableMap[numaID] = siblingNumaMBWAllocatable / int64(len(siblingSet)+1) 550 siblingNumaAvgMBWCapacityMap[numaID] = siblingNumaMBWCapacity / int64(len(siblingSet)+1) 551 } 552 553 return &SiblingNumaInfo{ 554 SiblingNumaMap: siblingNumaMap, 555 SiblingNumaAvgMBWCapacityMap: siblingNumaAvgMBWCapacityMap, 556 SiblingNumaAvgMBWAllocatableMap: siblingNumaAvgMBWAllocatableMap, 557 } 558 } 559 560 type NumaDistanceInfo struct { 561 NumaID int 562 Distance int 563 } 564 565 type ExtraTopologyInfo struct { 566 NumaDistanceMap map[int][]NumaDistanceInfo 567 *SiblingNumaInfo 568 } 569 570 type SiblingNumaInfo struct { 571 SiblingNumaMap map[int]sets.Int 572 573 // SiblingNumaAvgMBWAllocatableMap maps NUMA IDs to the allocatable memory bandwidth, 574 // averaged across each NUMA node and its siblings. 575 // SiblingNumaAvgMBWCapacityMap maps NUMA IDs to the capacity memory bandwidth, 576 // averaged similarly. 577 SiblingNumaAvgMBWAllocatableMap map[int]int64 578 SiblingNumaAvgMBWCapacityMap map[int]int64 579 }