github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/nomad/structs/network.go (about) 1 package structs 2 3 import ( 4 "fmt" 5 "math/rand" 6 "net" 7 "sync" 8 9 "golang.org/x/exp/maps" 10 ) 11 12 const ( 13 // DefaultMinDynamicPort is the smallest dynamic port generated by 14 // default 15 DefaultMinDynamicPort = 20000 16 17 // DefaultMaxDynamicPort is the largest dynamic port generated by 18 // default 19 DefaultMaxDynamicPort = 32000 20 21 // maxRandPortAttempts is the maximum number of attempt 22 // to assign a random port 23 maxRandPortAttempts = 20 24 25 // MaxValidPort is the max valid port number 26 MaxValidPort = 65536 27 ) 28 29 var ( 30 // bitmapPool is used to pool the bitmaps used for port collision 31 // checking. They are fairly large (8K) so we can re-use them to 32 // avoid GC pressure. Care should be taken to call Clear() on any 33 // bitmap coming from the pool. 34 bitmapPool = new(sync.Pool) 35 ) 36 37 // NetworkIndex is used to index the available network resources 38 // and the used network resources on a machine given allocations 39 // 40 // Fields are exported so they may be JSON serialized for debugging. 41 // Fields are *not* intended to be used directly. 42 type NetworkIndex struct { 43 // TaskNetworks are the node networks available for 44 // task.resources.network asks. 45 TaskNetworks []*NetworkResource 46 47 // GroupNetworks are the node networks available for group.network 48 // asks. 49 GroupNetworks []*NodeNetworkResource 50 51 // HostNetworks indexes addresses by host network alias 52 HostNetworks map[string][]NodeNetworkAddress 53 54 // UsedPorts tracks which ports are used on a per-IP address basis. For 55 // example if a node has `network_interface=lo` and port 22 reserved, 56 // then on a dual stack loopback interface UsedPorts would contain: 57 // { 58 // "127.0.0.1": Bitmap{22}, 59 // "::1": Bitmap{22}, 60 // } 61 UsedPorts map[string]Bitmap 62 63 // Deprecated bandwidth fields 64 AvailBandwidth map[string]int // Bandwidth by device 65 UsedBandwidth map[string]int // Bandwidth by device 66 67 MinDynamicPort int // The smallest dynamic port generated 68 MaxDynamicPort int // The largest dynamic port generated 69 } 70 71 // NewNetworkIndex is used to construct a new network index 72 func NewNetworkIndex() *NetworkIndex { 73 return &NetworkIndex{ 74 HostNetworks: make(map[string][]NodeNetworkAddress), 75 UsedPorts: make(map[string]Bitmap), 76 AvailBandwidth: make(map[string]int), 77 UsedBandwidth: make(map[string]int), 78 MinDynamicPort: DefaultMinDynamicPort, 79 MaxDynamicPort: DefaultMaxDynamicPort, 80 } 81 } 82 83 func (idx *NetworkIndex) getUsedPortsFor(ip string) Bitmap { 84 used := idx.UsedPorts[ip] 85 if used == nil { 86 // Try to get a bitmap from the pool, else create 87 raw := bitmapPool.Get() 88 if raw != nil { 89 used = raw.(Bitmap) 90 used.Clear() 91 } else { 92 used, _ = NewBitmap(MaxValidPort) 93 } 94 idx.UsedPorts[ip] = used 95 } 96 return used 97 } 98 99 func (idx *NetworkIndex) Copy() *NetworkIndex { 100 if idx == nil { 101 return nil 102 } 103 104 c := new(NetworkIndex) 105 *c = *idx 106 107 c.TaskNetworks = copyNetworkResources(idx.TaskNetworks) 108 c.GroupNetworks = copyNodeNetworks(idx.GroupNetworks) 109 c.HostNetworks = copyAvailAddresses(idx.HostNetworks) 110 if idx.AvailBandwidth != nil && len(idx.AvailBandwidth) == 0 { 111 c.AvailBandwidth = make(map[string]int) 112 } else { 113 c.AvailBandwidth = maps.Clone(idx.AvailBandwidth) 114 } 115 if len(idx.UsedPorts) > 0 { 116 c.UsedPorts = make(map[string]Bitmap, len(idx.UsedPorts)) 117 for k, v := range idx.UsedPorts { 118 c.UsedPorts[k], _ = v.Copy() 119 } 120 } 121 if idx.UsedBandwidth != nil && len(idx.UsedBandwidth) == 0 { 122 c.UsedBandwidth = make(map[string]int) 123 } else { 124 c.UsedBandwidth = maps.Clone(idx.UsedBandwidth) 125 } 126 127 return c 128 } 129 130 func copyNetworkResources(resources []*NetworkResource) []*NetworkResource { 131 l := len(resources) 132 if l == 0 { 133 return nil 134 } 135 136 c := make([]*NetworkResource, l) 137 for i, resource := range resources { 138 c[i] = resource.Copy() 139 } 140 return c 141 } 142 143 func copyNodeNetworks(resources []*NodeNetworkResource) []*NodeNetworkResource { 144 l := len(resources) 145 if l == 0 { 146 return nil 147 } 148 149 c := make([]*NodeNetworkResource, l) 150 for i, resource := range resources { 151 c[i] = resource.Copy() 152 } 153 return c 154 } 155 156 func copyAvailAddresses(a map[string][]NodeNetworkAddress) map[string][]NodeNetworkAddress { 157 l := len(a) 158 if l == 0 { 159 return nil 160 } 161 162 c := make(map[string][]NodeNetworkAddress, l) 163 for k, v := range a { 164 if len(v) == 0 { 165 continue 166 } 167 c[k] = make([]NodeNetworkAddress, len(v)) 168 copy(c[k], v) 169 } 170 171 return c 172 } 173 174 // Release is called when the network index is no longer needed 175 // to attempt to re-use some of the memory it has allocated 176 func (idx *NetworkIndex) Release() { 177 for _, b := range idx.UsedPorts { 178 bitmapPool.Put(b) 179 } 180 } 181 182 // Overcommitted checks if the network is overcommitted 183 func (idx *NetworkIndex) Overcommitted() bool { 184 // TODO remove since bandwidth is deprecated 185 /*for device, used := range idx.UsedBandwidth { 186 avail := idx.AvailBandwidth[device] 187 if used > avail { 188 return true 189 } 190 }*/ 191 return false 192 } 193 194 // SetNode is used to initialize a node's network index with available IPs, 195 // reserved ports, and other details from a node's configuration and 196 // fingerprinting. 197 // 198 // SetNode must be idempotent as preemption causes SetNode to be called 199 // multiple times on the same NetworkIndex, only clearing UsedPorts between 200 // calls. 201 // 202 // An error is returned if the Node cannot produce a consistent NetworkIndex 203 // such as if reserved_ports are unparseable. 204 // 205 // Any errors returned by SetNode indicate a bug! The bug may lie in client 206 // code not properly validating its configuration or it may lie in improper 207 // Node object handling by servers. Users should not be able to cause SetNode 208 // to error. Data that cause SetNode to error should be caught upstream such as 209 // a client agent refusing to start with an invalid configuration. 210 func (idx *NetworkIndex) SetNode(node *Node) error { 211 212 // COMPAT(0.11): Deprecated. taskNetworks are only used for 213 // task.resources.network asks which have been deprecated since before 214 // 0.11. 215 // Grab the network resources, handling both new and old Node layouts 216 // from clients. 217 var taskNetworks []*NetworkResource 218 if node.NodeResources != nil && len(node.NodeResources.Networks) != 0 { 219 taskNetworks = node.NodeResources.Networks 220 } else if node.Resources != nil { 221 taskNetworks = node.Resources.Networks 222 } 223 224 // Reserved ports get merged downward. For example given an agent 225 // config: 226 // 227 // client.reserved.reserved_ports = "22" 228 // client.host_network["eth0"] = {reserved_ports = "80,443"} 229 // client.host_network["eth1"] = {reserved_ports = "1-1000"} 230 // 231 // Addresses on taskNetworks reserve port 22 232 // Addresses on eth0 reserve 22,80,443 (note 22 is also reserved!) 233 // Addresses on eth1 reserve 1-1000 234 globalResPorts := []uint{} 235 236 if node.ReservedResources != nil && node.ReservedResources.Networks.ReservedHostPorts != "" { 237 resPorts, err := ParsePortRanges(node.ReservedResources.Networks.ReservedHostPorts) 238 if err != nil { 239 // This is a fatal error that should have been 240 // prevented by client validation. 241 return fmt.Errorf("error parsing reserved_ports: %w", err) 242 } 243 244 globalResPorts = make([]uint, len(resPorts)) 245 for i, p := range resPorts { 246 globalResPorts[i] = uint(p) 247 } 248 } else if node.Reserved != nil { 249 // COMPAT(0.11): Remove after 0.11. Nodes stopped reporting 250 // reserved ports under Node.Reserved.Resources in #4750 / v0.9 251 for _, n := range node.Reserved.Networks { 252 used := idx.getUsedPortsFor(n.IP) 253 for _, ports := range [][]Port{n.ReservedPorts, n.DynamicPorts} { 254 for _, p := range ports { 255 if p.Value > MaxValidPort || p.Value < 0 { 256 // This is a fatal error that 257 // should have been prevented 258 // by validation upstream. 259 return fmt.Errorf("invalid port %d for reserved_ports", p.Value) 260 } 261 262 globalResPorts = append(globalResPorts, uint(p.Value)) 263 used.Set(uint(p.Value)) 264 } 265 } 266 267 // Reserve mbits 268 if n.Device != "" { 269 idx.UsedBandwidth[n.Device] += n.MBits 270 } 271 } 272 } 273 274 // Filter task networks down to those with a device. For example 275 // taskNetworks may contain a "bridge" interface which has no device 276 // set and cannot be used to fulfill asks. 277 for _, n := range taskNetworks { 278 if n.Device != "" { 279 idx.TaskNetworks = append(idx.TaskNetworks, n) 280 idx.AvailBandwidth[n.Device] = n.MBits 281 282 // Reserve ports 283 used := idx.getUsedPortsFor(n.IP) 284 for _, p := range globalResPorts { 285 used.Set(p) 286 } 287 } 288 } 289 290 // nodeNetworks are used for group.network asks. 291 var nodeNetworks []*NodeNetworkResource 292 if node.NodeResources != nil && len(node.NodeResources.NodeNetworks) != 0 { 293 nodeNetworks = node.NodeResources.NodeNetworks 294 } 295 296 for _, n := range nodeNetworks { 297 for _, a := range n.Addresses { 298 // Index host networks by their unique alias for asks 299 // with group.network.port.host_network set. 300 idx.HostNetworks[a.Alias] = append(idx.HostNetworks[a.Alias], a) 301 302 // Mark reserved ports as used without worrying about 303 // collisions. This effectively merges 304 // client.reserved.reserved_ports into each 305 // host_network. 306 used := idx.getUsedPortsFor(a.Address) 307 for _, p := range globalResPorts { 308 used.Set(p) 309 } 310 311 // If ReservedPorts is set on the NodeNetwork, use it 312 // and the global reserved ports. 313 if a.ReservedPorts != "" { 314 rp, err := ParsePortRanges(a.ReservedPorts) 315 if err != nil { 316 // This is a fatal error that should 317 // have been prevented by validation 318 // upstream. 319 return fmt.Errorf("error parsing reserved_ports for network %q: %w", a.Alias, err) 320 } 321 for _, p := range rp { 322 used.Set(uint(p)) 323 } 324 } 325 } 326 } 327 328 // Set dynamic port range (applies to all addresses) 329 if node.NodeResources != nil && node.NodeResources.MinDynamicPort > 0 { 330 idx.MinDynamicPort = node.NodeResources.MinDynamicPort 331 } 332 333 if node.NodeResources != nil && node.NodeResources.MaxDynamicPort > 0 { 334 idx.MaxDynamicPort = node.NodeResources.MaxDynamicPort 335 } 336 337 return nil 338 } 339 340 // AddAllocs is used to add the used network resources. Returns 341 // true if there is a collision 342 // 343 // AddAllocs may be called multiple times for the same NetworkIndex with 344 // UsedPorts cleared between calls (by Release). Therefore AddAllocs must be 345 // determistic and must not manipulate state outside of UsedPorts as that state 346 // would persist between Release calls. 347 func (idx *NetworkIndex) AddAllocs(allocs []*Allocation) (collide bool, reason string) { 348 for _, alloc := range allocs { 349 // Do not consider the resource impact of terminal allocations 350 if alloc.ClientTerminalStatus() { 351 continue 352 } 353 354 if alloc.AllocatedResources != nil { 355 // Only look at AllocatedPorts if populated, otherwise use pre 0.12 logic 356 // COMPAT(1.0): Remove when network resources struct is removed. 357 if len(alloc.AllocatedResources.Shared.Ports) > 0 { 358 if c, r := idx.AddReservedPorts(alloc.AllocatedResources.Shared.Ports); c { 359 collide = true 360 reason = fmt.Sprintf("collision when reserving port for alloc %s: %v", alloc.ID, r) 361 } 362 } else { 363 // Add network resources that are at the task group level 364 if len(alloc.AllocatedResources.Shared.Networks) > 0 { 365 for _, network := range alloc.AllocatedResources.Shared.Networks { 366 if c, r := idx.AddReserved(network); c { 367 collide = true 368 reason = fmt.Sprintf("collision when reserving port for network %s in alloc %s: %v", network.IP, alloc.ID, r) 369 } 370 } 371 } 372 373 for task, resources := range alloc.AllocatedResources.Tasks { 374 if len(resources.Networks) == 0 { 375 continue 376 } 377 n := resources.Networks[0] 378 if c, r := idx.AddReserved(n); c { 379 collide = true 380 reason = fmt.Sprintf("collision when reserving port for network %s in task %s of alloc %s: %v", n.IP, task, alloc.ID, r) 381 } 382 } 383 } 384 } else { 385 // COMPAT(0.11): Remove in 0.11 386 for task, resources := range alloc.TaskResources { 387 if len(resources.Networks) == 0 { 388 continue 389 } 390 n := resources.Networks[0] 391 if c, r := idx.AddReserved(n); c { 392 collide = true 393 reason = fmt.Sprintf("(deprecated) collision when reserving port for network %s in task %s of alloc %s: %v", n.IP, task, alloc.ID, r) 394 } 395 } 396 } 397 } 398 return 399 } 400 401 // AddReserved is used to add a reserved network usage, returns true 402 // if there is a port collision 403 func (idx *NetworkIndex) AddReserved(n *NetworkResource) (collide bool, reasons []string) { 404 // Add the port usage 405 used := idx.getUsedPortsFor(n.IP) 406 407 for _, ports := range [][]Port{n.ReservedPorts, n.DynamicPorts} { 408 for _, port := range ports { 409 // Guard against invalid port 410 if port.Value < 0 || port.Value >= MaxValidPort { 411 return true, []string{fmt.Sprintf("invalid port %d", port.Value)} 412 } 413 if used.Check(uint(port.Value)) { 414 collide = true 415 reason := fmt.Sprintf("port %d already in use", port.Value) 416 reasons = append(reasons, reason) 417 } else { 418 used.Set(uint(port.Value)) 419 } 420 } 421 } 422 423 // Add the bandwidth 424 idx.UsedBandwidth[n.Device] += n.MBits 425 return 426 } 427 428 func (idx *NetworkIndex) AddReservedPorts(ports AllocatedPorts) (collide bool, reasons []string) { 429 for _, port := range ports { 430 used := idx.getUsedPortsFor(port.HostIP) 431 if port.Value < 0 || port.Value >= MaxValidPort { 432 return true, []string{fmt.Sprintf("invalid port %d", port.Value)} 433 } 434 if used.Check(uint(port.Value)) { 435 collide = true 436 reason := fmt.Sprintf("port %d already in use", port.Value) 437 reasons = append(reasons, reason) 438 } else { 439 used.Set(uint(port.Value)) 440 } 441 } 442 443 return 444 } 445 446 // AddReservedPortsForIP checks whether any reserved ports collide with those 447 // in use for the IP address. 448 func (idx *NetworkIndex) AddReservedPortsForIP(ports []uint64, ip string) (collide bool, reasons []string) { 449 used := idx.getUsedPortsFor(ip) 450 for _, port := range ports { 451 // Guard against invalid port 452 if port >= MaxValidPort { 453 return true, []string{fmt.Sprintf("invalid port %d", port)} 454 } 455 if used.Check(uint(port)) { 456 collide = true 457 reason := fmt.Sprintf("port %d already in use", port) 458 reasons = append(reasons, reason) 459 } else { 460 used.Set(uint(port)) 461 } 462 } 463 464 return 465 } 466 467 // yieldIP is used to iteratively invoke the callback with 468 // an available IP 469 func (idx *NetworkIndex) yieldIP(cb func(net *NetworkResource, offerIP net.IP) bool) { 470 for _, n := range idx.TaskNetworks { 471 ip, ipnet, err := net.ParseCIDR(n.CIDR) 472 if err != nil { 473 continue 474 } 475 for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); incIP(ip) { 476 if cb(n, ip) { 477 return 478 } 479 } 480 } 481 } 482 483 func incIP(ip net.IP) { 484 // Iterate over IP octects from right to left 485 for j := len(ip) - 1; j >= 0; j-- { 486 487 // Increment octect 488 ip[j]++ 489 490 // If this octect did not wrap around to 0, it's the next IP to 491 // try. If it did wrap (p[j]==0), then the next octect is 492 // incremented. 493 if ip[j] > 0 { 494 break 495 } 496 } 497 } 498 499 // AssignPorts based on an ask from the scheduler processing a group.network 500 // stanza. Supports multi-interfaces through node configured host_networks. 501 // 502 // AssignTaskNetwork supports the deprecated task.resources.network stanza. 503 func (idx *NetworkIndex) AssignPorts(ask *NetworkResource) (AllocatedPorts, error) { 504 var offer AllocatedPorts 505 506 // index of host network name to slice of reserved ports, used during dynamic port assignment 507 reservedIdx := map[string][]Port{} 508 509 for _, port := range ask.ReservedPorts { 510 reservedIdx[port.HostNetwork] = append(reservedIdx[port.HostNetwork], port) 511 512 // allocPort is set in the inner for loop if a port mapping can be created 513 // if allocPort is still nil after the loop, the port wasn't available for reservation 514 var allocPort *AllocatedPortMapping 515 var addrErr error 516 for _, addr := range idx.HostNetworks[port.HostNetwork] { 517 used := idx.getUsedPortsFor(addr.Address) 518 // Guard against invalid port 519 if port.Value < 0 || port.Value >= MaxValidPort { 520 return nil, fmt.Errorf("invalid port %d (out of range)", port.Value) 521 } 522 523 // Check if in use 524 if used != nil && used.Check(uint(port.Value)) { 525 return nil, fmt.Errorf("reserved port collision %s=%d", port.Label, port.Value) 526 } 527 528 allocPort = &AllocatedPortMapping{ 529 Label: port.Label, 530 Value: port.Value, 531 To: port.To, 532 HostIP: addr.Address, 533 } 534 break 535 } 536 537 if allocPort == nil { 538 if addrErr != nil { 539 return nil, addrErr 540 } 541 542 return nil, fmt.Errorf("no addresses available for %s network", port.HostNetwork) 543 } 544 545 offer = append(offer, *allocPort) 546 } 547 548 for _, port := range ask.DynamicPorts { 549 var allocPort *AllocatedPortMapping 550 var addrErr error 551 for _, addr := range idx.HostNetworks[port.HostNetwork] { 552 used := idx.getUsedPortsFor(addr.Address) 553 // Try to stochastically pick the dynamic ports as it is faster and 554 // lower memory usage. 555 var dynPorts []int 556 // TODO: its more efficient to find multiple dynamic ports at once 557 dynPorts, addrErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1) 558 if addrErr != nil { 559 // Fall back to the precise method if the random sampling failed. 560 dynPorts, addrErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1) 561 if addrErr != nil { 562 continue 563 } 564 } 565 566 allocPort = &AllocatedPortMapping{ 567 Label: port.Label, 568 Value: dynPorts[0], 569 To: port.To, 570 HostIP: addr.Address, 571 } 572 if allocPort.To == -1 { 573 allocPort.To = allocPort.Value 574 } 575 break 576 } 577 578 if allocPort == nil { 579 if addrErr != nil { 580 return nil, addrErr 581 } 582 583 return nil, fmt.Errorf("no addresses available for %s network", port.HostNetwork) 584 } 585 offer = append(offer, *allocPort) 586 } 587 588 return offer, nil 589 } 590 591 // AssignTaskNetwork is used to offer network resources given a 592 // task.resources.network ask. If the ask cannot be satisfied, returns nil 593 // 594 // AssignTaskNetwork and task.resources.network are deprecated in favor of 595 // AssignPorts and group.network. AssignTaskNetwork does not support multiple 596 // interfaces and only uses the node's default interface. AssignPorts is the 597 // method that is used for group.network asks. 598 func (idx *NetworkIndex) AssignTaskNetwork(ask *NetworkResource) (out *NetworkResource, err error) { 599 err = fmt.Errorf("no networks available") 600 idx.yieldIP(func(n *NetworkResource, offerIP net.IP) (stop bool) { 601 // Convert the IP to a string 602 offerIPStr := offerIP.String() 603 604 // Check if we would exceed the bandwidth cap 605 availBandwidth := idx.AvailBandwidth[n.Device] 606 usedBandwidth := idx.UsedBandwidth[n.Device] 607 if usedBandwidth+ask.MBits > availBandwidth { 608 err = fmt.Errorf("bandwidth exceeded") 609 return 610 } 611 612 used := idx.UsedPorts[offerIPStr] 613 614 // Check if any of the reserved ports are in use 615 for _, port := range ask.ReservedPorts { 616 // Guard against invalid port 617 if port.Value < 0 || port.Value >= MaxValidPort { 618 err = fmt.Errorf("invalid port %d (out of range)", port.Value) 619 return 620 } 621 622 // Check if in use 623 if used != nil && used.Check(uint(port.Value)) { 624 err = fmt.Errorf("reserved port collision %s=%d", port.Label, port.Value) 625 return 626 } 627 } 628 629 // Create the offer 630 offer := &NetworkResource{ 631 Mode: ask.Mode, 632 Device: n.Device, 633 IP: offerIPStr, 634 MBits: ask.MBits, 635 DNS: ask.DNS, 636 ReservedPorts: ask.ReservedPorts, 637 DynamicPorts: ask.DynamicPorts, 638 } 639 640 // Try to stochastically pick the dynamic ports as it is faster and 641 // lower memory usage. 642 var dynPorts []int 643 var dynErr error 644 dynPorts, dynErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts)) 645 if dynErr == nil { 646 goto BUILD_OFFER 647 } 648 649 // Fall back to the precise method if the random sampling failed. 650 dynPorts, dynErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts)) 651 if dynErr != nil { 652 err = dynErr 653 return 654 } 655 656 BUILD_OFFER: 657 for i, port := range dynPorts { 658 offer.DynamicPorts[i].Value = port 659 660 // This syntax allows you to set the mapped to port to the same port 661 // allocated by the scheduler on the host. 662 if offer.DynamicPorts[i].To == -1 { 663 offer.DynamicPorts[i].To = port 664 } 665 } 666 667 // Stop, we have an offer! 668 out = offer 669 err = nil 670 return true 671 }) 672 return 673 } 674 675 // getDynamicPortsPrecise takes the nodes used port bitmap which may be nil if 676 // no ports have been allocated yet, the network ask and returns a set of unused 677 // ports to fulfil the ask's DynamicPorts or an error if it failed. An error 678 // means the ask can not be satisfied as the method does a precise search. 679 func getDynamicPortsPrecise(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reserved []Port, numDyn int) ([]int, error) { 680 // Create a copy of the used ports and apply the new reserves 681 var usedSet Bitmap 682 var err error 683 if nodeUsed != nil { 684 usedSet, err = nodeUsed.Copy() 685 if err != nil { 686 return nil, err 687 } 688 } else { 689 usedSet, err = NewBitmap(MaxValidPort) 690 if err != nil { 691 return nil, err 692 } 693 } 694 695 for _, port := range reserved { 696 usedSet.Set(uint(port.Value)) 697 } 698 699 // Get the indexes of the unset 700 availablePorts := usedSet.IndexesInRange(false, uint(minDynamicPort), uint(maxDynamicPort)) 701 702 // Randomize the amount we need 703 if len(availablePorts) < numDyn { 704 return nil, fmt.Errorf("dynamic port selection failed") 705 } 706 707 numAvailable := len(availablePorts) 708 for i := 0; i < numDyn; i++ { 709 j := rand.Intn(numAvailable) 710 availablePorts[i], availablePorts[j] = availablePorts[j], availablePorts[i] 711 } 712 713 return availablePorts[:numDyn], nil 714 } 715 716 // getDynamicPortsStochastic takes the nodes used port bitmap which may be nil if 717 // no ports have been allocated yet, the network ask and returns a set of unused 718 // ports to fulfil the ask's DynamicPorts or an error if it failed. An error 719 // does not mean the ask can not be satisfied as the method has a fixed amount 720 // of random probes and if these fail, the search is aborted. 721 func getDynamicPortsStochastic(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reservedPorts []Port, count int) ([]int, error) { 722 var reserved, dynamic []int 723 for _, port := range reservedPorts { 724 reserved = append(reserved, port.Value) 725 } 726 727 for i := 0; i < count; i++ { 728 attempts := 0 729 PICK: 730 attempts++ 731 if attempts > maxRandPortAttempts { 732 return nil, fmt.Errorf("stochastic dynamic port selection failed") 733 } 734 735 randPort := minDynamicPort + rand.Intn(maxDynamicPort-minDynamicPort) 736 if nodeUsed != nil && nodeUsed.Check(uint(randPort)) { 737 goto PICK 738 } 739 740 for _, ports := range [][]int{reserved, dynamic} { 741 if isPortReserved(ports, randPort) { 742 goto PICK 743 } 744 } 745 dynamic = append(dynamic, randPort) 746 } 747 748 return dynamic, nil 749 } 750 751 // IntContains scans an integer slice for a value 752 func isPortReserved(haystack []int, needle int) bool { 753 for _, item := range haystack { 754 if item == needle { 755 return true 756 } 757 } 758 return false 759 } 760 761 // AllocatedPortsToNetworkResouce is a COMPAT(1.0) remove when NetworkResource 762 // is no longer used for materialized client view of ports. 763 func AllocatedPortsToNetworkResouce(ask *NetworkResource, ports AllocatedPorts, node *NodeResources) *NetworkResource { 764 out := ask.Copy() 765 766 for i, port := range ask.DynamicPorts { 767 if p, ok := ports.Get(port.Label); ok { 768 out.DynamicPorts[i].Value = p.Value 769 out.DynamicPorts[i].To = p.To 770 } 771 } 772 if len(node.NodeNetworks) > 0 { 773 for _, nw := range node.NodeNetworks { 774 if nw.Mode == "host" { 775 out.IP = nw.Addresses[0].Address 776 break 777 } 778 } 779 } else { 780 for _, nw := range node.Networks { 781 if nw.Mode == "host" { 782 out.IP = nw.IP 783 } 784 } 785 } 786 return out 787 } 788 789 type ClientHostNetworkConfig struct { 790 Name string `hcl:",key"` 791 CIDR string `hcl:"cidr"` 792 Interface string `hcl:"interface"` 793 ReservedPorts string `hcl:"reserved_ports"` 794 } 795 796 func (p *ClientHostNetworkConfig) Copy() *ClientHostNetworkConfig { 797 if p == nil { 798 return nil 799 } 800 801 c := new(ClientHostNetworkConfig) 802 *c = *p 803 return c 804 }