github.com/anth0d/nomad@v0.0.0-20221214183521-ae3a0a2cad06/nomad/structs/network.go (about)

     1  package structs
     2  
     3  import (
     4  	"fmt"
     5  	"math/rand"
     6  	"net"
     7  	"sync"
     8  
     9  	"golang.org/x/exp/maps"
    10  )
    11  
    12  const (
    13  	// DefaultMinDynamicPort is the smallest dynamic port generated by
    14  	// default
    15  	DefaultMinDynamicPort = 20000
    16  
    17  	// DefaultMaxDynamicPort is the largest dynamic port generated by
    18  	// default
    19  	DefaultMaxDynamicPort = 32000
    20  
    21  	// maxRandPortAttempts is the maximum number of attempt
    22  	// to assign a random port
    23  	maxRandPortAttempts = 20
    24  
    25  	// MaxValidPort is the max valid port number
    26  	MaxValidPort = 65536
    27  )
    28  
    29  var (
    30  	// bitmapPool is used to pool the bitmaps used for port collision
    31  	// checking. They are fairly large (8K) so we can re-use them to
    32  	// avoid GC pressure. Care should be taken to call Clear() on any
    33  	// bitmap coming from the pool.
    34  	bitmapPool = new(sync.Pool)
    35  )
    36  
    37  // NetworkIndex is used to index the available network resources
    38  // and the used network resources on a machine given allocations
    39  //
    40  // Fields are exported so they may be JSON serialized for debugging.
    41  // Fields are *not* intended to be used directly.
    42  type NetworkIndex struct {
    43  	// TaskNetworks are the node networks available for
    44  	// task.resources.network asks.
    45  	TaskNetworks []*NetworkResource
    46  
    47  	// GroupNetworks are the node networks available for group.network
    48  	// asks.
    49  	GroupNetworks []*NodeNetworkResource
    50  
    51  	// HostNetworks indexes addresses by host network alias
    52  	HostNetworks map[string][]NodeNetworkAddress
    53  
    54  	// UsedPorts tracks which ports are used on a per-IP address basis. For
    55  	// example if a node has `network_interface=lo` and port 22 reserved,
    56  	// then on a dual stack loopback interface UsedPorts would contain:
    57  	// {
    58  	//  "127.0.0.1": Bitmap{22},
    59  	//  "::1":       Bitmap{22},
    60  	// }
    61  	UsedPorts map[string]Bitmap
    62  
    63  	// Deprecated bandwidth fields
    64  	AvailBandwidth map[string]int // Bandwidth by device
    65  	UsedBandwidth  map[string]int // Bandwidth by device
    66  
    67  	MinDynamicPort int // The smallest dynamic port generated
    68  	MaxDynamicPort int // The largest dynamic port generated
    69  }
    70  
    71  // NewNetworkIndex is used to construct a new network index
    72  func NewNetworkIndex() *NetworkIndex {
    73  	return &NetworkIndex{
    74  		HostNetworks:   make(map[string][]NodeNetworkAddress),
    75  		UsedPorts:      make(map[string]Bitmap),
    76  		AvailBandwidth: make(map[string]int),
    77  		UsedBandwidth:  make(map[string]int),
    78  		MinDynamicPort: DefaultMinDynamicPort,
    79  		MaxDynamicPort: DefaultMaxDynamicPort,
    80  	}
    81  }
    82  
    83  func (idx *NetworkIndex) getUsedPortsFor(ip string) Bitmap {
    84  	used := idx.UsedPorts[ip]
    85  	if used == nil {
    86  		// Try to get a bitmap from the pool, else create
    87  		raw := bitmapPool.Get()
    88  		if raw != nil {
    89  			used = raw.(Bitmap)
    90  			used.Clear()
    91  		} else {
    92  			used, _ = NewBitmap(MaxValidPort)
    93  		}
    94  		idx.UsedPorts[ip] = used
    95  	}
    96  	return used
    97  }
    98  
    99  func (idx *NetworkIndex) Copy() *NetworkIndex {
   100  	if idx == nil {
   101  		return nil
   102  	}
   103  
   104  	c := new(NetworkIndex)
   105  	*c = *idx
   106  
   107  	c.TaskNetworks = copyNetworkResources(idx.TaskNetworks)
   108  	c.GroupNetworks = copyNodeNetworks(idx.GroupNetworks)
   109  	c.HostNetworks = copyAvailAddresses(idx.HostNetworks)
   110  	if idx.AvailBandwidth != nil && len(idx.AvailBandwidth) == 0 {
   111  		c.AvailBandwidth = make(map[string]int)
   112  	} else {
   113  		c.AvailBandwidth = maps.Clone(idx.AvailBandwidth)
   114  	}
   115  	if len(idx.UsedPorts) > 0 {
   116  		c.UsedPorts = make(map[string]Bitmap, len(idx.UsedPorts))
   117  		for k, v := range idx.UsedPorts {
   118  			c.UsedPorts[k], _ = v.Copy()
   119  		}
   120  	}
   121  	if idx.UsedBandwidth != nil && len(idx.UsedBandwidth) == 0 {
   122  		c.UsedBandwidth = make(map[string]int)
   123  	} else {
   124  		c.UsedBandwidth = maps.Clone(idx.UsedBandwidth)
   125  	}
   126  
   127  	return c
   128  }
   129  
   130  func copyNetworkResources(resources []*NetworkResource) []*NetworkResource {
   131  	l := len(resources)
   132  	if l == 0 {
   133  		return nil
   134  	}
   135  
   136  	c := make([]*NetworkResource, l)
   137  	for i, resource := range resources {
   138  		c[i] = resource.Copy()
   139  	}
   140  	return c
   141  }
   142  
   143  func copyNodeNetworks(resources []*NodeNetworkResource) []*NodeNetworkResource {
   144  	l := len(resources)
   145  	if l == 0 {
   146  		return nil
   147  	}
   148  
   149  	c := make([]*NodeNetworkResource, l)
   150  	for i, resource := range resources {
   151  		c[i] = resource.Copy()
   152  	}
   153  	return c
   154  }
   155  
   156  func copyAvailAddresses(a map[string][]NodeNetworkAddress) map[string][]NodeNetworkAddress {
   157  	l := len(a)
   158  	if l == 0 {
   159  		return nil
   160  	}
   161  
   162  	c := make(map[string][]NodeNetworkAddress, l)
   163  	for k, v := range a {
   164  		if len(v) == 0 {
   165  			continue
   166  		}
   167  		c[k] = make([]NodeNetworkAddress, len(v))
   168  		copy(c[k], v)
   169  	}
   170  
   171  	return c
   172  }
   173  
   174  // Release is called when the network index is no longer needed
   175  // to attempt to re-use some of the memory it has allocated
   176  func (idx *NetworkIndex) Release() {
   177  	for _, b := range idx.UsedPorts {
   178  		bitmapPool.Put(b)
   179  	}
   180  }
   181  
   182  // Overcommitted checks if the network is overcommitted
   183  func (idx *NetworkIndex) Overcommitted() bool {
   184  	// TODO remove since bandwidth is deprecated
   185  	/*for device, used := range idx.UsedBandwidth {
   186  		avail := idx.AvailBandwidth[device]
   187  		if used > avail {
   188  			return true
   189  		}
   190  	}*/
   191  	return false
   192  }
   193  
   194  // SetNode is used to initialize a node's network index with available IPs,
   195  // reserved ports, and other details from a node's configuration and
   196  // fingerprinting.
   197  //
   198  // SetNode must be idempotent as preemption causes SetNode to be called
   199  // multiple times on the same NetworkIndex, only clearing UsedPorts between
   200  // calls.
   201  //
   202  // An error is returned if the Node cannot produce a consistent NetworkIndex
   203  // such as if reserved_ports are unparseable.
   204  //
   205  // Any errors returned by SetNode indicate a bug! The bug may lie in client
   206  // code not properly validating its configuration or it may lie in improper
   207  // Node object handling by servers. Users should not be able to cause SetNode
   208  // to error. Data that cause SetNode to error should be caught upstream such as
   209  // a client agent refusing to start with an invalid configuration.
   210  func (idx *NetworkIndex) SetNode(node *Node) error {
   211  
   212  	// COMPAT(0.11): Deprecated. taskNetworks are only used for
   213  	// task.resources.network asks which have been deprecated since before
   214  	// 0.11.
   215  	// Grab the network resources, handling both new and old Node layouts
   216  	// from clients.
   217  	var taskNetworks []*NetworkResource
   218  	if node.NodeResources != nil && len(node.NodeResources.Networks) != 0 {
   219  		taskNetworks = node.NodeResources.Networks
   220  	} else if node.Resources != nil {
   221  		taskNetworks = node.Resources.Networks
   222  	}
   223  
   224  	// Reserved ports get merged downward. For example given an agent
   225  	// config:
   226  	//
   227  	// client.reserved.reserved_ports = "22"
   228  	// client.host_network["eth0"] = {reserved_ports = "80,443"}
   229  	// client.host_network["eth1"] = {reserved_ports = "1-1000"}
   230  	//
   231  	// Addresses on taskNetworks reserve port 22
   232  	// Addresses on eth0 reserve 22,80,443 (note 22 is also reserved!)
   233  	// Addresses on eth1 reserve 1-1000
   234  	globalResPorts := []uint{}
   235  
   236  	if node.ReservedResources != nil && node.ReservedResources.Networks.ReservedHostPorts != "" {
   237  		resPorts, err := ParsePortRanges(node.ReservedResources.Networks.ReservedHostPorts)
   238  		if err != nil {
   239  			// This is a fatal error that should have been
   240  			// prevented by client validation.
   241  			return fmt.Errorf("error parsing reserved_ports: %w", err)
   242  		}
   243  
   244  		globalResPorts = make([]uint, len(resPorts))
   245  		for i, p := range resPorts {
   246  			globalResPorts[i] = uint(p)
   247  		}
   248  	} else if node.Reserved != nil {
   249  		// COMPAT(0.11): Remove after 0.11. Nodes stopped reporting
   250  		// reserved ports under Node.Reserved.Resources in #4750 / v0.9
   251  		for _, n := range node.Reserved.Networks {
   252  			used := idx.getUsedPortsFor(n.IP)
   253  			for _, ports := range [][]Port{n.ReservedPorts, n.DynamicPorts} {
   254  				for _, p := range ports {
   255  					if p.Value > MaxValidPort || p.Value < 0 {
   256  						// This is a fatal error that
   257  						// should have been prevented
   258  						// by validation upstream.
   259  						return fmt.Errorf("invalid port %d for reserved_ports", p.Value)
   260  					}
   261  
   262  					globalResPorts = append(globalResPorts, uint(p.Value))
   263  					used.Set(uint(p.Value))
   264  				}
   265  			}
   266  
   267  			// Reserve mbits
   268  			if n.Device != "" {
   269  				idx.UsedBandwidth[n.Device] += n.MBits
   270  			}
   271  		}
   272  	}
   273  
   274  	// Filter task networks down to those with a device. For example
   275  	// taskNetworks may contain a "bridge" interface which has no device
   276  	// set and cannot be used to fulfill asks.
   277  	for _, n := range taskNetworks {
   278  		if n.Device != "" {
   279  			idx.TaskNetworks = append(idx.TaskNetworks, n)
   280  			idx.AvailBandwidth[n.Device] = n.MBits
   281  
   282  			// Reserve ports
   283  			used := idx.getUsedPortsFor(n.IP)
   284  			for _, p := range globalResPorts {
   285  				used.Set(p)
   286  			}
   287  		}
   288  	}
   289  
   290  	// nodeNetworks are used for group.network asks.
   291  	var nodeNetworks []*NodeNetworkResource
   292  	if node.NodeResources != nil && len(node.NodeResources.NodeNetworks) != 0 {
   293  		nodeNetworks = node.NodeResources.NodeNetworks
   294  	}
   295  
   296  	for _, n := range nodeNetworks {
   297  		for _, a := range n.Addresses {
   298  			// Index host networks by their unique alias for asks
   299  			// with group.network.port.host_network set.
   300  			idx.HostNetworks[a.Alias] = append(idx.HostNetworks[a.Alias], a)
   301  
   302  			// Mark reserved ports as used without worrying about
   303  			// collisions. This effectively merges
   304  			// client.reserved.reserved_ports into each
   305  			// host_network.
   306  			used := idx.getUsedPortsFor(a.Address)
   307  			for _, p := range globalResPorts {
   308  				used.Set(p)
   309  			}
   310  
   311  			// If ReservedPorts is set on the NodeNetwork, use it
   312  			// and the global reserved ports.
   313  			if a.ReservedPorts != "" {
   314  				rp, err := ParsePortRanges(a.ReservedPorts)
   315  				if err != nil {
   316  					// This is a fatal error that should
   317  					// have been prevented by validation
   318  					// upstream.
   319  					return fmt.Errorf("error parsing reserved_ports for network %q: %w", a.Alias, err)
   320  				}
   321  				for _, p := range rp {
   322  					used.Set(uint(p))
   323  				}
   324  			}
   325  		}
   326  	}
   327  
   328  	// Set dynamic port range (applies to all addresses)
   329  	if node.NodeResources != nil && node.NodeResources.MinDynamicPort > 0 {
   330  		idx.MinDynamicPort = node.NodeResources.MinDynamicPort
   331  	}
   332  
   333  	if node.NodeResources != nil && node.NodeResources.MaxDynamicPort > 0 {
   334  		idx.MaxDynamicPort = node.NodeResources.MaxDynamicPort
   335  	}
   336  
   337  	return nil
   338  }
   339  
   340  // AddAllocs is used to add the used network resources. Returns
   341  // true if there is a collision
   342  //
   343  // AddAllocs may be called multiple times for the same NetworkIndex with
   344  // UsedPorts cleared between calls (by Release). Therefore AddAllocs must be
   345  // determistic and must not manipulate state outside of UsedPorts as that state
   346  // would persist between Release calls.
   347  func (idx *NetworkIndex) AddAllocs(allocs []*Allocation) (collide bool, reason string) {
   348  	for _, alloc := range allocs {
   349  		// Do not consider the resource impact of terminal allocations
   350  		if alloc.ClientTerminalStatus() {
   351  			continue
   352  		}
   353  
   354  		if alloc.AllocatedResources != nil {
   355  			// Only look at AllocatedPorts if populated, otherwise use pre 0.12 logic
   356  			// COMPAT(1.0): Remove when network resources struct is removed.
   357  			if len(alloc.AllocatedResources.Shared.Ports) > 0 {
   358  				if c, r := idx.AddReservedPorts(alloc.AllocatedResources.Shared.Ports); c {
   359  					collide = true
   360  					reason = fmt.Sprintf("collision when reserving port for alloc %s: %v", alloc.ID, r)
   361  				}
   362  			} else {
   363  				// Add network resources that are at the task group level
   364  				if len(alloc.AllocatedResources.Shared.Networks) > 0 {
   365  					for _, network := range alloc.AllocatedResources.Shared.Networks {
   366  						if c, r := idx.AddReserved(network); c {
   367  							collide = true
   368  							reason = fmt.Sprintf("collision when reserving port for network %s in alloc %s: %v", network.IP, alloc.ID, r)
   369  						}
   370  					}
   371  				}
   372  
   373  				for task, resources := range alloc.AllocatedResources.Tasks {
   374  					if len(resources.Networks) == 0 {
   375  						continue
   376  					}
   377  					n := resources.Networks[0]
   378  					if c, r := idx.AddReserved(n); c {
   379  						collide = true
   380  						reason = fmt.Sprintf("collision when reserving port for network %s in task %s of alloc %s: %v", n.IP, task, alloc.ID, r)
   381  					}
   382  				}
   383  			}
   384  		} else {
   385  			// COMPAT(0.11): Remove in 0.11
   386  			for task, resources := range alloc.TaskResources {
   387  				if len(resources.Networks) == 0 {
   388  					continue
   389  				}
   390  				n := resources.Networks[0]
   391  				if c, r := idx.AddReserved(n); c {
   392  					collide = true
   393  					reason = fmt.Sprintf("(deprecated) collision when reserving port for network %s in task %s of alloc %s: %v", n.IP, task, alloc.ID, r)
   394  				}
   395  			}
   396  		}
   397  	}
   398  	return
   399  }
   400  
   401  // AddReserved is used to add a reserved network usage, returns true
   402  // if there is a port collision
   403  func (idx *NetworkIndex) AddReserved(n *NetworkResource) (collide bool, reasons []string) {
   404  	// Add the port usage
   405  	used := idx.getUsedPortsFor(n.IP)
   406  
   407  	for _, ports := range [][]Port{n.ReservedPorts, n.DynamicPorts} {
   408  		for _, port := range ports {
   409  			// Guard against invalid port
   410  			if port.Value < 0 || port.Value >= MaxValidPort {
   411  				return true, []string{fmt.Sprintf("invalid port %d", port.Value)}
   412  			}
   413  			if used.Check(uint(port.Value)) {
   414  				collide = true
   415  				reason := fmt.Sprintf("port %d already in use", port.Value)
   416  				reasons = append(reasons, reason)
   417  			} else {
   418  				used.Set(uint(port.Value))
   419  			}
   420  		}
   421  	}
   422  
   423  	// Add the bandwidth
   424  	idx.UsedBandwidth[n.Device] += n.MBits
   425  	return
   426  }
   427  
   428  func (idx *NetworkIndex) AddReservedPorts(ports AllocatedPorts) (collide bool, reasons []string) {
   429  	for _, port := range ports {
   430  		used := idx.getUsedPortsFor(port.HostIP)
   431  		if port.Value < 0 || port.Value >= MaxValidPort {
   432  			return true, []string{fmt.Sprintf("invalid port %d", port.Value)}
   433  		}
   434  		if used.Check(uint(port.Value)) {
   435  			collide = true
   436  			reason := fmt.Sprintf("port %d already in use", port.Value)
   437  			reasons = append(reasons, reason)
   438  		} else {
   439  			used.Set(uint(port.Value))
   440  		}
   441  	}
   442  
   443  	return
   444  }
   445  
   446  // AddReservedPortsForIP checks whether any reserved ports collide with those
   447  // in use for the IP address.
   448  func (idx *NetworkIndex) AddReservedPortsForIP(ports []uint64, ip string) (collide bool, reasons []string) {
   449  	used := idx.getUsedPortsFor(ip)
   450  	for _, port := range ports {
   451  		// Guard against invalid port
   452  		if port >= MaxValidPort {
   453  			return true, []string{fmt.Sprintf("invalid port %d", port)}
   454  		}
   455  		if used.Check(uint(port)) {
   456  			collide = true
   457  			reason := fmt.Sprintf("port %d already in use", port)
   458  			reasons = append(reasons, reason)
   459  		} else {
   460  			used.Set(uint(port))
   461  		}
   462  	}
   463  
   464  	return
   465  }
   466  
   467  // yieldIP is used to iteratively invoke the callback with
   468  // an available IP
   469  func (idx *NetworkIndex) yieldIP(cb func(net *NetworkResource, offerIP net.IP) bool) {
   470  	for _, n := range idx.TaskNetworks {
   471  		ip, ipnet, err := net.ParseCIDR(n.CIDR)
   472  		if err != nil {
   473  			continue
   474  		}
   475  		for ip := ip.Mask(ipnet.Mask); ipnet.Contains(ip); incIP(ip) {
   476  			if cb(n, ip) {
   477  				return
   478  			}
   479  		}
   480  	}
   481  }
   482  
   483  func incIP(ip net.IP) {
   484  	// Iterate over IP octects from right to left
   485  	for j := len(ip) - 1; j >= 0; j-- {
   486  
   487  		// Increment octect
   488  		ip[j]++
   489  
   490  		// If this octect did not wrap around to 0, it's the next IP to
   491  		// try. If it did wrap (p[j]==0), then the next octect is
   492  		// incremented.
   493  		if ip[j] > 0 {
   494  			break
   495  		}
   496  	}
   497  }
   498  
   499  // AssignPorts based on an ask from the scheduler processing a group.network
   500  // stanza. Supports multi-interfaces through node configured host_networks.
   501  //
   502  // AssignTaskNetwork supports the deprecated task.resources.network stanza.
   503  func (idx *NetworkIndex) AssignPorts(ask *NetworkResource) (AllocatedPorts, error) {
   504  	var offer AllocatedPorts
   505  
   506  	// index of host network name to slice of reserved ports, used during dynamic port assignment
   507  	reservedIdx := map[string][]Port{}
   508  
   509  	for _, port := range ask.ReservedPorts {
   510  		reservedIdx[port.HostNetwork] = append(reservedIdx[port.HostNetwork], port)
   511  
   512  		// allocPort is set in the inner for loop if a port mapping can be created
   513  		// if allocPort is still nil after the loop, the port wasn't available for reservation
   514  		var allocPort *AllocatedPortMapping
   515  		var addrErr error
   516  		for _, addr := range idx.HostNetworks[port.HostNetwork] {
   517  			used := idx.getUsedPortsFor(addr.Address)
   518  			// Guard against invalid port
   519  			if port.Value < 0 || port.Value >= MaxValidPort {
   520  				return nil, fmt.Errorf("invalid port %d (out of range)", port.Value)
   521  			}
   522  
   523  			// Check if in use
   524  			if used != nil && used.Check(uint(port.Value)) {
   525  				return nil, fmt.Errorf("reserved port collision %s=%d", port.Label, port.Value)
   526  			}
   527  
   528  			allocPort = &AllocatedPortMapping{
   529  				Label:  port.Label,
   530  				Value:  port.Value,
   531  				To:     port.To,
   532  				HostIP: addr.Address,
   533  			}
   534  			break
   535  		}
   536  
   537  		if allocPort == nil {
   538  			if addrErr != nil {
   539  				return nil, addrErr
   540  			}
   541  
   542  			return nil, fmt.Errorf("no addresses available for %s network", port.HostNetwork)
   543  		}
   544  
   545  		offer = append(offer, *allocPort)
   546  	}
   547  
   548  	for _, port := range ask.DynamicPorts {
   549  		var allocPort *AllocatedPortMapping
   550  		var addrErr error
   551  		for _, addr := range idx.HostNetworks[port.HostNetwork] {
   552  			used := idx.getUsedPortsFor(addr.Address)
   553  			// Try to stochastically pick the dynamic ports as it is faster and
   554  			// lower memory usage.
   555  			var dynPorts []int
   556  			// TODO: its more efficient to find multiple dynamic ports at once
   557  			dynPorts, addrErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
   558  			if addrErr != nil {
   559  				// Fall back to the precise method if the random sampling failed.
   560  				dynPorts, addrErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, reservedIdx[port.HostNetwork], 1)
   561  				if addrErr != nil {
   562  					continue
   563  				}
   564  			}
   565  
   566  			allocPort = &AllocatedPortMapping{
   567  				Label:  port.Label,
   568  				Value:  dynPorts[0],
   569  				To:     port.To,
   570  				HostIP: addr.Address,
   571  			}
   572  			if allocPort.To == -1 {
   573  				allocPort.To = allocPort.Value
   574  			}
   575  			break
   576  		}
   577  
   578  		if allocPort == nil {
   579  			if addrErr != nil {
   580  				return nil, addrErr
   581  			}
   582  
   583  			return nil, fmt.Errorf("no addresses available for %s network", port.HostNetwork)
   584  		}
   585  		offer = append(offer, *allocPort)
   586  	}
   587  
   588  	return offer, nil
   589  }
   590  
   591  // AssignTaskNetwork is used to offer network resources given a
   592  // task.resources.network ask.  If the ask cannot be satisfied, returns nil
   593  //
   594  // AssignTaskNetwork and task.resources.network are deprecated in favor of
   595  // AssignPorts and group.network. AssignTaskNetwork does not support multiple
   596  // interfaces and only uses the node's default interface. AssignPorts is the
   597  // method that is used for group.network asks.
   598  func (idx *NetworkIndex) AssignTaskNetwork(ask *NetworkResource) (out *NetworkResource, err error) {
   599  	err = fmt.Errorf("no networks available")
   600  	idx.yieldIP(func(n *NetworkResource, offerIP net.IP) (stop bool) {
   601  		// Convert the IP to a string
   602  		offerIPStr := offerIP.String()
   603  
   604  		// Check if we would exceed the bandwidth cap
   605  		availBandwidth := idx.AvailBandwidth[n.Device]
   606  		usedBandwidth := idx.UsedBandwidth[n.Device]
   607  		if usedBandwidth+ask.MBits > availBandwidth {
   608  			err = fmt.Errorf("bandwidth exceeded")
   609  			return
   610  		}
   611  
   612  		used := idx.UsedPorts[offerIPStr]
   613  
   614  		// Check if any of the reserved ports are in use
   615  		for _, port := range ask.ReservedPorts {
   616  			// Guard against invalid port
   617  			if port.Value < 0 || port.Value >= MaxValidPort {
   618  				err = fmt.Errorf("invalid port %d (out of range)", port.Value)
   619  				return
   620  			}
   621  
   622  			// Check if in use
   623  			if used != nil && used.Check(uint(port.Value)) {
   624  				err = fmt.Errorf("reserved port collision %s=%d", port.Label, port.Value)
   625  				return
   626  			}
   627  		}
   628  
   629  		// Create the offer
   630  		offer := &NetworkResource{
   631  			Mode:          ask.Mode,
   632  			Device:        n.Device,
   633  			IP:            offerIPStr,
   634  			MBits:         ask.MBits,
   635  			DNS:           ask.DNS,
   636  			ReservedPorts: ask.ReservedPorts,
   637  			DynamicPorts:  ask.DynamicPorts,
   638  		}
   639  
   640  		// Try to stochastically pick the dynamic ports as it is faster and
   641  		// lower memory usage.
   642  		var dynPorts []int
   643  		var dynErr error
   644  		dynPorts, dynErr = getDynamicPortsStochastic(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
   645  		if dynErr == nil {
   646  			goto BUILD_OFFER
   647  		}
   648  
   649  		// Fall back to the precise method if the random sampling failed.
   650  		dynPorts, dynErr = getDynamicPortsPrecise(used, idx.MinDynamicPort, idx.MaxDynamicPort, ask.ReservedPorts, len(ask.DynamicPorts))
   651  		if dynErr != nil {
   652  			err = dynErr
   653  			return
   654  		}
   655  
   656  	BUILD_OFFER:
   657  		for i, port := range dynPorts {
   658  			offer.DynamicPorts[i].Value = port
   659  
   660  			// This syntax allows you to set the mapped to port to the same port
   661  			// allocated by the scheduler on the host.
   662  			if offer.DynamicPorts[i].To == -1 {
   663  				offer.DynamicPorts[i].To = port
   664  			}
   665  		}
   666  
   667  		// Stop, we have an offer!
   668  		out = offer
   669  		err = nil
   670  		return true
   671  	})
   672  	return
   673  }
   674  
   675  // getDynamicPortsPrecise takes the nodes used port bitmap which may be nil if
   676  // no ports have been allocated yet, the network ask and returns a set of unused
   677  // ports to fulfil the ask's DynamicPorts or an error if it failed. An error
   678  // means the ask can not be satisfied as the method does a precise search.
   679  func getDynamicPortsPrecise(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reserved []Port, numDyn int) ([]int, error) {
   680  	// Create a copy of the used ports and apply the new reserves
   681  	var usedSet Bitmap
   682  	var err error
   683  	if nodeUsed != nil {
   684  		usedSet, err = nodeUsed.Copy()
   685  		if err != nil {
   686  			return nil, err
   687  		}
   688  	} else {
   689  		usedSet, err = NewBitmap(MaxValidPort)
   690  		if err != nil {
   691  			return nil, err
   692  		}
   693  	}
   694  
   695  	for _, port := range reserved {
   696  		usedSet.Set(uint(port.Value))
   697  	}
   698  
   699  	// Get the indexes of the unset
   700  	availablePorts := usedSet.IndexesInRange(false, uint(minDynamicPort), uint(maxDynamicPort))
   701  
   702  	// Randomize the amount we need
   703  	if len(availablePorts) < numDyn {
   704  		return nil, fmt.Errorf("dynamic port selection failed")
   705  	}
   706  
   707  	numAvailable := len(availablePorts)
   708  	for i := 0; i < numDyn; i++ {
   709  		j := rand.Intn(numAvailable)
   710  		availablePorts[i], availablePorts[j] = availablePorts[j], availablePorts[i]
   711  	}
   712  
   713  	return availablePorts[:numDyn], nil
   714  }
   715  
   716  // getDynamicPortsStochastic takes the nodes used port bitmap which may be nil if
   717  // no ports have been allocated yet, the network ask and returns a set of unused
   718  // ports to fulfil the ask's DynamicPorts or an error if it failed. An error
   719  // does not mean the ask can not be satisfied as the method has a fixed amount
   720  // of random probes and if these fail, the search is aborted.
   721  func getDynamicPortsStochastic(nodeUsed Bitmap, minDynamicPort, maxDynamicPort int, reservedPorts []Port, count int) ([]int, error) {
   722  	var reserved, dynamic []int
   723  	for _, port := range reservedPorts {
   724  		reserved = append(reserved, port.Value)
   725  	}
   726  
   727  	for i := 0; i < count; i++ {
   728  		attempts := 0
   729  	PICK:
   730  		attempts++
   731  		if attempts > maxRandPortAttempts {
   732  			return nil, fmt.Errorf("stochastic dynamic port selection failed")
   733  		}
   734  
   735  		randPort := minDynamicPort + rand.Intn(maxDynamicPort-minDynamicPort)
   736  		if nodeUsed != nil && nodeUsed.Check(uint(randPort)) {
   737  			goto PICK
   738  		}
   739  
   740  		for _, ports := range [][]int{reserved, dynamic} {
   741  			if isPortReserved(ports, randPort) {
   742  				goto PICK
   743  			}
   744  		}
   745  		dynamic = append(dynamic, randPort)
   746  	}
   747  
   748  	return dynamic, nil
   749  }
   750  
   751  // IntContains scans an integer slice for a value
   752  func isPortReserved(haystack []int, needle int) bool {
   753  	for _, item := range haystack {
   754  		if item == needle {
   755  			return true
   756  		}
   757  	}
   758  	return false
   759  }
   760  
   761  // AllocatedPortsToNetworkResouce is a COMPAT(1.0) remove when NetworkResource
   762  // is no longer used for materialized client view of ports.
   763  func AllocatedPortsToNetworkResouce(ask *NetworkResource, ports AllocatedPorts, node *NodeResources) *NetworkResource {
   764  	out := ask.Copy()
   765  
   766  	for i, port := range ask.DynamicPorts {
   767  		if p, ok := ports.Get(port.Label); ok {
   768  			out.DynamicPorts[i].Value = p.Value
   769  			out.DynamicPorts[i].To = p.To
   770  		}
   771  	}
   772  	if len(node.NodeNetworks) > 0 {
   773  		for _, nw := range node.NodeNetworks {
   774  			if nw.Mode == "host" {
   775  				out.IP = nw.Addresses[0].Address
   776  				break
   777  			}
   778  		}
   779  	} else {
   780  		for _, nw := range node.Networks {
   781  			if nw.Mode == "host" {
   782  				out.IP = nw.IP
   783  			}
   784  		}
   785  	}
   786  	return out
   787  }
   788  
   789  type ClientHostNetworkConfig struct {
   790  	Name          string `hcl:",key"`
   791  	CIDR          string `hcl:"cidr"`
   792  	Interface     string `hcl:"interface"`
   793  	ReservedPorts string `hcl:"reserved_ports"`
   794  }
   795  
   796  func (p *ClientHostNetworkConfig) Copy() *ClientHostNetworkConfig {
   797  	if p == nil {
   798  		return nil
   799  	}
   800  
   801  	c := new(ClientHostNetworkConfig)
   802  	*c = *p
   803  	return c
   804  }