github.com/jfrazelle/docker@v1.1.2-0.20210712172922-bf78e25fe508/libnetwork/ipam/allocator.go (about)

     1  package ipam
     2  
     3  import (
     4  	"fmt"
     5  	"net"
     6  	"sort"
     7  	"sync"
     8  
     9  	"github.com/docker/docker/libnetwork/bitseq"
    10  	"github.com/docker/docker/libnetwork/datastore"
    11  	"github.com/docker/docker/libnetwork/discoverapi"
    12  	"github.com/docker/docker/libnetwork/ipamapi"
    13  	"github.com/docker/docker/libnetwork/ipamutils"
    14  	"github.com/docker/docker/libnetwork/types"
    15  	"github.com/sirupsen/logrus"
    16  )
    17  
    18  const (
    19  	localAddressSpace  = "LocalDefault"
    20  	globalAddressSpace = "GlobalDefault"
    21  	// datastore keyes for ipam objects
    22  	dsConfigKey = "ipam/" + ipamapi.DefaultIPAM + "/config"
    23  	dsDataKey   = "ipam/" + ipamapi.DefaultIPAM + "/data"
    24  )
    25  
    26  // Allocator provides per address space ipv4/ipv6 book keeping
    27  type Allocator struct {
    28  	// Predefined pools for default address spaces
    29  	// Separate from the addrSpace because they should not be serialized
    30  	predefined             map[string][]*net.IPNet
    31  	predefinedStartIndices map[string]int
    32  	// The (potentially serialized) address spaces
    33  	addrSpaces map[string]*addrSpace
    34  	// stores        []datastore.Datastore
    35  	// Allocated addresses in each address space's subnet
    36  	addresses map[SubnetKey]*bitseq.Handle
    37  	sync.Mutex
    38  }
    39  
    40  // NewAllocator returns an instance of libnetwork ipam
    41  func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) {
    42  	a := &Allocator{}
    43  
    44  	// Load predefined subnet pools
    45  
    46  	a.predefined = map[string][]*net.IPNet{
    47  		localAddressSpace:  ipamutils.GetLocalScopeDefaultNetworks(),
    48  		globalAddressSpace: ipamutils.GetGlobalScopeDefaultNetworks(),
    49  	}
    50  
    51  	// Initialize asIndices map
    52  	a.predefinedStartIndices = make(map[string]int)
    53  
    54  	// Initialize bitseq map
    55  	a.addresses = make(map[SubnetKey]*bitseq.Handle)
    56  
    57  	// Initialize address spaces
    58  	a.addrSpaces = make(map[string]*addrSpace)
    59  	for _, aspc := range []struct {
    60  		as string
    61  		ds datastore.DataStore
    62  	}{
    63  		{localAddressSpace, lcDs},
    64  		{globalAddressSpace, glDs},
    65  	} {
    66  		a.initializeAddressSpace(aspc.as, aspc.ds)
    67  	}
    68  
    69  	return a, nil
    70  }
    71  
    72  func (a *Allocator) refresh(as string) error {
    73  	aSpace, err := a.getAddressSpaceFromStore(as)
    74  	if err != nil {
    75  		return types.InternalErrorf("error getting pools config from store: %v", err)
    76  	}
    77  
    78  	if aSpace == nil {
    79  		return nil
    80  	}
    81  
    82  	a.Lock()
    83  	a.addrSpaces[as] = aSpace
    84  	a.Unlock()
    85  
    86  	return nil
    87  }
    88  
    89  func (a *Allocator) updateBitMasks(aSpace *addrSpace) error {
    90  	var inserterList []func() error
    91  
    92  	aSpace.Lock()
    93  	for k, v := range aSpace.subnets {
    94  		if v.Range == nil {
    95  			kk := k
    96  			vv := v
    97  			inserterList = append(inserterList, func() error { return a.insertBitMask(kk, vv.Pool) })
    98  		}
    99  	}
   100  	aSpace.Unlock()
   101  
   102  	// Add the bitmasks (data could come from datastore)
   103  	for _, f := range inserterList {
   104  		if err := f(); err != nil {
   105  			return err
   106  		}
   107  	}
   108  
   109  	return nil
   110  }
   111  
   112  // Checks for and fixes damaged bitmask.
   113  func (a *Allocator) checkConsistency(as string) {
   114  	var sKeyList []SubnetKey
   115  
   116  	// Retrieve this address space's configuration and bitmasks from the datastore
   117  	a.refresh(as)
   118  	a.Lock()
   119  	aSpace, ok := a.addrSpaces[as]
   120  	a.Unlock()
   121  	if !ok {
   122  		return
   123  	}
   124  	a.updateBitMasks(aSpace)
   125  
   126  	aSpace.Lock()
   127  	for sk, pd := range aSpace.subnets {
   128  		if pd.Range != nil {
   129  			continue
   130  		}
   131  		sKeyList = append(sKeyList, sk)
   132  	}
   133  	aSpace.Unlock()
   134  
   135  	for _, sk := range sKeyList {
   136  		a.Lock()
   137  		bm := a.addresses[sk]
   138  		a.Unlock()
   139  		if err := bm.CheckConsistency(); err != nil {
   140  			logrus.Warnf("Error while running consistency check for %s: %v", sk, err)
   141  		}
   142  	}
   143  }
   144  
   145  func (a *Allocator) initializeAddressSpace(as string, ds datastore.DataStore) error {
   146  	scope := ""
   147  	if ds != nil {
   148  		scope = ds.Scope()
   149  	}
   150  
   151  	a.Lock()
   152  	if currAS, ok := a.addrSpaces[as]; ok {
   153  		if currAS.ds != nil {
   154  			a.Unlock()
   155  			return types.ForbiddenErrorf("a datastore is already configured for the address space %s", as)
   156  		}
   157  	}
   158  	a.addrSpaces[as] = &addrSpace{
   159  		subnets: map[SubnetKey]*PoolData{},
   160  		id:      dsConfigKey + "/" + as,
   161  		scope:   scope,
   162  		ds:      ds,
   163  		alloc:   a,
   164  	}
   165  	a.Unlock()
   166  
   167  	a.checkConsistency(as)
   168  
   169  	return nil
   170  }
   171  
   172  // DiscoverNew informs the allocator about a new global scope datastore
   173  func (a *Allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
   174  	if dType != discoverapi.DatastoreConfig {
   175  		return nil
   176  	}
   177  
   178  	dsc, ok := data.(discoverapi.DatastoreConfigData)
   179  	if !ok {
   180  		return types.InternalErrorf("incorrect data in datastore update notification: %v", data)
   181  	}
   182  
   183  	ds, err := datastore.NewDataStoreFromConfig(dsc)
   184  	if err != nil {
   185  		return err
   186  	}
   187  
   188  	return a.initializeAddressSpace(globalAddressSpace, ds)
   189  }
   190  
   191  // DiscoverDelete is a notification of no interest for the allocator
   192  func (a *Allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
   193  	return nil
   194  }
   195  
   196  // GetDefaultAddressSpaces returns the local and global default address spaces
   197  func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) {
   198  	return localAddressSpace, globalAddressSpace, nil
   199  }
   200  
   201  // RequestPool returns an address pool along with its unique id.
   202  // addressSpace must be a valid address space name and must not be the empty string.
   203  // If pool is the empty string then the default predefined pool for addressSpace will be used, otherwise pool must be a valid IP address and length in CIDR notation.
   204  // If subPool is not empty, it must be a valid IP address and length in CIDR notation which is a sub-range of pool.
   205  // subPool must be empty if pool is empty.
   206  func (a *Allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) {
   207  	logrus.Debugf("RequestPool(%s, %s, %s, %v, %t)", addressSpace, pool, subPool, options, v6)
   208  
   209  	k, nw, ipr, err := a.parsePoolRequest(addressSpace, pool, subPool, v6)
   210  	if err != nil {
   211  		return "", nil, nil, types.InternalErrorf("failed to parse pool request for address space %q pool %q subpool %q: %v", addressSpace, pool, subPool, err)
   212  	}
   213  
   214  	pdf := k == nil
   215  
   216  retry:
   217  	if pdf {
   218  		if nw, err = a.getPredefinedPool(addressSpace, v6); err != nil {
   219  			return "", nil, nil, err
   220  		}
   221  		k = &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String()}
   222  	}
   223  
   224  	if err := a.refresh(addressSpace); err != nil {
   225  		return "", nil, nil, err
   226  	}
   227  
   228  	aSpace, err := a.getAddrSpace(addressSpace)
   229  	if err != nil {
   230  		return "", nil, nil, err
   231  	}
   232  
   233  	insert, err := aSpace.updatePoolDBOnAdd(*k, nw, ipr, pdf)
   234  	if err != nil {
   235  		if _, ok := err.(types.MaskableError); ok {
   236  			logrus.Debugf("Retrying predefined pool search: %v", err)
   237  			goto retry
   238  		}
   239  		return "", nil, nil, err
   240  	}
   241  
   242  	if err := a.writeToStore(aSpace); err != nil {
   243  		if _, ok := err.(types.RetryError); !ok {
   244  			return "", nil, nil, types.InternalErrorf("pool configuration failed because of %s", err.Error())
   245  		}
   246  
   247  		goto retry
   248  	}
   249  
   250  	return k.String(), nw, nil, insert()
   251  }
   252  
   253  // ReleasePool releases the address pool identified by the passed id
   254  func (a *Allocator) ReleasePool(poolID string) error {
   255  	logrus.Debugf("ReleasePool(%s)", poolID)
   256  	k := SubnetKey{}
   257  	if err := k.FromString(poolID); err != nil {
   258  		return types.BadRequestErrorf("invalid pool id: %s", poolID)
   259  	}
   260  
   261  retry:
   262  	if err := a.refresh(k.AddressSpace); err != nil {
   263  		return err
   264  	}
   265  
   266  	aSpace, err := a.getAddrSpace(k.AddressSpace)
   267  	if err != nil {
   268  		return err
   269  	}
   270  
   271  	remove, err := aSpace.updatePoolDBOnRemoval(k)
   272  	if err != nil {
   273  		return err
   274  	}
   275  
   276  	if err = a.writeToStore(aSpace); err != nil {
   277  		if _, ok := err.(types.RetryError); !ok {
   278  			return types.InternalErrorf("pool (%s) removal failed because of %v", poolID, err)
   279  		}
   280  		goto retry
   281  	}
   282  
   283  	return remove()
   284  }
   285  
   286  // Given the address space, returns the local or global PoolConfig based on whether the
   287  // address space is local or global. AddressSpace locality is registered with IPAM out of band.
   288  func (a *Allocator) getAddrSpace(as string) (*addrSpace, error) {
   289  	a.Lock()
   290  	defer a.Unlock()
   291  	aSpace, ok := a.addrSpaces[as]
   292  	if !ok {
   293  		return nil, types.BadRequestErrorf("cannot find address space %s (most likely the backing datastore is not configured)", as)
   294  	}
   295  	return aSpace, nil
   296  }
   297  
   298  // parsePoolRequest parses and validates a request to create a new pool under addressSpace and returns
   299  // a SubnetKey, network and range describing the request.
   300  func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *AddressRange, error) {
   301  	var (
   302  		nw  *net.IPNet
   303  		ipr *AddressRange
   304  		err error
   305  	)
   306  
   307  	if addressSpace == "" {
   308  		return nil, nil, nil, ipamapi.ErrInvalidAddressSpace
   309  	}
   310  
   311  	if pool == "" && subPool != "" {
   312  		return nil, nil, nil, ipamapi.ErrInvalidSubPool
   313  	}
   314  
   315  	if pool == "" {
   316  		return nil, nil, nil, nil
   317  	}
   318  
   319  	if _, nw, err = net.ParseCIDR(pool); err != nil {
   320  		return nil, nil, nil, ipamapi.ErrInvalidPool
   321  	}
   322  
   323  	if subPool != "" {
   324  		if ipr, err = getAddressRange(subPool, nw); err != nil {
   325  			return nil, nil, nil, err
   326  		}
   327  	}
   328  
   329  	return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, ipr, nil
   330  }
   331  
   332  func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error {
   333  	//logrus.Debugf("Inserting bitmask (%s, %s)", key.String(), pool.String())
   334  
   335  	store := a.getStore(key.AddressSpace)
   336  	ipVer := getAddressVersion(pool.IP)
   337  	ones, bits := pool.Mask.Size()
   338  	numAddresses := uint64(1 << uint(bits-ones))
   339  
   340  	// Allow /64 subnet
   341  	if ipVer == v6 && numAddresses == 0 {
   342  		numAddresses--
   343  	}
   344  
   345  	// Generate the new address masks. AddressMask content may come from datastore
   346  	h, err := bitseq.NewHandle(dsDataKey, store, key.String(), numAddresses)
   347  	if err != nil {
   348  		return err
   349  	}
   350  
   351  	// Do not let network identifier address be reserved
   352  	// Do the same for IPv6 so that bridge ip starts with XXXX...::1
   353  	h.Set(0)
   354  
   355  	// Do not let broadcast address be reserved
   356  	if ipVer == v4 {
   357  		h.Set(numAddresses - 1)
   358  	}
   359  
   360  	a.Lock()
   361  	a.addresses[key] = h
   362  	a.Unlock()
   363  	return nil
   364  }
   365  
   366  func (a *Allocator) retrieveBitmask(k SubnetKey, n *net.IPNet) (*bitseq.Handle, error) {
   367  	a.Lock()
   368  	bm, ok := a.addresses[k]
   369  	a.Unlock()
   370  	if !ok {
   371  		logrus.Debugf("Retrieving bitmask (%s, %s)", k.String(), n.String())
   372  		if err := a.insertBitMask(k, n); err != nil {
   373  			return nil, types.InternalErrorf("could not find bitmask in datastore for %s", k.String())
   374  		}
   375  		a.Lock()
   376  		bm = a.addresses[k]
   377  		a.Unlock()
   378  	}
   379  	return bm, nil
   380  }
   381  
   382  func (a *Allocator) getPredefineds(as string) []*net.IPNet {
   383  	a.Lock()
   384  	defer a.Unlock()
   385  
   386  	p := a.predefined[as]
   387  	i := a.predefinedStartIndices[as]
   388  	// defensive in case the list changed since last update
   389  	if i >= len(p) {
   390  		i = 0
   391  	}
   392  	return append(p[i:], p[:i]...)
   393  }
   394  
   395  func (a *Allocator) updateStartIndex(as string, amt int) {
   396  	a.Lock()
   397  	i := a.predefinedStartIndices[as] + amt
   398  	if i < 0 || i >= len(a.predefined[as]) {
   399  		i = 0
   400  	}
   401  	a.predefinedStartIndices[as] = i
   402  	a.Unlock()
   403  }
   404  
   405  func (a *Allocator) getPredefinedPool(as string, ipV6 bool) (*net.IPNet, error) {
   406  	var v ipVersion
   407  	v = v4
   408  	if ipV6 {
   409  		v = v6
   410  	}
   411  
   412  	if as != localAddressSpace && as != globalAddressSpace {
   413  		return nil, types.NotImplementedErrorf("no default pool available for non-default address spaces")
   414  	}
   415  
   416  	aSpace, err := a.getAddrSpace(as)
   417  	if err != nil {
   418  		return nil, err
   419  	}
   420  
   421  	predefined := a.getPredefineds(as)
   422  
   423  	aSpace.Lock()
   424  	for i, nw := range predefined {
   425  		if v != getAddressVersion(nw.IP) {
   426  			continue
   427  		}
   428  		// Checks whether pool has already been allocated
   429  		if _, ok := aSpace.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}]; ok {
   430  			continue
   431  		}
   432  		// Shouldn't be necessary, but check prevents IP collisions should
   433  		// predefined pools overlap for any reason.
   434  		if !aSpace.contains(as, nw) {
   435  			aSpace.Unlock()
   436  			a.updateStartIndex(as, i+1)
   437  			return nw, nil
   438  		}
   439  	}
   440  	aSpace.Unlock()
   441  
   442  	return nil, types.NotFoundErrorf("could not find an available, non-overlapping IPv%d address pool among the defaults to assign to the network", v)
   443  }
   444  
   445  // RequestAddress returns an address from the specified pool ID
   446  func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[string]string) (*net.IPNet, map[string]string, error) {
   447  	logrus.Debugf("RequestAddress(%s, %v, %v)", poolID, prefAddress, opts)
   448  	k := SubnetKey{}
   449  	if err := k.FromString(poolID); err != nil {
   450  		return nil, nil, types.BadRequestErrorf("invalid pool id: %s", poolID)
   451  	}
   452  
   453  	if err := a.refresh(k.AddressSpace); err != nil {
   454  		return nil, nil, err
   455  	}
   456  
   457  	aSpace, err := a.getAddrSpace(k.AddressSpace)
   458  	if err != nil {
   459  		return nil, nil, err
   460  	}
   461  
   462  	aSpace.Lock()
   463  	p, ok := aSpace.subnets[k]
   464  	if !ok {
   465  		aSpace.Unlock()
   466  		return nil, nil, types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID)
   467  	}
   468  
   469  	if prefAddress != nil && !p.Pool.Contains(prefAddress) {
   470  		aSpace.Unlock()
   471  		return nil, nil, ipamapi.ErrIPOutOfRange
   472  	}
   473  
   474  	c := p
   475  	for c.Range != nil {
   476  		k = c.ParentKey
   477  		c = aSpace.subnets[k]
   478  	}
   479  	aSpace.Unlock()
   480  
   481  	bm, err := a.retrieveBitmask(k, c.Pool)
   482  	if err != nil {
   483  		return nil, nil, types.InternalErrorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v",
   484  			k.String(), prefAddress, poolID, err)
   485  	}
   486  	// In order to request for a serial ip address allocation, callers can pass in the option to request
   487  	// IP allocation serially or first available IP in the subnet
   488  	var serial bool
   489  	if opts != nil {
   490  		if val, ok := opts[ipamapi.AllocSerialPrefix]; ok {
   491  			serial = (val == "true")
   492  		}
   493  	}
   494  	ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range, serial)
   495  	if err != nil {
   496  		return nil, nil, err
   497  	}
   498  
   499  	return &net.IPNet{IP: ip, Mask: p.Pool.Mask}, nil, nil
   500  }
   501  
   502  // ReleaseAddress releases the address from the specified pool ID
   503  func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error {
   504  	logrus.Debugf("ReleaseAddress(%s, %v)", poolID, address)
   505  	k := SubnetKey{}
   506  	if err := k.FromString(poolID); err != nil {
   507  		return types.BadRequestErrorf("invalid pool id: %s", poolID)
   508  	}
   509  
   510  	if err := a.refresh(k.AddressSpace); err != nil {
   511  		return err
   512  	}
   513  
   514  	aSpace, err := a.getAddrSpace(k.AddressSpace)
   515  	if err != nil {
   516  		return err
   517  	}
   518  
   519  	aSpace.Lock()
   520  	p, ok := aSpace.subnets[k]
   521  	if !ok {
   522  		aSpace.Unlock()
   523  		return types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID)
   524  	}
   525  
   526  	if address == nil {
   527  		aSpace.Unlock()
   528  		return types.BadRequestErrorf("invalid address: nil")
   529  	}
   530  
   531  	if !p.Pool.Contains(address) {
   532  		aSpace.Unlock()
   533  		return ipamapi.ErrIPOutOfRange
   534  	}
   535  
   536  	c := p
   537  	for c.Range != nil {
   538  		k = c.ParentKey
   539  		c = aSpace.subnets[k]
   540  	}
   541  	aSpace.Unlock()
   542  
   543  	mask := p.Pool.Mask
   544  
   545  	h, err := types.GetHostPartIP(address, mask)
   546  	if err != nil {
   547  		return types.InternalErrorf("failed to release address %s: %v", address.String(), err)
   548  	}
   549  
   550  	bm, err := a.retrieveBitmask(k, c.Pool)
   551  	if err != nil {
   552  		return types.InternalErrorf("could not find bitmask in datastore for %s on address %v release from pool %s: %v",
   553  			k.String(), address, poolID, err)
   554  	}
   555  	defer logrus.Debugf("Released address PoolID:%s, Address:%v Sequence:%s", poolID, address, bm.String())
   556  
   557  	return bm.Unset(ipToUint64(h))
   558  }
   559  
   560  func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange, serial bool) (net.IP, error) {
   561  	var (
   562  		ordinal uint64
   563  		err     error
   564  		base    *net.IPNet
   565  	)
   566  
   567  	logrus.Debugf("Request address PoolID:%v %s Serial:%v PrefAddress:%v ", nw, bitmask.String(), serial, prefAddress)
   568  	base = types.GetIPNetCopy(nw)
   569  
   570  	if bitmask.Unselected() == 0 {
   571  		return nil, ipamapi.ErrNoAvailableIPs
   572  	}
   573  	if ipr == nil && prefAddress == nil {
   574  		ordinal, err = bitmask.SetAny(serial)
   575  	} else if prefAddress != nil {
   576  		hostPart, e := types.GetHostPartIP(prefAddress, base.Mask)
   577  		if e != nil {
   578  			return nil, types.InternalErrorf("failed to allocate requested address %s: %v", prefAddress.String(), e)
   579  		}
   580  		ordinal = ipToUint64(types.GetMinimalIP(hostPart))
   581  		err = bitmask.Set(ordinal)
   582  	} else {
   583  		ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End, serial)
   584  	}
   585  
   586  	switch err {
   587  	case nil:
   588  		// Convert IP ordinal for this subnet into IP address
   589  		return generateAddress(ordinal, base), nil
   590  	case bitseq.ErrBitAllocated:
   591  		return nil, ipamapi.ErrIPAlreadyAllocated
   592  	case bitseq.ErrNoBitAvailable:
   593  		return nil, ipamapi.ErrNoAvailableIPs
   594  	default:
   595  		return nil, err
   596  	}
   597  }
   598  
   599  // DumpDatabase dumps the internal info
   600  func (a *Allocator) DumpDatabase() string {
   601  	a.Lock()
   602  	aspaces := make(map[string]*addrSpace, len(a.addrSpaces))
   603  	orderedAS := make([]string, 0, len(a.addrSpaces))
   604  	for as, aSpace := range a.addrSpaces {
   605  		orderedAS = append(orderedAS, as)
   606  		aspaces[as] = aSpace
   607  	}
   608  	a.Unlock()
   609  
   610  	sort.Strings(orderedAS)
   611  
   612  	var s string
   613  	for _, as := range orderedAS {
   614  		aSpace := aspaces[as]
   615  		s = fmt.Sprintf("\n\n%s Config", as)
   616  		aSpace.Lock()
   617  		for k, config := range aSpace.subnets {
   618  			s += fmt.Sprintf("\n%v: %v", k, config)
   619  			if config.Range == nil {
   620  				a.retrieveBitmask(k, config.Pool)
   621  			}
   622  		}
   623  		aSpace.Unlock()
   624  	}
   625  
   626  	s = fmt.Sprintf("%s\n\nBitmasks", s)
   627  	for k, bm := range a.addresses {
   628  		s += fmt.Sprintf("\n%s: %s", k, bm)
   629  	}
   630  
   631  	return s
   632  }
   633  
   634  // IsBuiltIn returns true for builtin drivers
   635  func (a *Allocator) IsBuiltIn() bool {
   636  	return true
   637  }