github.com/jfrazelle/docker@v1.1.2-0.20210712172922-bf78e25fe508/libnetwork/ipam/allocator.go (about) 1 package ipam 2 3 import ( 4 "fmt" 5 "net" 6 "sort" 7 "sync" 8 9 "github.com/docker/docker/libnetwork/bitseq" 10 "github.com/docker/docker/libnetwork/datastore" 11 "github.com/docker/docker/libnetwork/discoverapi" 12 "github.com/docker/docker/libnetwork/ipamapi" 13 "github.com/docker/docker/libnetwork/ipamutils" 14 "github.com/docker/docker/libnetwork/types" 15 "github.com/sirupsen/logrus" 16 ) 17 18 const ( 19 localAddressSpace = "LocalDefault" 20 globalAddressSpace = "GlobalDefault" 21 // datastore keyes for ipam objects 22 dsConfigKey = "ipam/" + ipamapi.DefaultIPAM + "/config" 23 dsDataKey = "ipam/" + ipamapi.DefaultIPAM + "/data" 24 ) 25 26 // Allocator provides per address space ipv4/ipv6 book keeping 27 type Allocator struct { 28 // Predefined pools for default address spaces 29 // Separate from the addrSpace because they should not be serialized 30 predefined map[string][]*net.IPNet 31 predefinedStartIndices map[string]int 32 // The (potentially serialized) address spaces 33 addrSpaces map[string]*addrSpace 34 // stores []datastore.Datastore 35 // Allocated addresses in each address space's subnet 36 addresses map[SubnetKey]*bitseq.Handle 37 sync.Mutex 38 } 39 40 // NewAllocator returns an instance of libnetwork ipam 41 func NewAllocator(lcDs, glDs datastore.DataStore) (*Allocator, error) { 42 a := &Allocator{} 43 44 // Load predefined subnet pools 45 46 a.predefined = map[string][]*net.IPNet{ 47 localAddressSpace: ipamutils.GetLocalScopeDefaultNetworks(), 48 globalAddressSpace: ipamutils.GetGlobalScopeDefaultNetworks(), 49 } 50 51 // Initialize asIndices map 52 a.predefinedStartIndices = make(map[string]int) 53 54 // Initialize bitseq map 55 a.addresses = make(map[SubnetKey]*bitseq.Handle) 56 57 // Initialize address spaces 58 a.addrSpaces = make(map[string]*addrSpace) 59 for _, aspc := range []struct { 60 as string 61 ds datastore.DataStore 62 }{ 63 {localAddressSpace, lcDs}, 64 {globalAddressSpace, glDs}, 65 } { 66 a.initializeAddressSpace(aspc.as, aspc.ds) 67 } 68 69 return a, nil 70 } 71 72 func (a *Allocator) refresh(as string) error { 73 aSpace, err := a.getAddressSpaceFromStore(as) 74 if err != nil { 75 return types.InternalErrorf("error getting pools config from store: %v", err) 76 } 77 78 if aSpace == nil { 79 return nil 80 } 81 82 a.Lock() 83 a.addrSpaces[as] = aSpace 84 a.Unlock() 85 86 return nil 87 } 88 89 func (a *Allocator) updateBitMasks(aSpace *addrSpace) error { 90 var inserterList []func() error 91 92 aSpace.Lock() 93 for k, v := range aSpace.subnets { 94 if v.Range == nil { 95 kk := k 96 vv := v 97 inserterList = append(inserterList, func() error { return a.insertBitMask(kk, vv.Pool) }) 98 } 99 } 100 aSpace.Unlock() 101 102 // Add the bitmasks (data could come from datastore) 103 for _, f := range inserterList { 104 if err := f(); err != nil { 105 return err 106 } 107 } 108 109 return nil 110 } 111 112 // Checks for and fixes damaged bitmask. 113 func (a *Allocator) checkConsistency(as string) { 114 var sKeyList []SubnetKey 115 116 // Retrieve this address space's configuration and bitmasks from the datastore 117 a.refresh(as) 118 a.Lock() 119 aSpace, ok := a.addrSpaces[as] 120 a.Unlock() 121 if !ok { 122 return 123 } 124 a.updateBitMasks(aSpace) 125 126 aSpace.Lock() 127 for sk, pd := range aSpace.subnets { 128 if pd.Range != nil { 129 continue 130 } 131 sKeyList = append(sKeyList, sk) 132 } 133 aSpace.Unlock() 134 135 for _, sk := range sKeyList { 136 a.Lock() 137 bm := a.addresses[sk] 138 a.Unlock() 139 if err := bm.CheckConsistency(); err != nil { 140 logrus.Warnf("Error while running consistency check for %s: %v", sk, err) 141 } 142 } 143 } 144 145 func (a *Allocator) initializeAddressSpace(as string, ds datastore.DataStore) error { 146 scope := "" 147 if ds != nil { 148 scope = ds.Scope() 149 } 150 151 a.Lock() 152 if currAS, ok := a.addrSpaces[as]; ok { 153 if currAS.ds != nil { 154 a.Unlock() 155 return types.ForbiddenErrorf("a datastore is already configured for the address space %s", as) 156 } 157 } 158 a.addrSpaces[as] = &addrSpace{ 159 subnets: map[SubnetKey]*PoolData{}, 160 id: dsConfigKey + "/" + as, 161 scope: scope, 162 ds: ds, 163 alloc: a, 164 } 165 a.Unlock() 166 167 a.checkConsistency(as) 168 169 return nil 170 } 171 172 // DiscoverNew informs the allocator about a new global scope datastore 173 func (a *Allocator) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { 174 if dType != discoverapi.DatastoreConfig { 175 return nil 176 } 177 178 dsc, ok := data.(discoverapi.DatastoreConfigData) 179 if !ok { 180 return types.InternalErrorf("incorrect data in datastore update notification: %v", data) 181 } 182 183 ds, err := datastore.NewDataStoreFromConfig(dsc) 184 if err != nil { 185 return err 186 } 187 188 return a.initializeAddressSpace(globalAddressSpace, ds) 189 } 190 191 // DiscoverDelete is a notification of no interest for the allocator 192 func (a *Allocator) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { 193 return nil 194 } 195 196 // GetDefaultAddressSpaces returns the local and global default address spaces 197 func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) { 198 return localAddressSpace, globalAddressSpace, nil 199 } 200 201 // RequestPool returns an address pool along with its unique id. 202 // addressSpace must be a valid address space name and must not be the empty string. 203 // If pool is the empty string then the default predefined pool for addressSpace will be used, otherwise pool must be a valid IP address and length in CIDR notation. 204 // If subPool is not empty, it must be a valid IP address and length in CIDR notation which is a sub-range of pool. 205 // subPool must be empty if pool is empty. 206 func (a *Allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) { 207 logrus.Debugf("RequestPool(%s, %s, %s, %v, %t)", addressSpace, pool, subPool, options, v6) 208 209 k, nw, ipr, err := a.parsePoolRequest(addressSpace, pool, subPool, v6) 210 if err != nil { 211 return "", nil, nil, types.InternalErrorf("failed to parse pool request for address space %q pool %q subpool %q: %v", addressSpace, pool, subPool, err) 212 } 213 214 pdf := k == nil 215 216 retry: 217 if pdf { 218 if nw, err = a.getPredefinedPool(addressSpace, v6); err != nil { 219 return "", nil, nil, err 220 } 221 k = &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String()} 222 } 223 224 if err := a.refresh(addressSpace); err != nil { 225 return "", nil, nil, err 226 } 227 228 aSpace, err := a.getAddrSpace(addressSpace) 229 if err != nil { 230 return "", nil, nil, err 231 } 232 233 insert, err := aSpace.updatePoolDBOnAdd(*k, nw, ipr, pdf) 234 if err != nil { 235 if _, ok := err.(types.MaskableError); ok { 236 logrus.Debugf("Retrying predefined pool search: %v", err) 237 goto retry 238 } 239 return "", nil, nil, err 240 } 241 242 if err := a.writeToStore(aSpace); err != nil { 243 if _, ok := err.(types.RetryError); !ok { 244 return "", nil, nil, types.InternalErrorf("pool configuration failed because of %s", err.Error()) 245 } 246 247 goto retry 248 } 249 250 return k.String(), nw, nil, insert() 251 } 252 253 // ReleasePool releases the address pool identified by the passed id 254 func (a *Allocator) ReleasePool(poolID string) error { 255 logrus.Debugf("ReleasePool(%s)", poolID) 256 k := SubnetKey{} 257 if err := k.FromString(poolID); err != nil { 258 return types.BadRequestErrorf("invalid pool id: %s", poolID) 259 } 260 261 retry: 262 if err := a.refresh(k.AddressSpace); err != nil { 263 return err 264 } 265 266 aSpace, err := a.getAddrSpace(k.AddressSpace) 267 if err != nil { 268 return err 269 } 270 271 remove, err := aSpace.updatePoolDBOnRemoval(k) 272 if err != nil { 273 return err 274 } 275 276 if err = a.writeToStore(aSpace); err != nil { 277 if _, ok := err.(types.RetryError); !ok { 278 return types.InternalErrorf("pool (%s) removal failed because of %v", poolID, err) 279 } 280 goto retry 281 } 282 283 return remove() 284 } 285 286 // Given the address space, returns the local or global PoolConfig based on whether the 287 // address space is local or global. AddressSpace locality is registered with IPAM out of band. 288 func (a *Allocator) getAddrSpace(as string) (*addrSpace, error) { 289 a.Lock() 290 defer a.Unlock() 291 aSpace, ok := a.addrSpaces[as] 292 if !ok { 293 return nil, types.BadRequestErrorf("cannot find address space %s (most likely the backing datastore is not configured)", as) 294 } 295 return aSpace, nil 296 } 297 298 // parsePoolRequest parses and validates a request to create a new pool under addressSpace and returns 299 // a SubnetKey, network and range describing the request. 300 func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *AddressRange, error) { 301 var ( 302 nw *net.IPNet 303 ipr *AddressRange 304 err error 305 ) 306 307 if addressSpace == "" { 308 return nil, nil, nil, ipamapi.ErrInvalidAddressSpace 309 } 310 311 if pool == "" && subPool != "" { 312 return nil, nil, nil, ipamapi.ErrInvalidSubPool 313 } 314 315 if pool == "" { 316 return nil, nil, nil, nil 317 } 318 319 if _, nw, err = net.ParseCIDR(pool); err != nil { 320 return nil, nil, nil, ipamapi.ErrInvalidPool 321 } 322 323 if subPool != "" { 324 if ipr, err = getAddressRange(subPool, nw); err != nil { 325 return nil, nil, nil, err 326 } 327 } 328 329 return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, ipr, nil 330 } 331 332 func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error { 333 //logrus.Debugf("Inserting bitmask (%s, %s)", key.String(), pool.String()) 334 335 store := a.getStore(key.AddressSpace) 336 ipVer := getAddressVersion(pool.IP) 337 ones, bits := pool.Mask.Size() 338 numAddresses := uint64(1 << uint(bits-ones)) 339 340 // Allow /64 subnet 341 if ipVer == v6 && numAddresses == 0 { 342 numAddresses-- 343 } 344 345 // Generate the new address masks. AddressMask content may come from datastore 346 h, err := bitseq.NewHandle(dsDataKey, store, key.String(), numAddresses) 347 if err != nil { 348 return err 349 } 350 351 // Do not let network identifier address be reserved 352 // Do the same for IPv6 so that bridge ip starts with XXXX...::1 353 h.Set(0) 354 355 // Do not let broadcast address be reserved 356 if ipVer == v4 { 357 h.Set(numAddresses - 1) 358 } 359 360 a.Lock() 361 a.addresses[key] = h 362 a.Unlock() 363 return nil 364 } 365 366 func (a *Allocator) retrieveBitmask(k SubnetKey, n *net.IPNet) (*bitseq.Handle, error) { 367 a.Lock() 368 bm, ok := a.addresses[k] 369 a.Unlock() 370 if !ok { 371 logrus.Debugf("Retrieving bitmask (%s, %s)", k.String(), n.String()) 372 if err := a.insertBitMask(k, n); err != nil { 373 return nil, types.InternalErrorf("could not find bitmask in datastore for %s", k.String()) 374 } 375 a.Lock() 376 bm = a.addresses[k] 377 a.Unlock() 378 } 379 return bm, nil 380 } 381 382 func (a *Allocator) getPredefineds(as string) []*net.IPNet { 383 a.Lock() 384 defer a.Unlock() 385 386 p := a.predefined[as] 387 i := a.predefinedStartIndices[as] 388 // defensive in case the list changed since last update 389 if i >= len(p) { 390 i = 0 391 } 392 return append(p[i:], p[:i]...) 393 } 394 395 func (a *Allocator) updateStartIndex(as string, amt int) { 396 a.Lock() 397 i := a.predefinedStartIndices[as] + amt 398 if i < 0 || i >= len(a.predefined[as]) { 399 i = 0 400 } 401 a.predefinedStartIndices[as] = i 402 a.Unlock() 403 } 404 405 func (a *Allocator) getPredefinedPool(as string, ipV6 bool) (*net.IPNet, error) { 406 var v ipVersion 407 v = v4 408 if ipV6 { 409 v = v6 410 } 411 412 if as != localAddressSpace && as != globalAddressSpace { 413 return nil, types.NotImplementedErrorf("no default pool available for non-default address spaces") 414 } 415 416 aSpace, err := a.getAddrSpace(as) 417 if err != nil { 418 return nil, err 419 } 420 421 predefined := a.getPredefineds(as) 422 423 aSpace.Lock() 424 for i, nw := range predefined { 425 if v != getAddressVersion(nw.IP) { 426 continue 427 } 428 // Checks whether pool has already been allocated 429 if _, ok := aSpace.subnets[SubnetKey{AddressSpace: as, Subnet: nw.String()}]; ok { 430 continue 431 } 432 // Shouldn't be necessary, but check prevents IP collisions should 433 // predefined pools overlap for any reason. 434 if !aSpace.contains(as, nw) { 435 aSpace.Unlock() 436 a.updateStartIndex(as, i+1) 437 return nw, nil 438 } 439 } 440 aSpace.Unlock() 441 442 return nil, types.NotFoundErrorf("could not find an available, non-overlapping IPv%d address pool among the defaults to assign to the network", v) 443 } 444 445 // RequestAddress returns an address from the specified pool ID 446 func (a *Allocator) RequestAddress(poolID string, prefAddress net.IP, opts map[string]string) (*net.IPNet, map[string]string, error) { 447 logrus.Debugf("RequestAddress(%s, %v, %v)", poolID, prefAddress, opts) 448 k := SubnetKey{} 449 if err := k.FromString(poolID); err != nil { 450 return nil, nil, types.BadRequestErrorf("invalid pool id: %s", poolID) 451 } 452 453 if err := a.refresh(k.AddressSpace); err != nil { 454 return nil, nil, err 455 } 456 457 aSpace, err := a.getAddrSpace(k.AddressSpace) 458 if err != nil { 459 return nil, nil, err 460 } 461 462 aSpace.Lock() 463 p, ok := aSpace.subnets[k] 464 if !ok { 465 aSpace.Unlock() 466 return nil, nil, types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID) 467 } 468 469 if prefAddress != nil && !p.Pool.Contains(prefAddress) { 470 aSpace.Unlock() 471 return nil, nil, ipamapi.ErrIPOutOfRange 472 } 473 474 c := p 475 for c.Range != nil { 476 k = c.ParentKey 477 c = aSpace.subnets[k] 478 } 479 aSpace.Unlock() 480 481 bm, err := a.retrieveBitmask(k, c.Pool) 482 if err != nil { 483 return nil, nil, types.InternalErrorf("could not find bitmask in datastore for %s on address %v request from pool %s: %v", 484 k.String(), prefAddress, poolID, err) 485 } 486 // In order to request for a serial ip address allocation, callers can pass in the option to request 487 // IP allocation serially or first available IP in the subnet 488 var serial bool 489 if opts != nil { 490 if val, ok := opts[ipamapi.AllocSerialPrefix]; ok { 491 serial = (val == "true") 492 } 493 } 494 ip, err := a.getAddress(p.Pool, bm, prefAddress, p.Range, serial) 495 if err != nil { 496 return nil, nil, err 497 } 498 499 return &net.IPNet{IP: ip, Mask: p.Pool.Mask}, nil, nil 500 } 501 502 // ReleaseAddress releases the address from the specified pool ID 503 func (a *Allocator) ReleaseAddress(poolID string, address net.IP) error { 504 logrus.Debugf("ReleaseAddress(%s, %v)", poolID, address) 505 k := SubnetKey{} 506 if err := k.FromString(poolID); err != nil { 507 return types.BadRequestErrorf("invalid pool id: %s", poolID) 508 } 509 510 if err := a.refresh(k.AddressSpace); err != nil { 511 return err 512 } 513 514 aSpace, err := a.getAddrSpace(k.AddressSpace) 515 if err != nil { 516 return err 517 } 518 519 aSpace.Lock() 520 p, ok := aSpace.subnets[k] 521 if !ok { 522 aSpace.Unlock() 523 return types.NotFoundErrorf("cannot find address pool for poolID:%s", poolID) 524 } 525 526 if address == nil { 527 aSpace.Unlock() 528 return types.BadRequestErrorf("invalid address: nil") 529 } 530 531 if !p.Pool.Contains(address) { 532 aSpace.Unlock() 533 return ipamapi.ErrIPOutOfRange 534 } 535 536 c := p 537 for c.Range != nil { 538 k = c.ParentKey 539 c = aSpace.subnets[k] 540 } 541 aSpace.Unlock() 542 543 mask := p.Pool.Mask 544 545 h, err := types.GetHostPartIP(address, mask) 546 if err != nil { 547 return types.InternalErrorf("failed to release address %s: %v", address.String(), err) 548 } 549 550 bm, err := a.retrieveBitmask(k, c.Pool) 551 if err != nil { 552 return types.InternalErrorf("could not find bitmask in datastore for %s on address %v release from pool %s: %v", 553 k.String(), address, poolID, err) 554 } 555 defer logrus.Debugf("Released address PoolID:%s, Address:%v Sequence:%s", poolID, address, bm.String()) 556 557 return bm.Unset(ipToUint64(h)) 558 } 559 560 func (a *Allocator) getAddress(nw *net.IPNet, bitmask *bitseq.Handle, prefAddress net.IP, ipr *AddressRange, serial bool) (net.IP, error) { 561 var ( 562 ordinal uint64 563 err error 564 base *net.IPNet 565 ) 566 567 logrus.Debugf("Request address PoolID:%v %s Serial:%v PrefAddress:%v ", nw, bitmask.String(), serial, prefAddress) 568 base = types.GetIPNetCopy(nw) 569 570 if bitmask.Unselected() == 0 { 571 return nil, ipamapi.ErrNoAvailableIPs 572 } 573 if ipr == nil && prefAddress == nil { 574 ordinal, err = bitmask.SetAny(serial) 575 } else if prefAddress != nil { 576 hostPart, e := types.GetHostPartIP(prefAddress, base.Mask) 577 if e != nil { 578 return nil, types.InternalErrorf("failed to allocate requested address %s: %v", prefAddress.String(), e) 579 } 580 ordinal = ipToUint64(types.GetMinimalIP(hostPart)) 581 err = bitmask.Set(ordinal) 582 } else { 583 ordinal, err = bitmask.SetAnyInRange(ipr.Start, ipr.End, serial) 584 } 585 586 switch err { 587 case nil: 588 // Convert IP ordinal for this subnet into IP address 589 return generateAddress(ordinal, base), nil 590 case bitseq.ErrBitAllocated: 591 return nil, ipamapi.ErrIPAlreadyAllocated 592 case bitseq.ErrNoBitAvailable: 593 return nil, ipamapi.ErrNoAvailableIPs 594 default: 595 return nil, err 596 } 597 } 598 599 // DumpDatabase dumps the internal info 600 func (a *Allocator) DumpDatabase() string { 601 a.Lock() 602 aspaces := make(map[string]*addrSpace, len(a.addrSpaces)) 603 orderedAS := make([]string, 0, len(a.addrSpaces)) 604 for as, aSpace := range a.addrSpaces { 605 orderedAS = append(orderedAS, as) 606 aspaces[as] = aSpace 607 } 608 a.Unlock() 609 610 sort.Strings(orderedAS) 611 612 var s string 613 for _, as := range orderedAS { 614 aSpace := aspaces[as] 615 s = fmt.Sprintf("\n\n%s Config", as) 616 aSpace.Lock() 617 for k, config := range aSpace.subnets { 618 s += fmt.Sprintf("\n%v: %v", k, config) 619 if config.Range == nil { 620 a.retrieveBitmask(k, config.Pool) 621 } 622 } 623 aSpace.Unlock() 624 } 625 626 s = fmt.Sprintf("%s\n\nBitmasks", s) 627 for k, bm := range a.addresses { 628 s += fmt.Sprintf("\n%s: %s", k, bm) 629 } 630 631 return s 632 } 633 634 // IsBuiltIn returns true for builtin drivers 635 func (a *Allocator) IsBuiltIn() bool { 636 return true 637 }