github.com/rumpl/bof@v23.0.0-rc.2+incompatible/libnetwork/sandbox.go (about) 1 package libnetwork 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "net" 7 "sort" 8 "strings" 9 "sync" 10 "time" 11 12 "github.com/docker/docker/libnetwork/etchosts" 13 "github.com/docker/docker/libnetwork/netlabel" 14 "github.com/docker/docker/libnetwork/osl" 15 "github.com/docker/docker/libnetwork/types" 16 "github.com/sirupsen/logrus" 17 ) 18 19 // Sandbox provides the control over the network container entity. It is a one to one mapping with the container. 20 type Sandbox interface { 21 // ID returns the ID of the sandbox 22 ID() string 23 // Key returns the sandbox's key 24 Key() string 25 // ContainerID returns the container id associated to this sandbox 26 ContainerID() string 27 // Labels returns the sandbox's labels 28 Labels() map[string]interface{} 29 // Statistics retrieves the interfaces' statistics for the sandbox 30 Statistics() (map[string]*types.InterfaceStatistics, error) 31 // Refresh leaves all the endpoints, resets and re-applies the options, 32 // re-joins all the endpoints without destroying the osl sandbox 33 Refresh(options ...SandboxOption) error 34 // SetKey updates the Sandbox Key 35 SetKey(key string) error 36 // Rename changes the name of all attached Endpoints 37 Rename(name string) error 38 // Delete destroys this container after detaching it from all connected endpoints. 39 Delete() error 40 // Endpoints returns all the endpoints connected to the sandbox 41 Endpoints() []Endpoint 42 // ResolveService returns all the backend details about the containers or hosts 43 // backing a service. Its purpose is to satisfy an SRV query 44 ResolveService(name string) ([]*net.SRV, []net.IP) 45 // EnableService makes a managed container's service available by adding the 46 // endpoint to the service load balancer and service discovery 47 EnableService() error 48 // DisableService removes a managed container's endpoints from the load balancer 49 // and service discovery 50 DisableService() error 51 } 52 53 // SandboxOption is an option setter function type used to pass various options to 54 // NewNetContainer method. The various setter functions of type SandboxOption are 55 // provided by libnetwork, they look like ContainerOptionXXXX(...) 56 type SandboxOption func(sb *sandbox) 57 58 func (sb *sandbox) processOptions(options ...SandboxOption) { 59 for _, opt := range options { 60 if opt != nil { 61 opt(sb) 62 } 63 } 64 } 65 66 type sandbox struct { 67 id string 68 containerID string 69 config containerConfig 70 extDNS []extDNSEntry 71 osSbox osl.Sandbox 72 controller *controller 73 resolver Resolver 74 resolverOnce sync.Once 75 endpoints []*endpoint 76 epPriority map[string]int 77 populatedEndpoints map[string]struct{} 78 joinLeaveDone chan struct{} 79 dbIndex uint64 80 dbExists bool 81 isStub bool 82 inDelete bool 83 ingress bool 84 ndotsSet bool 85 oslTypes []osl.SandboxType // slice of properties of this sandbox 86 loadBalancerNID string // NID that this SB is a load balancer for 87 sync.Mutex 88 // This mutex is used to serialize service related operation for an endpoint 89 // The lock is here because the endpoint is saved into the store so is not unique 90 Service sync.Mutex 91 } 92 93 // These are the container configs used to customize container /etc/hosts file. 94 type hostsPathConfig struct { 95 // Note(cpuguy83): The linter is drunk and says none of these fields are used while they are 96 hostName string 97 domainName string 98 hostsPath string 99 originHostsPath string 100 extraHosts []extraHost 101 parentUpdates []parentUpdate 102 } 103 104 type parentUpdate struct { 105 cid string 106 name string 107 ip string 108 } 109 110 type extraHost struct { 111 name string 112 IP string 113 } 114 115 // These are the container configs used to customize container /etc/resolv.conf file. 116 type resolvConfPathConfig struct { 117 // Note(cpuguy83): The linter is drunk and says none of these fields are used while they are 118 resolvConfPath string 119 originResolvConfPath string 120 resolvConfHashFile string 121 dnsList []string 122 dnsSearchList []string 123 dnsOptionsList []string 124 } 125 126 type containerConfig struct { 127 hostsPathConfig 128 resolvConfPathConfig 129 generic map[string]interface{} 130 useDefaultSandBox bool 131 useExternalKey bool 132 exposedPorts []types.TransportPort 133 } 134 135 const ( 136 resolverIPSandbox = "127.0.0.11" 137 ) 138 139 func (sb *sandbox) ID() string { 140 return sb.id 141 } 142 143 func (sb *sandbox) ContainerID() string { 144 return sb.containerID 145 } 146 147 func (sb *sandbox) Key() string { 148 if sb.config.useDefaultSandBox { 149 return osl.GenerateKey("default") 150 } 151 return osl.GenerateKey(sb.id) 152 } 153 154 func (sb *sandbox) Labels() map[string]interface{} { 155 sb.Lock() 156 defer sb.Unlock() 157 opts := make(map[string]interface{}, len(sb.config.generic)) 158 for k, v := range sb.config.generic { 159 opts[k] = v 160 } 161 return opts 162 } 163 164 func (sb *sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) { 165 m := make(map[string]*types.InterfaceStatistics) 166 167 sb.Lock() 168 osb := sb.osSbox 169 sb.Unlock() 170 if osb == nil { 171 return m, nil 172 } 173 174 var err error 175 for _, i := range osb.Info().Interfaces() { 176 if m[i.DstName()], err = i.Statistics(); err != nil { 177 return m, err 178 } 179 } 180 181 return m, nil 182 } 183 184 func (sb *sandbox) Delete() error { 185 return sb.delete(false) 186 } 187 188 func (sb *sandbox) delete(force bool) error { 189 sb.Lock() 190 if sb.inDelete { 191 sb.Unlock() 192 return types.ForbiddenErrorf("another sandbox delete in progress") 193 } 194 // Set the inDelete flag. This will ensure that we don't 195 // update the store until we have completed all the endpoint 196 // leaves and deletes. And when endpoint leaves and deletes 197 // are completed then we can finally delete the sandbox object 198 // altogether from the data store. If the daemon exits 199 // ungracefully in the middle of a sandbox delete this way we 200 // will have all the references to the endpoints in the 201 // sandbox so that we can clean them up when we restart 202 sb.inDelete = true 203 sb.Unlock() 204 205 c := sb.controller 206 207 // Detach from all endpoints 208 retain := false 209 for _, ep := range sb.getConnectedEndpoints() { 210 // gw network endpoint detach and removal are automatic 211 if ep.endpointInGWNetwork() && !force { 212 continue 213 } 214 // Retain the sanbdox if we can't obtain the network from store. 215 if _, err := c.getNetworkFromStore(ep.getNetwork().ID()); err != nil { 216 if c.isDistributedControl() { 217 retain = true 218 } 219 logrus.Warnf("Failed getting network for ep %s during sandbox %s delete: %v", ep.ID(), sb.ID(), err) 220 continue 221 } 222 223 if !force { 224 if err := ep.Leave(sb); err != nil { 225 logrus.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err) 226 } 227 } 228 229 if err := ep.Delete(force); err != nil { 230 logrus.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err) 231 } 232 } 233 234 if retain { 235 sb.Lock() 236 sb.inDelete = false 237 sb.Unlock() 238 return fmt.Errorf("could not cleanup all the endpoints in container %s / sandbox %s", sb.containerID, sb.id) 239 } 240 // Container is going away. Path cache in etchosts is most 241 // likely not required any more. Drop it. 242 etchosts.Drop(sb.config.hostsPath) 243 244 if sb.resolver != nil { 245 sb.resolver.Stop() 246 } 247 248 if sb.osSbox != nil && !sb.config.useDefaultSandBox { 249 if err := sb.osSbox.Destroy(); err != nil { 250 logrus.WithError(err).Warn("error destroying network sandbox") 251 } 252 } 253 254 if err := sb.storeDelete(); err != nil { 255 logrus.Warnf("Failed to delete sandbox %s from store: %v", sb.ID(), err) 256 } 257 258 c.Lock() 259 if sb.ingress { 260 c.ingressSandbox = nil 261 } 262 delete(c.sandboxes, sb.ID()) 263 c.Unlock() 264 265 return nil 266 } 267 268 func (sb *sandbox) Rename(name string) error { 269 var err error 270 271 for _, ep := range sb.getConnectedEndpoints() { 272 if ep.endpointInGWNetwork() { 273 continue 274 } 275 276 oldName := ep.Name() 277 lEp := ep 278 if err = ep.rename(name); err != nil { 279 break 280 } 281 282 defer func() { 283 if err != nil { 284 if err2 := lEp.rename(oldName); err2 != nil { 285 logrus.WithField("old", oldName).WithField("origError", err).WithError(err2).Error("error renaming sandbox") 286 } 287 } 288 }() 289 } 290 291 return err 292 } 293 294 func (sb *sandbox) Refresh(options ...SandboxOption) error { 295 // Store connected endpoints 296 epList := sb.getConnectedEndpoints() 297 298 // Detach from all endpoints 299 for _, ep := range epList { 300 if err := ep.Leave(sb); err != nil { 301 logrus.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err) 302 } 303 } 304 305 // Re-apply options 306 sb.config = containerConfig{} 307 sb.processOptions(options...) 308 309 // Setup discovery files 310 if err := sb.setupResolutionFiles(); err != nil { 311 return err 312 } 313 314 // Re-connect to all endpoints 315 for _, ep := range epList { 316 if err := ep.Join(sb); err != nil { 317 logrus.Warnf("Failed attach sandbox %s to endpoint %s: %v\n", sb.ID(), ep.ID(), err) 318 } 319 } 320 321 return nil 322 } 323 324 func (sb *sandbox) MarshalJSON() ([]byte, error) { 325 sb.Lock() 326 defer sb.Unlock() 327 328 // We are just interested in the container ID. This can be expanded to include all of containerInfo if there is a need 329 return json.Marshal(sb.id) 330 } 331 332 func (sb *sandbox) UnmarshalJSON(b []byte) (err error) { 333 sb.Lock() 334 defer sb.Unlock() 335 336 var id string 337 if err := json.Unmarshal(b, &id); err != nil { 338 return err 339 } 340 sb.id = id 341 return nil 342 } 343 344 func (sb *sandbox) Endpoints() []Endpoint { 345 sb.Lock() 346 defer sb.Unlock() 347 348 endpoints := make([]Endpoint, len(sb.endpoints)) 349 for i, ep := range sb.endpoints { 350 endpoints[i] = ep 351 } 352 return endpoints 353 } 354 355 func (sb *sandbox) getConnectedEndpoints() []*endpoint { 356 sb.Lock() 357 defer sb.Unlock() 358 359 eps := make([]*endpoint, len(sb.endpoints)) 360 copy(eps, sb.endpoints) 361 362 return eps 363 } 364 365 func (sb *sandbox) addEndpoint(ep *endpoint) { 366 sb.Lock() 367 defer sb.Unlock() 368 369 l := len(sb.endpoints) 370 i := sort.Search(l, func(j int) bool { 371 return ep.Less(sb.endpoints[j]) 372 }) 373 374 sb.endpoints = append(sb.endpoints, nil) 375 copy(sb.endpoints[i+1:], sb.endpoints[i:]) 376 sb.endpoints[i] = ep 377 } 378 379 func (sb *sandbox) removeEndpoint(ep *endpoint) { 380 sb.Lock() 381 defer sb.Unlock() 382 383 sb.removeEndpointRaw(ep) 384 } 385 386 func (sb *sandbox) removeEndpointRaw(ep *endpoint) { 387 for i, e := range sb.endpoints { 388 if e == ep { 389 sb.endpoints = append(sb.endpoints[:i], sb.endpoints[i+1:]...) 390 return 391 } 392 } 393 } 394 395 func (sb *sandbox) getEndpoint(id string) *endpoint { 396 sb.Lock() 397 defer sb.Unlock() 398 399 for _, ep := range sb.endpoints { 400 if ep.id == id { 401 return ep 402 } 403 } 404 405 return nil 406 } 407 408 func (sb *sandbox) updateGateway(ep *endpoint) error { 409 sb.Lock() 410 osSbox := sb.osSbox 411 sb.Unlock() 412 if osSbox == nil { 413 return nil 414 } 415 osSbox.UnsetGateway() //nolint:errcheck 416 osSbox.UnsetGatewayIPv6() //nolint:errcheck 417 418 if ep == nil { 419 return nil 420 } 421 422 ep.Lock() 423 joinInfo := ep.joinInfo 424 ep.Unlock() 425 426 if err := osSbox.SetGateway(joinInfo.gw); err != nil { 427 return fmt.Errorf("failed to set gateway while updating gateway: %v", err) 428 } 429 430 if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil { 431 return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err) 432 } 433 434 return nil 435 } 436 437 func (sb *sandbox) HandleQueryResp(name string, ip net.IP) { 438 for _, ep := range sb.getConnectedEndpoints() { 439 n := ep.getNetwork() 440 n.HandleQueryResp(name, ip) 441 } 442 } 443 444 func (sb *sandbox) ResolveIP(ip string) string { 445 var svc string 446 logrus.Debugf("IP To resolve %v", ip) 447 448 for _, ep := range sb.getConnectedEndpoints() { 449 n := ep.getNetwork() 450 svc = n.ResolveIP(ip) 451 if len(svc) != 0 { 452 return svc 453 } 454 } 455 456 return svc 457 } 458 459 func (sb *sandbox) ExecFunc(f func()) error { 460 sb.Lock() 461 osSbox := sb.osSbox 462 sb.Unlock() 463 if osSbox != nil { 464 return osSbox.InvokeFunc(f) 465 } 466 return fmt.Errorf("osl sandbox unavailable in ExecFunc for %v", sb.ContainerID()) 467 } 468 469 func (sb *sandbox) ResolveService(name string) ([]*net.SRV, []net.IP) { 470 srv := []*net.SRV{} 471 ip := []net.IP{} 472 473 logrus.Debugf("Service name To resolve: %v", name) 474 475 // There are DNS implementations that allow SRV queries for names not in 476 // the format defined by RFC 2782. Hence specific validations checks are 477 // not done 478 parts := strings.Split(name, ".") 479 if len(parts) < 3 { 480 return nil, nil 481 } 482 483 for _, ep := range sb.getConnectedEndpoints() { 484 n := ep.getNetwork() 485 486 srv, ip = n.ResolveService(name) 487 if len(srv) > 0 { 488 break 489 } 490 } 491 return srv, ip 492 } 493 494 func getDynamicNwEndpoints(epList []*endpoint) []*endpoint { 495 eps := []*endpoint{} 496 for _, ep := range epList { 497 n := ep.getNetwork() 498 if n.dynamic && !n.ingress { 499 eps = append(eps, ep) 500 } 501 } 502 return eps 503 } 504 505 func getIngressNwEndpoint(epList []*endpoint) *endpoint { 506 for _, ep := range epList { 507 n := ep.getNetwork() 508 if n.ingress { 509 return ep 510 } 511 } 512 return nil 513 } 514 515 func getLocalNwEndpoints(epList []*endpoint) []*endpoint { 516 eps := []*endpoint{} 517 for _, ep := range epList { 518 n := ep.getNetwork() 519 if !n.dynamic && !n.ingress { 520 eps = append(eps, ep) 521 } 522 } 523 return eps 524 } 525 526 func (sb *sandbox) ResolveName(name string, ipType int) ([]net.IP, bool) { 527 // Embedded server owns the docker network domain. Resolution should work 528 // for both container_name and container_name.network_name 529 // We allow '.' in service name and network name. For a name a.b.c.d the 530 // following have to tried; 531 // {a.b.c.d in the networks container is connected to} 532 // {a.b.c in network d}, 533 // {a.b in network c.d}, 534 // {a in network b.c.d}, 535 536 logrus.Debugf("Name To resolve: %v", name) 537 name = strings.TrimSuffix(name, ".") 538 reqName := []string{name} 539 networkName := []string{""} 540 541 if strings.Contains(name, ".") { 542 var i int 543 dup := name 544 for { 545 if i = strings.LastIndex(dup, "."); i == -1 { 546 break 547 } 548 networkName = append(networkName, name[i+1:]) 549 reqName = append(reqName, name[:i]) 550 551 dup = dup[:i] 552 } 553 } 554 555 epList := sb.getConnectedEndpoints() 556 557 // In swarm mode services with exposed ports are connected to user overlay 558 // network, ingress network and docker_gwbridge network. Name resolution 559 // should prioritize returning the VIP/IPs on user overlay network. 560 newList := []*endpoint{} 561 if !sb.controller.isDistributedControl() { 562 newList = append(newList, getDynamicNwEndpoints(epList)...) 563 ingressEP := getIngressNwEndpoint(epList) 564 if ingressEP != nil { 565 newList = append(newList, ingressEP) 566 } 567 newList = append(newList, getLocalNwEndpoints(epList)...) 568 epList = newList 569 } 570 571 for i := 0; i < len(reqName); i++ { 572 573 // First check for local container alias 574 ip, ipv6Miss := sb.resolveName(reqName[i], networkName[i], epList, true, ipType) 575 if ip != nil { 576 return ip, false 577 } 578 if ipv6Miss { 579 return ip, ipv6Miss 580 } 581 582 // Resolve the actual container name 583 ip, ipv6Miss = sb.resolveName(reqName[i], networkName[i], epList, false, ipType) 584 if ip != nil { 585 return ip, false 586 } 587 if ipv6Miss { 588 return ip, ipv6Miss 589 } 590 } 591 return nil, false 592 } 593 594 func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoint, alias bool, ipType int) ([]net.IP, bool) { 595 var ipv6Miss bool 596 597 for _, ep := range epList { 598 name := req 599 n := ep.getNetwork() 600 601 if networkName != "" && networkName != n.Name() { 602 continue 603 } 604 605 if alias { 606 if ep.aliases == nil { 607 continue 608 } 609 610 var ok bool 611 ep.Lock() 612 name, ok = ep.aliases[req] 613 ep.Unlock() 614 if !ok { 615 continue 616 } 617 } else { 618 // If it is a regular lookup and if the requested name is an alias 619 // don't perform a svc lookup for this endpoint. 620 ep.Lock() 621 if _, ok := ep.aliases[req]; ok { 622 ep.Unlock() 623 continue 624 } 625 ep.Unlock() 626 } 627 628 ip, miss := n.ResolveName(name, ipType) 629 630 if ip != nil { 631 return ip, false 632 } 633 634 if miss { 635 ipv6Miss = miss 636 } 637 } 638 return nil, ipv6Miss 639 } 640 641 func (sb *sandbox) SetKey(basePath string) error { 642 start := time.Now() 643 defer func() { 644 logrus.Debugf("sandbox set key processing took %s for container %s", time.Since(start), sb.ContainerID()) 645 }() 646 647 if basePath == "" { 648 return types.BadRequestErrorf("invalid sandbox key") 649 } 650 651 sb.Lock() 652 if sb.inDelete { 653 sb.Unlock() 654 return types.ForbiddenErrorf("failed to SetKey: sandbox %q delete in progress", sb.id) 655 } 656 oldosSbox := sb.osSbox 657 sb.Unlock() 658 659 if oldosSbox != nil { 660 // If we already have an OS sandbox, release the network resources from that 661 // and destroy the OS snab. We are moving into a new home further down. Note that none 662 // of the network resources gets destroyed during the move. 663 sb.releaseOSSbox() 664 } 665 666 osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key()) 667 if err != nil { 668 return err 669 } 670 671 sb.Lock() 672 sb.osSbox = osSbox 673 sb.Unlock() 674 675 // If the resolver was setup before stop it and set it up in the 676 // new osl sandbox. 677 if oldosSbox != nil && sb.resolver != nil { 678 sb.resolver.Stop() 679 680 if err := sb.osSbox.InvokeFunc(sb.resolver.SetupFunc(0)); err == nil { 681 if err := sb.resolver.Start(); err != nil { 682 logrus.Errorf("Resolver Start failed for container %s, %q", sb.ContainerID(), err) 683 } 684 } else { 685 logrus.Errorf("Resolver Setup Function failed for container %s, %q", sb.ContainerID(), err) 686 } 687 } 688 689 for _, ep := range sb.getConnectedEndpoints() { 690 if err = sb.populateNetworkResources(ep); err != nil { 691 return err 692 } 693 } 694 return nil 695 } 696 697 func (sb *sandbox) EnableService() (err error) { 698 logrus.Debugf("EnableService %s START", sb.containerID) 699 defer func() { 700 if err != nil { 701 if err2 := sb.DisableService(); err2 != nil { 702 logrus.WithError(err2).WithField("origError", err).Error("Error while disabling service after original error") 703 } 704 } 705 }() 706 for _, ep := range sb.getConnectedEndpoints() { 707 if !ep.isServiceEnabled() { 708 if err := ep.addServiceInfoToCluster(sb); err != nil { 709 return fmt.Errorf("could not update state for endpoint %s into cluster: %v", ep.Name(), err) 710 } 711 ep.enableService() 712 } 713 } 714 logrus.Debugf("EnableService %s DONE", sb.containerID) 715 return nil 716 } 717 718 func (sb *sandbox) DisableService() (err error) { 719 logrus.Debugf("DisableService %s START", sb.containerID) 720 failedEps := []string{} 721 defer func() { 722 if len(failedEps) > 0 { 723 err = fmt.Errorf("failed to disable service on sandbox:%s, for endpoints %s", sb.ID(), strings.Join(failedEps, ",")) 724 } 725 }() 726 for _, ep := range sb.getConnectedEndpoints() { 727 if ep.isServiceEnabled() { 728 if err := ep.deleteServiceInfoFromCluster(sb, false, "DisableService"); err != nil { 729 failedEps = append(failedEps, ep.Name()) 730 logrus.Warnf("failed update state for endpoint %s into cluster: %v", ep.Name(), err) 731 } 732 ep.disableService() 733 } 734 } 735 logrus.Debugf("DisableService %s DONE", sb.containerID) 736 return nil 737 } 738 739 func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) { 740 for _, i := range osSbox.Info().Interfaces() { 741 // Only remove the interfaces owned by this endpoint from the sandbox. 742 if ep.hasInterface(i.SrcName()) { 743 if err := i.Remove(); err != nil { 744 logrus.Debugf("Remove interface %s failed: %v", i.SrcName(), err) 745 } 746 } 747 } 748 749 ep.Lock() 750 joinInfo := ep.joinInfo 751 vip := ep.virtualIP 752 lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR 753 ep.Unlock() 754 755 if len(vip) > 0 && lbModeIsDSR { 756 ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)} 757 if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil { 758 logrus.WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet) 759 } 760 } 761 762 if joinInfo == nil { 763 return 764 } 765 766 // Remove non-interface routes. 767 for _, r := range joinInfo.StaticRoutes { 768 if err := osSbox.RemoveStaticRoute(r); err != nil { 769 logrus.Debugf("Remove route failed: %v", err) 770 } 771 } 772 } 773 774 func (sb *sandbox) releaseOSSbox() { 775 sb.Lock() 776 osSbox := sb.osSbox 777 sb.osSbox = nil 778 sb.Unlock() 779 780 if osSbox == nil { 781 return 782 } 783 784 for _, ep := range sb.getConnectedEndpoints() { 785 releaseOSSboxResources(osSbox, ep) 786 } 787 788 if err := osSbox.Destroy(); err != nil { 789 logrus.WithError(err).Error("Error destroying os sandbox") 790 } 791 } 792 793 func (sb *sandbox) restoreOslSandbox() error { 794 var routes []*types.StaticRoute 795 796 // restore osl sandbox 797 Ifaces := make(map[string][]osl.IfaceOption) 798 for _, ep := range sb.endpoints { 799 var ifaceOptions []osl.IfaceOption 800 ep.Lock() 801 joinInfo := ep.joinInfo 802 i := ep.iface 803 ep.Unlock() 804 805 if i == nil { 806 logrus.Errorf("error restoring endpoint %s for container %s", ep.Name(), sb.ContainerID()) 807 continue 808 } 809 810 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes)) 811 if i.addrv6 != nil && i.addrv6.IP.To16() != nil { 812 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) 813 } 814 if i.mac != nil { 815 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) 816 } 817 if len(i.llAddrs) != 0 { 818 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) 819 } 820 Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions 821 if joinInfo != nil { 822 routes = append(routes, joinInfo.StaticRoutes...) 823 } 824 if ep.needResolver() { 825 sb.startResolver(true) 826 } 827 } 828 829 gwep := sb.getGatewayEndpoint() 830 if gwep == nil { 831 return nil 832 } 833 834 // restore osl sandbox 835 err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6) 836 return err 837 } 838 839 func (sb *sandbox) populateNetworkResources(ep *endpoint) error { 840 sb.Lock() 841 if sb.osSbox == nil { 842 sb.Unlock() 843 return nil 844 } 845 inDelete := sb.inDelete 846 sb.Unlock() 847 848 ep.Lock() 849 joinInfo := ep.joinInfo 850 i := ep.iface 851 lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR 852 ep.Unlock() 853 854 if ep.needResolver() { 855 sb.startResolver(false) 856 } 857 858 if i != nil && i.srcName != "" { 859 var ifaceOptions []osl.IfaceOption 860 861 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes)) 862 if i.addrv6 != nil && i.addrv6.IP.To16() != nil { 863 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) 864 } 865 if len(i.llAddrs) != 0 { 866 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) 867 } 868 if i.mac != nil { 869 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) 870 } 871 872 if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil { 873 return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err) 874 } 875 876 if len(ep.virtualIP) > 0 && lbModeIsDSR { 877 if sb.loadBalancerNID == "" { 878 if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil { 879 return fmt.Errorf("failed disable ARP for VIP: %v", err) 880 } 881 } 882 ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)} 883 if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil { 884 return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err) 885 } 886 } 887 } 888 889 if joinInfo != nil { 890 // Set up non-interface routes. 891 for _, r := range joinInfo.StaticRoutes { 892 if err := sb.osSbox.AddStaticRoute(r); err != nil { 893 return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err) 894 } 895 } 896 } 897 898 if ep == sb.getGatewayEndpoint() { 899 if err := sb.updateGateway(ep); err != nil { 900 return err 901 } 902 } 903 904 // Make sure to add the endpoint to the populated endpoint set 905 // before populating loadbalancers. 906 sb.Lock() 907 sb.populatedEndpoints[ep.ID()] = struct{}{} 908 sb.Unlock() 909 910 // Populate load balancer only after updating all the other 911 // information including gateway and other routes so that 912 // loadbalancers are populated all the network state is in 913 // place in the sandbox. 914 sb.populateLoadBalancers(ep) 915 916 // Only update the store if we did not come here as part of 917 // sandbox delete. If we came here as part of delete then do 918 // not bother updating the store. The sandbox object will be 919 // deleted anyway 920 if !inDelete { 921 return sb.storeUpdate() 922 } 923 924 return nil 925 } 926 927 func (sb *sandbox) clearNetworkResources(origEp *endpoint) error { 928 ep := sb.getEndpoint(origEp.id) 929 if ep == nil { 930 return fmt.Errorf("could not find the sandbox endpoint data for endpoint %s", 931 origEp.id) 932 } 933 934 sb.Lock() 935 osSbox := sb.osSbox 936 inDelete := sb.inDelete 937 sb.Unlock() 938 if osSbox != nil { 939 releaseOSSboxResources(osSbox, ep) 940 } 941 942 sb.Lock() 943 delete(sb.populatedEndpoints, ep.ID()) 944 945 if len(sb.endpoints) == 0 { 946 // sb.endpoints should never be empty and this is unexpected error condition 947 // We log an error message to note this down for debugging purposes. 948 logrus.Errorf("No endpoints in sandbox while trying to remove endpoint %s", ep.Name()) 949 sb.Unlock() 950 return nil 951 } 952 953 var ( 954 gwepBefore, gwepAfter *endpoint 955 index = -1 956 ) 957 for i, e := range sb.endpoints { 958 if e == ep { 959 index = i 960 } 961 if len(e.Gateway()) > 0 && gwepBefore == nil { 962 gwepBefore = e 963 } 964 if index != -1 && gwepBefore != nil { 965 break 966 } 967 } 968 969 if index == -1 { 970 logrus.Warnf("Endpoint %s has already been deleted", ep.Name()) 971 sb.Unlock() 972 return nil 973 } 974 975 sb.removeEndpointRaw(ep) 976 for _, e := range sb.endpoints { 977 if len(e.Gateway()) > 0 { 978 gwepAfter = e 979 break 980 } 981 } 982 delete(sb.epPriority, ep.ID()) 983 sb.Unlock() 984 985 if gwepAfter != nil && gwepBefore != gwepAfter { 986 if err := sb.updateGateway(gwepAfter); err != nil { 987 return err 988 } 989 } 990 991 // Only update the store if we did not come here as part of 992 // sandbox delete. If we came here as part of delete then do 993 // not bother updating the store. The sandbox object will be 994 // deleted anyway 995 if !inDelete { 996 return sb.storeUpdate() 997 } 998 999 return nil 1000 } 1001 1002 // joinLeaveStart waits to ensure there are no joins or leaves in progress and 1003 // marks this join/leave in progress without race 1004 func (sb *sandbox) joinLeaveStart() { 1005 sb.Lock() 1006 defer sb.Unlock() 1007 1008 for sb.joinLeaveDone != nil { 1009 joinLeaveDone := sb.joinLeaveDone 1010 sb.Unlock() 1011 1012 <-joinLeaveDone 1013 1014 sb.Lock() 1015 } 1016 1017 sb.joinLeaveDone = make(chan struct{}) 1018 } 1019 1020 // joinLeaveEnd marks the end of this join/leave operation and 1021 // signals the same without race to other join and leave waiters 1022 func (sb *sandbox) joinLeaveEnd() { 1023 sb.Lock() 1024 defer sb.Unlock() 1025 1026 if sb.joinLeaveDone != nil { 1027 close(sb.joinLeaveDone) 1028 sb.joinLeaveDone = nil 1029 } 1030 } 1031 1032 // OptionHostname function returns an option setter for hostname option to 1033 // be passed to NewSandbox method. 1034 func OptionHostname(name string) SandboxOption { 1035 return func(sb *sandbox) { 1036 sb.config.hostName = name 1037 } 1038 } 1039 1040 // OptionDomainname function returns an option setter for domainname option to 1041 // be passed to NewSandbox method. 1042 func OptionDomainname(name string) SandboxOption { 1043 return func(sb *sandbox) { 1044 sb.config.domainName = name 1045 } 1046 } 1047 1048 // OptionHostsPath function returns an option setter for hostspath option to 1049 // be passed to NewSandbox method. 1050 func OptionHostsPath(path string) SandboxOption { 1051 return func(sb *sandbox) { 1052 sb.config.hostsPath = path 1053 } 1054 } 1055 1056 // OptionOriginHostsPath function returns an option setter for origin hosts file path 1057 // to be passed to NewSandbox method. 1058 func OptionOriginHostsPath(path string) SandboxOption { 1059 return func(sb *sandbox) { 1060 sb.config.originHostsPath = path 1061 } 1062 } 1063 1064 // OptionExtraHost function returns an option setter for extra /etc/hosts options 1065 // which is a name and IP as strings. 1066 func OptionExtraHost(name string, IP string) SandboxOption { 1067 return func(sb *sandbox) { 1068 sb.config.extraHosts = append(sb.config.extraHosts, extraHost{name: name, IP: IP}) 1069 } 1070 } 1071 1072 // OptionParentUpdate function returns an option setter for parent container 1073 // which needs to update the IP address for the linked container. 1074 func OptionParentUpdate(cid string, name, ip string) SandboxOption { 1075 return func(sb *sandbox) { 1076 sb.config.parentUpdates = append(sb.config.parentUpdates, parentUpdate{cid: cid, name: name, ip: ip}) 1077 } 1078 } 1079 1080 // OptionResolvConfPath function returns an option setter for resolvconfpath option to 1081 // be passed to net container methods. 1082 func OptionResolvConfPath(path string) SandboxOption { 1083 return func(sb *sandbox) { 1084 sb.config.resolvConfPath = path 1085 } 1086 } 1087 1088 // OptionOriginResolvConfPath function returns an option setter to set the path to the 1089 // origin resolv.conf file to be passed to net container methods. 1090 func OptionOriginResolvConfPath(path string) SandboxOption { 1091 return func(sb *sandbox) { 1092 sb.config.originResolvConfPath = path 1093 } 1094 } 1095 1096 // OptionDNS function returns an option setter for dns entry option to 1097 // be passed to container Create method. 1098 func OptionDNS(dns string) SandboxOption { 1099 return func(sb *sandbox) { 1100 sb.config.dnsList = append(sb.config.dnsList, dns) 1101 } 1102 } 1103 1104 // OptionDNSSearch function returns an option setter for dns search entry option to 1105 // be passed to container Create method. 1106 func OptionDNSSearch(search string) SandboxOption { 1107 return func(sb *sandbox) { 1108 sb.config.dnsSearchList = append(sb.config.dnsSearchList, search) 1109 } 1110 } 1111 1112 // OptionDNSOptions function returns an option setter for dns options entry option to 1113 // be passed to container Create method. 1114 func OptionDNSOptions(options string) SandboxOption { 1115 return func(sb *sandbox) { 1116 sb.config.dnsOptionsList = append(sb.config.dnsOptionsList, options) 1117 } 1118 } 1119 1120 // OptionUseDefaultSandbox function returns an option setter for using default sandbox 1121 // (host namespace) to be passed to container Create method. 1122 func OptionUseDefaultSandbox() SandboxOption { 1123 return func(sb *sandbox) { 1124 sb.config.useDefaultSandBox = true 1125 } 1126 } 1127 1128 // OptionUseExternalKey function returns an option setter for using provided namespace 1129 // instead of creating one. 1130 func OptionUseExternalKey() SandboxOption { 1131 return func(sb *sandbox) { 1132 sb.config.useExternalKey = true 1133 } 1134 } 1135 1136 // OptionGeneric function returns an option setter for Generic configuration 1137 // that is not managed by libNetwork but can be used by the Drivers during the call to 1138 // net container creation method. Container Labels are a good example. 1139 func OptionGeneric(generic map[string]interface{}) SandboxOption { 1140 return func(sb *sandbox) { 1141 if sb.config.generic == nil { 1142 sb.config.generic = make(map[string]interface{}, len(generic)) 1143 } 1144 for k, v := range generic { 1145 sb.config.generic[k] = v 1146 } 1147 } 1148 } 1149 1150 // OptionExposedPorts function returns an option setter for the container exposed 1151 // ports option to be passed to container Create method. 1152 func OptionExposedPorts(exposedPorts []types.TransportPort) SandboxOption { 1153 return func(sb *sandbox) { 1154 if sb.config.generic == nil { 1155 sb.config.generic = make(map[string]interface{}) 1156 } 1157 // Defensive copy 1158 eps := make([]types.TransportPort, len(exposedPorts)) 1159 copy(eps, exposedPorts) 1160 // Store endpoint label and in generic because driver needs it 1161 sb.config.exposedPorts = eps 1162 sb.config.generic[netlabel.ExposedPorts] = eps 1163 } 1164 } 1165 1166 // OptionPortMapping function returns an option setter for the mapping 1167 // ports option to be passed to container Create method. 1168 func OptionPortMapping(portBindings []types.PortBinding) SandboxOption { 1169 return func(sb *sandbox) { 1170 if sb.config.generic == nil { 1171 sb.config.generic = make(map[string]interface{}) 1172 } 1173 // Store a copy of the bindings as generic data to pass to the driver 1174 pbs := make([]types.PortBinding, len(portBindings)) 1175 copy(pbs, portBindings) 1176 sb.config.generic[netlabel.PortMap] = pbs 1177 } 1178 } 1179 1180 // OptionIngress function returns an option setter for marking a 1181 // sandbox as the controller's ingress sandbox. 1182 func OptionIngress() SandboxOption { 1183 return func(sb *sandbox) { 1184 sb.ingress = true 1185 sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeIngress) 1186 } 1187 } 1188 1189 // OptionLoadBalancer function returns an option setter for marking a 1190 // sandbox as a load balancer sandbox. 1191 func OptionLoadBalancer(nid string) SandboxOption { 1192 return func(sb *sandbox) { 1193 sb.loadBalancerNID = nid 1194 sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeLoadBalancer) 1195 } 1196 } 1197 1198 // <=> Returns true if a < b, false if a > b and advances to next level if a == b 1199 // epi.prio <=> epj.prio # 2 < 1 1200 // epi.gw <=> epj.gw # non-gw < gw 1201 // epi.internal <=> epj.internal # non-internal < internal 1202 // epi.joininfo <=> epj.joininfo # ipv6 < ipv4 1203 // epi.name <=> epj.name # bar < foo 1204 func (epi *endpoint) Less(epj *endpoint) bool { 1205 var ( 1206 prioi, prioj int 1207 ) 1208 1209 sbi, _ := epi.getSandbox() 1210 sbj, _ := epj.getSandbox() 1211 1212 // Prio defaults to 0 1213 if sbi != nil { 1214 prioi = sbi.epPriority[epi.ID()] 1215 } 1216 if sbj != nil { 1217 prioj = sbj.epPriority[epj.ID()] 1218 } 1219 1220 if prioi != prioj { 1221 return prioi > prioj 1222 } 1223 1224 gwi := epi.endpointInGWNetwork() 1225 gwj := epj.endpointInGWNetwork() 1226 if gwi != gwj { 1227 return gwj 1228 } 1229 1230 inti := epi.getNetwork().Internal() 1231 intj := epj.getNetwork().Internal() 1232 if inti != intj { 1233 return intj 1234 } 1235 1236 jii := 0 1237 if epi.joinInfo != nil { 1238 if epi.joinInfo.gw != nil { 1239 jii = jii + 1 1240 } 1241 if epi.joinInfo.gw6 != nil { 1242 jii = jii + 2 1243 } 1244 } 1245 1246 jij := 0 1247 if epj.joinInfo != nil { 1248 if epj.joinInfo.gw != nil { 1249 jij = jij + 1 1250 } 1251 if epj.joinInfo.gw6 != nil { 1252 jij = jij + 2 1253 } 1254 } 1255 1256 if jii != jij { 1257 return jii > jij 1258 } 1259 1260 return epi.network.Name() < epj.network.Name() 1261 } 1262 1263 func (sb *sandbox) NdotsSet() bool { 1264 return sb.ndotsSet 1265 }