github.com/adityamillind98/moby@v23.0.0-rc.4+incompatible/libnetwork/sandbox.go (about) 1 package libnetwork 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "net" 7 "sort" 8 "strings" 9 "sync" 10 "time" 11 12 "github.com/docker/docker/libnetwork/etchosts" 13 "github.com/docker/docker/libnetwork/netlabel" 14 "github.com/docker/docker/libnetwork/osl" 15 "github.com/docker/docker/libnetwork/types" 16 "github.com/sirupsen/logrus" 17 ) 18 19 // Sandbox provides the control over the network container entity. It is a one to one mapping with the container. 20 type Sandbox interface { 21 // ID returns the ID of the sandbox 22 ID() string 23 // Key returns the sandbox's key 24 Key() string 25 // ContainerID returns the container id associated to this sandbox 26 ContainerID() string 27 // Labels returns the sandbox's labels 28 Labels() map[string]interface{} 29 // Statistics retrieves the interfaces' statistics for the sandbox 30 Statistics() (map[string]*types.InterfaceStatistics, error) 31 // Refresh leaves all the endpoints, resets and re-applies the options, 32 // re-joins all the endpoints without destroying the osl sandbox 33 Refresh(options ...SandboxOption) error 34 // SetKey updates the Sandbox Key 35 SetKey(key string) error 36 // Rename changes the name of all attached Endpoints 37 Rename(name string) error 38 // Delete destroys this container after detaching it from all connected endpoints. 39 Delete() error 40 // Endpoints returns all the endpoints connected to the sandbox 41 Endpoints() []Endpoint 42 // ResolveService returns all the backend details about the containers or hosts 43 // backing a service. Its purpose is to satisfy an SRV query 44 ResolveService(name string) ([]*net.SRV, []net.IP) 45 // EnableService makes a managed container's service available by adding the 46 // endpoint to the service load balancer and service discovery 47 EnableService() error 48 // DisableService removes a managed container's endpoints from the load balancer 49 // and service discovery 50 DisableService() error 51 } 52 53 // SandboxOption is an option setter function type used to pass various options to 54 // NewNetContainer method. The various setter functions of type SandboxOption are 55 // provided by libnetwork, they look like ContainerOptionXXXX(...) 56 type SandboxOption func(sb *sandbox) 57 58 func (sb *sandbox) processOptions(options ...SandboxOption) { 59 for _, opt := range options { 60 if opt != nil { 61 opt(sb) 62 } 63 } 64 } 65 66 type sandbox struct { 67 id string 68 containerID string 69 config containerConfig 70 extDNS []extDNSEntry 71 osSbox osl.Sandbox 72 controller *controller 73 resolver Resolver 74 resolverOnce sync.Once 75 endpoints []*endpoint 76 epPriority map[string]int 77 populatedEndpoints map[string]struct{} 78 joinLeaveDone chan struct{} 79 dbIndex uint64 80 dbExists bool 81 isStub bool 82 inDelete bool 83 ingress bool 84 ndotsSet bool 85 oslTypes []osl.SandboxType // slice of properties of this sandbox 86 loadBalancerNID string // NID that this SB is a load balancer for 87 sync.Mutex 88 // This mutex is used to serialize service related operation for an endpoint 89 // The lock is here because the endpoint is saved into the store so is not unique 90 Service sync.Mutex 91 } 92 93 // These are the container configs used to customize container /etc/hosts file. 94 type hostsPathConfig struct { 95 // Note(cpuguy83): The linter is drunk and says none of these fields are used while they are 96 hostName string 97 domainName string 98 hostsPath string 99 originHostsPath string 100 extraHosts []extraHost 101 parentUpdates []parentUpdate 102 } 103 104 type parentUpdate struct { 105 cid string 106 name string 107 ip string 108 } 109 110 type extraHost struct { 111 name string 112 IP string 113 } 114 115 // These are the container configs used to customize container /etc/resolv.conf file. 116 type resolvConfPathConfig struct { 117 // Note(cpuguy83): The linter is drunk and says none of these fields are used while they are 118 resolvConfPath string 119 originResolvConfPath string 120 resolvConfHashFile string 121 dnsList []string 122 dnsSearchList []string 123 dnsOptionsList []string 124 } 125 126 type containerConfig struct { 127 hostsPathConfig 128 resolvConfPathConfig 129 generic map[string]interface{} 130 useDefaultSandBox bool 131 useExternalKey bool 132 exposedPorts []types.TransportPort 133 } 134 135 const ( 136 resolverIPSandbox = "127.0.0.11" 137 ) 138 139 func (sb *sandbox) ID() string { 140 return sb.id 141 } 142 143 func (sb *sandbox) ContainerID() string { 144 return sb.containerID 145 } 146 147 func (sb *sandbox) Key() string { 148 if sb.config.useDefaultSandBox { 149 return osl.GenerateKey("default") 150 } 151 return osl.GenerateKey(sb.id) 152 } 153 154 func (sb *sandbox) Labels() map[string]interface{} { 155 sb.Lock() 156 defer sb.Unlock() 157 opts := make(map[string]interface{}, len(sb.config.generic)) 158 for k, v := range sb.config.generic { 159 opts[k] = v 160 } 161 return opts 162 } 163 164 func (sb *sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) { 165 m := make(map[string]*types.InterfaceStatistics) 166 167 sb.Lock() 168 osb := sb.osSbox 169 sb.Unlock() 170 if osb == nil { 171 return m, nil 172 } 173 174 var err error 175 for _, i := range osb.Info().Interfaces() { 176 if m[i.DstName()], err = i.Statistics(); err != nil { 177 return m, err 178 } 179 } 180 181 return m, nil 182 } 183 184 func (sb *sandbox) Delete() error { 185 return sb.delete(false) 186 } 187 188 func (sb *sandbox) delete(force bool) error { 189 sb.Lock() 190 if sb.inDelete { 191 sb.Unlock() 192 return types.ForbiddenErrorf("another sandbox delete in progress") 193 } 194 // Set the inDelete flag. This will ensure that we don't 195 // update the store until we have completed all the endpoint 196 // leaves and deletes. And when endpoint leaves and deletes 197 // are completed then we can finally delete the sandbox object 198 // altogether from the data store. If the daemon exits 199 // ungracefully in the middle of a sandbox delete this way we 200 // will have all the references to the endpoints in the 201 // sandbox so that we can clean them up when we restart 202 sb.inDelete = true 203 sb.Unlock() 204 205 c := sb.controller 206 207 // Detach from all endpoints 208 retain := false 209 for _, ep := range sb.getConnectedEndpoints() { 210 // gw network endpoint detach and removal are automatic 211 if ep.endpointInGWNetwork() && !force { 212 continue 213 } 214 // Retain the sanbdox if we can't obtain the network from store. 215 if _, err := c.getNetworkFromStore(ep.getNetwork().ID()); err != nil { 216 if c.isDistributedControl() { 217 retain = true 218 } 219 logrus.Warnf("Failed getting network for ep %s during sandbox %s delete: %v", ep.ID(), sb.ID(), err) 220 continue 221 } 222 223 if !force { 224 if err := ep.Leave(sb); err != nil { 225 logrus.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err) 226 } 227 } 228 229 if err := ep.Delete(force); err != nil { 230 logrus.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err) 231 } 232 } 233 234 if retain { 235 sb.Lock() 236 sb.inDelete = false 237 sb.Unlock() 238 return fmt.Errorf("could not cleanup all the endpoints in container %s / sandbox %s", sb.containerID, sb.id) 239 } 240 // Container is going away. Path cache in etchosts is most 241 // likely not required any more. Drop it. 242 etchosts.Drop(sb.config.hostsPath) 243 244 if sb.resolver != nil { 245 sb.resolver.Stop() 246 } 247 248 if sb.osSbox != nil && !sb.config.useDefaultSandBox { 249 if err := sb.osSbox.Destroy(); err != nil { 250 logrus.WithError(err).Warn("error destroying network sandbox") 251 } 252 } 253 254 if err := sb.storeDelete(); err != nil { 255 logrus.Warnf("Failed to delete sandbox %s from store: %v", sb.ID(), err) 256 } 257 258 c.Lock() 259 if sb.ingress { 260 c.ingressSandbox = nil 261 } 262 delete(c.sandboxes, sb.ID()) 263 c.Unlock() 264 265 return nil 266 } 267 268 func (sb *sandbox) Rename(name string) error { 269 var err error 270 271 for _, ep := range sb.getConnectedEndpoints() { 272 if ep.endpointInGWNetwork() { 273 continue 274 } 275 276 oldName := ep.Name() 277 lEp := ep 278 if err = ep.rename(name); err != nil { 279 break 280 } 281 282 defer func() { 283 if err != nil { 284 if err2 := lEp.rename(oldName); err2 != nil { 285 logrus.WithField("old", oldName).WithField("origError", err).WithError(err2).Error("error renaming sandbox") 286 } 287 } 288 }() 289 } 290 291 return err 292 } 293 294 func (sb *sandbox) Refresh(options ...SandboxOption) error { 295 // Store connected endpoints 296 epList := sb.getConnectedEndpoints() 297 298 // Detach from all endpoints 299 for _, ep := range epList { 300 if err := ep.Leave(sb); err != nil { 301 logrus.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err) 302 } 303 } 304 305 // Re-apply options 306 sb.config = containerConfig{} 307 sb.processOptions(options...) 308 309 // Setup discovery files 310 if err := sb.setupResolutionFiles(); err != nil { 311 return err 312 } 313 314 // Re-connect to all endpoints 315 for _, ep := range epList { 316 if err := ep.Join(sb); err != nil { 317 logrus.Warnf("Failed attach sandbox %s to endpoint %s: %v\n", sb.ID(), ep.ID(), err) 318 } 319 } 320 321 return nil 322 } 323 324 func (sb *sandbox) MarshalJSON() ([]byte, error) { 325 sb.Lock() 326 defer sb.Unlock() 327 328 // We are just interested in the container ID. This can be expanded to include all of containerInfo if there is a need 329 return json.Marshal(sb.id) 330 } 331 332 func (sb *sandbox) UnmarshalJSON(b []byte) (err error) { 333 sb.Lock() 334 defer sb.Unlock() 335 336 var id string 337 if err := json.Unmarshal(b, &id); err != nil { 338 return err 339 } 340 sb.id = id 341 return nil 342 } 343 344 func (sb *sandbox) Endpoints() []Endpoint { 345 sb.Lock() 346 defer sb.Unlock() 347 348 endpoints := make([]Endpoint, len(sb.endpoints)) 349 for i, ep := range sb.endpoints { 350 endpoints[i] = ep 351 } 352 return endpoints 353 } 354 355 func (sb *sandbox) getConnectedEndpoints() []*endpoint { 356 sb.Lock() 357 defer sb.Unlock() 358 359 eps := make([]*endpoint, len(sb.endpoints)) 360 copy(eps, sb.endpoints) 361 362 return eps 363 } 364 365 func (sb *sandbox) addEndpoint(ep *endpoint) { 366 sb.Lock() 367 defer sb.Unlock() 368 369 l := len(sb.endpoints) 370 i := sort.Search(l, func(j int) bool { 371 return ep.Less(sb.endpoints[j]) 372 }) 373 374 sb.endpoints = append(sb.endpoints, nil) 375 copy(sb.endpoints[i+1:], sb.endpoints[i:]) 376 sb.endpoints[i] = ep 377 } 378 379 func (sb *sandbox) removeEndpoint(ep *endpoint) { 380 sb.Lock() 381 defer sb.Unlock() 382 383 sb.removeEndpointRaw(ep) 384 } 385 386 func (sb *sandbox) removeEndpointRaw(ep *endpoint) { 387 for i, e := range sb.endpoints { 388 if e == ep { 389 sb.endpoints = append(sb.endpoints[:i], sb.endpoints[i+1:]...) 390 return 391 } 392 } 393 } 394 395 func (sb *sandbox) getEndpoint(id string) *endpoint { 396 sb.Lock() 397 defer sb.Unlock() 398 399 for _, ep := range sb.endpoints { 400 if ep.id == id { 401 return ep 402 } 403 } 404 405 return nil 406 } 407 408 func (sb *sandbox) updateGateway(ep *endpoint) error { 409 sb.Lock() 410 osSbox := sb.osSbox 411 sb.Unlock() 412 if osSbox == nil { 413 return nil 414 } 415 osSbox.UnsetGateway() //nolint:errcheck 416 osSbox.UnsetGatewayIPv6() //nolint:errcheck 417 418 if ep == nil { 419 return nil 420 } 421 422 ep.Lock() 423 joinInfo := ep.joinInfo 424 ep.Unlock() 425 426 if err := osSbox.SetGateway(joinInfo.gw); err != nil { 427 return fmt.Errorf("failed to set gateway while updating gateway: %v", err) 428 } 429 430 if err := osSbox.SetGatewayIPv6(joinInfo.gw6); err != nil { 431 return fmt.Errorf("failed to set IPv6 gateway while updating gateway: %v", err) 432 } 433 434 return nil 435 } 436 437 func (sb *sandbox) HandleQueryResp(name string, ip net.IP) { 438 for _, ep := range sb.getConnectedEndpoints() { 439 n := ep.getNetwork() 440 n.HandleQueryResp(name, ip) 441 } 442 } 443 444 func (sb *sandbox) ResolveIP(ip string) string { 445 var svc string 446 logrus.Debugf("IP To resolve %v", ip) 447 448 for _, ep := range sb.getConnectedEndpoints() { 449 n := ep.getNetwork() 450 svc = n.ResolveIP(ip) 451 if len(svc) != 0 { 452 return svc 453 } 454 } 455 456 return svc 457 } 458 459 func (sb *sandbox) ExecFunc(f func()) error { 460 sb.Lock() 461 osSbox := sb.osSbox 462 sb.Unlock() 463 if osSbox != nil { 464 return osSbox.InvokeFunc(f) 465 } 466 return fmt.Errorf("osl sandbox unavailable in ExecFunc for %v", sb.ContainerID()) 467 } 468 469 func (sb *sandbox) ResolveService(name string) ([]*net.SRV, []net.IP) { 470 srv := []*net.SRV{} 471 ip := []net.IP{} 472 473 logrus.Debugf("Service name To resolve: %v", name) 474 475 // There are DNS implementations that allow SRV queries for names not in 476 // the format defined by RFC 2782. Hence specific validations checks are 477 // not done 478 parts := strings.Split(name, ".") 479 if len(parts) < 3 { 480 return nil, nil 481 } 482 483 for _, ep := range sb.getConnectedEndpoints() { 484 n := ep.getNetwork() 485 486 srv, ip = n.ResolveService(name) 487 if len(srv) > 0 { 488 break 489 } 490 } 491 return srv, ip 492 } 493 494 func getDynamicNwEndpoints(epList []*endpoint) []*endpoint { 495 eps := []*endpoint{} 496 for _, ep := range epList { 497 n := ep.getNetwork() 498 if n.dynamic && !n.ingress { 499 eps = append(eps, ep) 500 } 501 } 502 return eps 503 } 504 505 func getIngressNwEndpoint(epList []*endpoint) *endpoint { 506 for _, ep := range epList { 507 n := ep.getNetwork() 508 if n.ingress { 509 return ep 510 } 511 } 512 return nil 513 } 514 515 func getLocalNwEndpoints(epList []*endpoint) []*endpoint { 516 eps := []*endpoint{} 517 for _, ep := range epList { 518 n := ep.getNetwork() 519 if !n.dynamic && !n.ingress { 520 eps = append(eps, ep) 521 } 522 } 523 return eps 524 } 525 526 func (sb *sandbox) ResolveName(name string, ipType int) ([]net.IP, bool) { 527 // Embedded server owns the docker network domain. Resolution should work 528 // for both container_name and container_name.network_name 529 // We allow '.' in service name and network name. For a name a.b.c.d the 530 // following have to tried; 531 // {a.b.c.d in the networks container is connected to} 532 // {a.b.c in network d}, 533 // {a.b in network c.d}, 534 // {a in network b.c.d}, 535 536 logrus.Debugf("Name To resolve: %v", name) 537 name = strings.TrimSuffix(name, ".") 538 reqName := []string{name} 539 networkName := []string{""} 540 541 if strings.Contains(name, ".") { 542 var i int 543 dup := name 544 for { 545 if i = strings.LastIndex(dup, "."); i == -1 { 546 break 547 } 548 networkName = append(networkName, name[i+1:]) 549 reqName = append(reqName, name[:i]) 550 551 dup = dup[:i] 552 } 553 } 554 555 epList := sb.getConnectedEndpoints() 556 557 // In swarm mode services with exposed ports are connected to user overlay 558 // network, ingress network and docker_gwbridge network. Name resolution 559 // should prioritize returning the VIP/IPs on user overlay network. 560 newList := []*endpoint{} 561 if !sb.controller.isDistributedControl() { 562 newList = append(newList, getDynamicNwEndpoints(epList)...) 563 ingressEP := getIngressNwEndpoint(epList) 564 if ingressEP != nil { 565 newList = append(newList, ingressEP) 566 } 567 newList = append(newList, getLocalNwEndpoints(epList)...) 568 epList = newList 569 } 570 571 for i := 0; i < len(reqName); i++ { 572 // First check for local container alias 573 ip, ipv6Miss := sb.resolveName(reqName[i], networkName[i], epList, true, ipType) 574 if ip != nil { 575 return ip, false 576 } 577 if ipv6Miss { 578 return ip, ipv6Miss 579 } 580 581 // Resolve the actual container name 582 ip, ipv6Miss = sb.resolveName(reqName[i], networkName[i], epList, false, ipType) 583 if ip != nil { 584 return ip, false 585 } 586 if ipv6Miss { 587 return ip, ipv6Miss 588 } 589 } 590 return nil, false 591 } 592 593 func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoint, alias bool, ipType int) ([]net.IP, bool) { 594 var ipv6Miss bool 595 596 for _, ep := range epList { 597 name := req 598 n := ep.getNetwork() 599 600 if networkName != "" && networkName != n.Name() { 601 continue 602 } 603 604 if alias { 605 if ep.aliases == nil { 606 continue 607 } 608 609 var ok bool 610 ep.Lock() 611 name, ok = ep.aliases[req] 612 ep.Unlock() 613 if !ok { 614 continue 615 } 616 } else { 617 // If it is a regular lookup and if the requested name is an alias 618 // don't perform a svc lookup for this endpoint. 619 ep.Lock() 620 if _, ok := ep.aliases[req]; ok { 621 ep.Unlock() 622 continue 623 } 624 ep.Unlock() 625 } 626 627 ip, miss := n.ResolveName(name, ipType) 628 629 if ip != nil { 630 return ip, false 631 } 632 633 if miss { 634 ipv6Miss = miss 635 } 636 } 637 return nil, ipv6Miss 638 } 639 640 func (sb *sandbox) SetKey(basePath string) error { 641 start := time.Now() 642 defer func() { 643 logrus.Debugf("sandbox set key processing took %s for container %s", time.Since(start), sb.ContainerID()) 644 }() 645 646 if basePath == "" { 647 return types.BadRequestErrorf("invalid sandbox key") 648 } 649 650 sb.Lock() 651 if sb.inDelete { 652 sb.Unlock() 653 return types.ForbiddenErrorf("failed to SetKey: sandbox %q delete in progress", sb.id) 654 } 655 oldosSbox := sb.osSbox 656 sb.Unlock() 657 658 if oldosSbox != nil { 659 // If we already have an OS sandbox, release the network resources from that 660 // and destroy the OS snab. We are moving into a new home further down. Note that none 661 // of the network resources gets destroyed during the move. 662 sb.releaseOSSbox() 663 } 664 665 osSbox, err := osl.GetSandboxForExternalKey(basePath, sb.Key()) 666 if err != nil { 667 return err 668 } 669 670 sb.Lock() 671 sb.osSbox = osSbox 672 sb.Unlock() 673 674 // If the resolver was setup before stop it and set it up in the 675 // new osl sandbox. 676 if oldosSbox != nil && sb.resolver != nil { 677 sb.resolver.Stop() 678 679 if err := sb.osSbox.InvokeFunc(sb.resolver.SetupFunc(0)); err == nil { 680 if err := sb.resolver.Start(); err != nil { 681 logrus.Errorf("Resolver Start failed for container %s, %q", sb.ContainerID(), err) 682 } 683 } else { 684 logrus.Errorf("Resolver Setup Function failed for container %s, %q", sb.ContainerID(), err) 685 } 686 } 687 688 for _, ep := range sb.getConnectedEndpoints() { 689 if err = sb.populateNetworkResources(ep); err != nil { 690 return err 691 } 692 } 693 return nil 694 } 695 696 func (sb *sandbox) EnableService() (err error) { 697 logrus.Debugf("EnableService %s START", sb.containerID) 698 defer func() { 699 if err != nil { 700 if err2 := sb.DisableService(); err2 != nil { 701 logrus.WithError(err2).WithField("origError", err).Error("Error while disabling service after original error") 702 } 703 } 704 }() 705 for _, ep := range sb.getConnectedEndpoints() { 706 if !ep.isServiceEnabled() { 707 if err := ep.addServiceInfoToCluster(sb); err != nil { 708 return fmt.Errorf("could not update state for endpoint %s into cluster: %v", ep.Name(), err) 709 } 710 ep.enableService() 711 } 712 } 713 logrus.Debugf("EnableService %s DONE", sb.containerID) 714 return nil 715 } 716 717 func (sb *sandbox) DisableService() (err error) { 718 logrus.Debugf("DisableService %s START", sb.containerID) 719 failedEps := []string{} 720 defer func() { 721 if len(failedEps) > 0 { 722 err = fmt.Errorf("failed to disable service on sandbox:%s, for endpoints %s", sb.ID(), strings.Join(failedEps, ",")) 723 } 724 }() 725 for _, ep := range sb.getConnectedEndpoints() { 726 if ep.isServiceEnabled() { 727 if err := ep.deleteServiceInfoFromCluster(sb, false, "DisableService"); err != nil { 728 failedEps = append(failedEps, ep.Name()) 729 logrus.Warnf("failed update state for endpoint %s into cluster: %v", ep.Name(), err) 730 } 731 ep.disableService() 732 } 733 } 734 logrus.Debugf("DisableService %s DONE", sb.containerID) 735 return nil 736 } 737 738 func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) { 739 for _, i := range osSbox.Info().Interfaces() { 740 // Only remove the interfaces owned by this endpoint from the sandbox. 741 if ep.hasInterface(i.SrcName()) { 742 if err := i.Remove(); err != nil { 743 logrus.Debugf("Remove interface %s failed: %v", i.SrcName(), err) 744 } 745 } 746 } 747 748 ep.Lock() 749 joinInfo := ep.joinInfo 750 vip := ep.virtualIP 751 lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR 752 ep.Unlock() 753 754 if len(vip) > 0 && lbModeIsDSR { 755 ipNet := &net.IPNet{IP: vip, Mask: net.CIDRMask(32, 32)} 756 if err := osSbox.RemoveAliasIP(osSbox.GetLoopbackIfaceName(), ipNet); err != nil { 757 logrus.WithError(err).Debugf("failed to remove virtual ip %v to loopback", ipNet) 758 } 759 } 760 761 if joinInfo == nil { 762 return 763 } 764 765 // Remove non-interface routes. 766 for _, r := range joinInfo.StaticRoutes { 767 if err := osSbox.RemoveStaticRoute(r); err != nil { 768 logrus.Debugf("Remove route failed: %v", err) 769 } 770 } 771 } 772 773 func (sb *sandbox) releaseOSSbox() { 774 sb.Lock() 775 osSbox := sb.osSbox 776 sb.osSbox = nil 777 sb.Unlock() 778 779 if osSbox == nil { 780 return 781 } 782 783 for _, ep := range sb.getConnectedEndpoints() { 784 releaseOSSboxResources(osSbox, ep) 785 } 786 787 if err := osSbox.Destroy(); err != nil { 788 logrus.WithError(err).Error("Error destroying os sandbox") 789 } 790 } 791 792 func (sb *sandbox) restoreOslSandbox() error { 793 var routes []*types.StaticRoute 794 795 // restore osl sandbox 796 Ifaces := make(map[string][]osl.IfaceOption) 797 for _, ep := range sb.endpoints { 798 var ifaceOptions []osl.IfaceOption 799 ep.Lock() 800 joinInfo := ep.joinInfo 801 i := ep.iface 802 ep.Unlock() 803 804 if i == nil { 805 logrus.Errorf("error restoring endpoint %s for container %s", ep.Name(), sb.ContainerID()) 806 continue 807 } 808 809 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes)) 810 if i.addrv6 != nil && i.addrv6.IP.To16() != nil { 811 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) 812 } 813 if i.mac != nil { 814 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) 815 } 816 if len(i.llAddrs) != 0 { 817 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) 818 } 819 Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions 820 if joinInfo != nil { 821 routes = append(routes, joinInfo.StaticRoutes...) 822 } 823 if ep.needResolver() { 824 sb.startResolver(true) 825 } 826 } 827 828 gwep := sb.getGatewayEndpoint() 829 if gwep == nil { 830 return nil 831 } 832 833 // restore osl sandbox 834 err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6) 835 return err 836 } 837 838 func (sb *sandbox) populateNetworkResources(ep *endpoint) error { 839 sb.Lock() 840 if sb.osSbox == nil { 841 sb.Unlock() 842 return nil 843 } 844 inDelete := sb.inDelete 845 sb.Unlock() 846 847 ep.Lock() 848 joinInfo := ep.joinInfo 849 i := ep.iface 850 lbModeIsDSR := ep.network.loadBalancerMode == loadBalancerModeDSR 851 ep.Unlock() 852 853 if ep.needResolver() { 854 sb.startResolver(false) 855 } 856 857 if i != nil && i.srcName != "" { 858 var ifaceOptions []osl.IfaceOption 859 860 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes)) 861 if i.addrv6 != nil && i.addrv6.IP.To16() != nil { 862 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) 863 } 864 if len(i.llAddrs) != 0 { 865 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) 866 } 867 if i.mac != nil { 868 ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) 869 } 870 871 if err := sb.osSbox.AddInterface(i.srcName, i.dstPrefix, ifaceOptions...); err != nil { 872 return fmt.Errorf("failed to add interface %s to sandbox: %v", i.srcName, err) 873 } 874 875 if len(ep.virtualIP) > 0 && lbModeIsDSR { 876 if sb.loadBalancerNID == "" { 877 if err := sb.osSbox.DisableARPForVIP(i.srcName); err != nil { 878 return fmt.Errorf("failed disable ARP for VIP: %v", err) 879 } 880 } 881 ipNet := &net.IPNet{IP: ep.virtualIP, Mask: net.CIDRMask(32, 32)} 882 if err := sb.osSbox.AddAliasIP(sb.osSbox.GetLoopbackIfaceName(), ipNet); err != nil { 883 return fmt.Errorf("failed to add virtual ip %v to loopback: %v", ipNet, err) 884 } 885 } 886 } 887 888 if joinInfo != nil { 889 // Set up non-interface routes. 890 for _, r := range joinInfo.StaticRoutes { 891 if err := sb.osSbox.AddStaticRoute(r); err != nil { 892 return fmt.Errorf("failed to add static route %s: %v", r.Destination.String(), err) 893 } 894 } 895 } 896 897 if ep == sb.getGatewayEndpoint() { 898 if err := sb.updateGateway(ep); err != nil { 899 return err 900 } 901 } 902 903 // Make sure to add the endpoint to the populated endpoint set 904 // before populating loadbalancers. 905 sb.Lock() 906 sb.populatedEndpoints[ep.ID()] = struct{}{} 907 sb.Unlock() 908 909 // Populate load balancer only after updating all the other 910 // information including gateway and other routes so that 911 // loadbalancers are populated all the network state is in 912 // place in the sandbox. 913 sb.populateLoadBalancers(ep) 914 915 // Only update the store if we did not come here as part of 916 // sandbox delete. If we came here as part of delete then do 917 // not bother updating the store. The sandbox object will be 918 // deleted anyway 919 if !inDelete { 920 return sb.storeUpdate() 921 } 922 923 return nil 924 } 925 926 func (sb *sandbox) clearNetworkResources(origEp *endpoint) error { 927 ep := sb.getEndpoint(origEp.id) 928 if ep == nil { 929 return fmt.Errorf("could not find the sandbox endpoint data for endpoint %s", 930 origEp.id) 931 } 932 933 sb.Lock() 934 osSbox := sb.osSbox 935 inDelete := sb.inDelete 936 sb.Unlock() 937 if osSbox != nil { 938 releaseOSSboxResources(osSbox, ep) 939 } 940 941 sb.Lock() 942 delete(sb.populatedEndpoints, ep.ID()) 943 944 if len(sb.endpoints) == 0 { 945 // sb.endpoints should never be empty and this is unexpected error condition 946 // We log an error message to note this down for debugging purposes. 947 logrus.Errorf("No endpoints in sandbox while trying to remove endpoint %s", ep.Name()) 948 sb.Unlock() 949 return nil 950 } 951 952 var ( 953 gwepBefore, gwepAfter *endpoint 954 index = -1 955 ) 956 for i, e := range sb.endpoints { 957 if e == ep { 958 index = i 959 } 960 if len(e.Gateway()) > 0 && gwepBefore == nil { 961 gwepBefore = e 962 } 963 if index != -1 && gwepBefore != nil { 964 break 965 } 966 } 967 968 if index == -1 { 969 logrus.Warnf("Endpoint %s has already been deleted", ep.Name()) 970 sb.Unlock() 971 return nil 972 } 973 974 sb.removeEndpointRaw(ep) 975 for _, e := range sb.endpoints { 976 if len(e.Gateway()) > 0 { 977 gwepAfter = e 978 break 979 } 980 } 981 delete(sb.epPriority, ep.ID()) 982 sb.Unlock() 983 984 if gwepAfter != nil && gwepBefore != gwepAfter { 985 if err := sb.updateGateway(gwepAfter); err != nil { 986 return err 987 } 988 } 989 990 // Only update the store if we did not come here as part of 991 // sandbox delete. If we came here as part of delete then do 992 // not bother updating the store. The sandbox object will be 993 // deleted anyway 994 if !inDelete { 995 return sb.storeUpdate() 996 } 997 998 return nil 999 } 1000 1001 // joinLeaveStart waits to ensure there are no joins or leaves in progress and 1002 // marks this join/leave in progress without race 1003 func (sb *sandbox) joinLeaveStart() { 1004 sb.Lock() 1005 defer sb.Unlock() 1006 1007 for sb.joinLeaveDone != nil { 1008 joinLeaveDone := sb.joinLeaveDone 1009 sb.Unlock() 1010 1011 <-joinLeaveDone 1012 1013 sb.Lock() 1014 } 1015 1016 sb.joinLeaveDone = make(chan struct{}) 1017 } 1018 1019 // joinLeaveEnd marks the end of this join/leave operation and 1020 // signals the same without race to other join and leave waiters 1021 func (sb *sandbox) joinLeaveEnd() { 1022 sb.Lock() 1023 defer sb.Unlock() 1024 1025 if sb.joinLeaveDone != nil { 1026 close(sb.joinLeaveDone) 1027 sb.joinLeaveDone = nil 1028 } 1029 } 1030 1031 // OptionHostname function returns an option setter for hostname option to 1032 // be passed to NewSandbox method. 1033 func OptionHostname(name string) SandboxOption { 1034 return func(sb *sandbox) { 1035 sb.config.hostName = name 1036 } 1037 } 1038 1039 // OptionDomainname function returns an option setter for domainname option to 1040 // be passed to NewSandbox method. 1041 func OptionDomainname(name string) SandboxOption { 1042 return func(sb *sandbox) { 1043 sb.config.domainName = name 1044 } 1045 } 1046 1047 // OptionHostsPath function returns an option setter for hostspath option to 1048 // be passed to NewSandbox method. 1049 func OptionHostsPath(path string) SandboxOption { 1050 return func(sb *sandbox) { 1051 sb.config.hostsPath = path 1052 } 1053 } 1054 1055 // OptionOriginHostsPath function returns an option setter for origin hosts file path 1056 // to be passed to NewSandbox method. 1057 func OptionOriginHostsPath(path string) SandboxOption { 1058 return func(sb *sandbox) { 1059 sb.config.originHostsPath = path 1060 } 1061 } 1062 1063 // OptionExtraHost function returns an option setter for extra /etc/hosts options 1064 // which is a name and IP as strings. 1065 func OptionExtraHost(name string, IP string) SandboxOption { 1066 return func(sb *sandbox) { 1067 sb.config.extraHosts = append(sb.config.extraHosts, extraHost{name: name, IP: IP}) 1068 } 1069 } 1070 1071 // OptionParentUpdate function returns an option setter for parent container 1072 // which needs to update the IP address for the linked container. 1073 func OptionParentUpdate(cid string, name, ip string) SandboxOption { 1074 return func(sb *sandbox) { 1075 sb.config.parentUpdates = append(sb.config.parentUpdates, parentUpdate{cid: cid, name: name, ip: ip}) 1076 } 1077 } 1078 1079 // OptionResolvConfPath function returns an option setter for resolvconfpath option to 1080 // be passed to net container methods. 1081 func OptionResolvConfPath(path string) SandboxOption { 1082 return func(sb *sandbox) { 1083 sb.config.resolvConfPath = path 1084 } 1085 } 1086 1087 // OptionOriginResolvConfPath function returns an option setter to set the path to the 1088 // origin resolv.conf file to be passed to net container methods. 1089 func OptionOriginResolvConfPath(path string) SandboxOption { 1090 return func(sb *sandbox) { 1091 sb.config.originResolvConfPath = path 1092 } 1093 } 1094 1095 // OptionDNS function returns an option setter for dns entry option to 1096 // be passed to container Create method. 1097 func OptionDNS(dns string) SandboxOption { 1098 return func(sb *sandbox) { 1099 sb.config.dnsList = append(sb.config.dnsList, dns) 1100 } 1101 } 1102 1103 // OptionDNSSearch function returns an option setter for dns search entry option to 1104 // be passed to container Create method. 1105 func OptionDNSSearch(search string) SandboxOption { 1106 return func(sb *sandbox) { 1107 sb.config.dnsSearchList = append(sb.config.dnsSearchList, search) 1108 } 1109 } 1110 1111 // OptionDNSOptions function returns an option setter for dns options entry option to 1112 // be passed to container Create method. 1113 func OptionDNSOptions(options string) SandboxOption { 1114 return func(sb *sandbox) { 1115 sb.config.dnsOptionsList = append(sb.config.dnsOptionsList, options) 1116 } 1117 } 1118 1119 // OptionUseDefaultSandbox function returns an option setter for using default sandbox 1120 // (host namespace) to be passed to container Create method. 1121 func OptionUseDefaultSandbox() SandboxOption { 1122 return func(sb *sandbox) { 1123 sb.config.useDefaultSandBox = true 1124 } 1125 } 1126 1127 // OptionUseExternalKey function returns an option setter for using provided namespace 1128 // instead of creating one. 1129 func OptionUseExternalKey() SandboxOption { 1130 return func(sb *sandbox) { 1131 sb.config.useExternalKey = true 1132 } 1133 } 1134 1135 // OptionGeneric function returns an option setter for Generic configuration 1136 // that is not managed by libNetwork but can be used by the Drivers during the call to 1137 // net container creation method. Container Labels are a good example. 1138 func OptionGeneric(generic map[string]interface{}) SandboxOption { 1139 return func(sb *sandbox) { 1140 if sb.config.generic == nil { 1141 sb.config.generic = make(map[string]interface{}, len(generic)) 1142 } 1143 for k, v := range generic { 1144 sb.config.generic[k] = v 1145 } 1146 } 1147 } 1148 1149 // OptionExposedPorts function returns an option setter for the container exposed 1150 // ports option to be passed to container Create method. 1151 func OptionExposedPorts(exposedPorts []types.TransportPort) SandboxOption { 1152 return func(sb *sandbox) { 1153 if sb.config.generic == nil { 1154 sb.config.generic = make(map[string]interface{}) 1155 } 1156 // Defensive copy 1157 eps := make([]types.TransportPort, len(exposedPorts)) 1158 copy(eps, exposedPorts) 1159 // Store endpoint label and in generic because driver needs it 1160 sb.config.exposedPorts = eps 1161 sb.config.generic[netlabel.ExposedPorts] = eps 1162 } 1163 } 1164 1165 // OptionPortMapping function returns an option setter for the mapping 1166 // ports option to be passed to container Create method. 1167 func OptionPortMapping(portBindings []types.PortBinding) SandboxOption { 1168 return func(sb *sandbox) { 1169 if sb.config.generic == nil { 1170 sb.config.generic = make(map[string]interface{}) 1171 } 1172 // Store a copy of the bindings as generic data to pass to the driver 1173 pbs := make([]types.PortBinding, len(portBindings)) 1174 copy(pbs, portBindings) 1175 sb.config.generic[netlabel.PortMap] = pbs 1176 } 1177 } 1178 1179 // OptionIngress function returns an option setter for marking a 1180 // sandbox as the controller's ingress sandbox. 1181 func OptionIngress() SandboxOption { 1182 return func(sb *sandbox) { 1183 sb.ingress = true 1184 sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeIngress) 1185 } 1186 } 1187 1188 // OptionLoadBalancer function returns an option setter for marking a 1189 // sandbox as a load balancer sandbox. 1190 func OptionLoadBalancer(nid string) SandboxOption { 1191 return func(sb *sandbox) { 1192 sb.loadBalancerNID = nid 1193 sb.oslTypes = append(sb.oslTypes, osl.SandboxTypeLoadBalancer) 1194 } 1195 } 1196 1197 // <=> Returns true if a < b, false if a > b and advances to next level if a == b 1198 // epi.prio <=> epj.prio # 2 < 1 1199 // epi.gw <=> epj.gw # non-gw < gw 1200 // epi.internal <=> epj.internal # non-internal < internal 1201 // epi.joininfo <=> epj.joininfo # ipv6 < ipv4 1202 // epi.name <=> epj.name # bar < foo 1203 func (epi *endpoint) Less(epj *endpoint) bool { 1204 var ( 1205 prioi, prioj int 1206 ) 1207 1208 sbi, _ := epi.getSandbox() 1209 sbj, _ := epj.getSandbox() 1210 1211 // Prio defaults to 0 1212 if sbi != nil { 1213 prioi = sbi.epPriority[epi.ID()] 1214 } 1215 if sbj != nil { 1216 prioj = sbj.epPriority[epj.ID()] 1217 } 1218 1219 if prioi != prioj { 1220 return prioi > prioj 1221 } 1222 1223 gwi := epi.endpointInGWNetwork() 1224 gwj := epj.endpointInGWNetwork() 1225 if gwi != gwj { 1226 return gwj 1227 } 1228 1229 inti := epi.getNetwork().Internal() 1230 intj := epj.getNetwork().Internal() 1231 if inti != intj { 1232 return intj 1233 } 1234 1235 jii := 0 1236 if epi.joinInfo != nil { 1237 if epi.joinInfo.gw != nil { 1238 jii = jii + 1 1239 } 1240 if epi.joinInfo.gw6 != nil { 1241 jii = jii + 2 1242 } 1243 } 1244 1245 jij := 0 1246 if epj.joinInfo != nil { 1247 if epj.joinInfo.gw != nil { 1248 jij = jij + 1 1249 } 1250 if epj.joinInfo.gw6 != nil { 1251 jij = jij + 2 1252 } 1253 } 1254 1255 if jii != jij { 1256 return jii > jij 1257 } 1258 1259 return epi.network.Name() < epj.network.Name() 1260 } 1261 1262 func (sb *sandbox) NdotsSet() bool { 1263 return sb.ndotsSet 1264 }