github.com/rawahars/moby@v24.0.4+incompatible/libnetwork/service_common.go (about) 1 //go:build linux || windows 2 // +build linux windows 3 4 package libnetwork 5 6 import ( 7 "net" 8 9 "github.com/sirupsen/logrus" 10 ) 11 12 const maxSetStringLen = 350 13 14 func (c *Controller) addEndpointNameResolution(svcName, svcID, nID, eID, containerName string, vip net.IP, serviceAliases, taskAliases []string, ip net.IP, addService bool, method string) error { 15 n, err := c.NetworkByID(nID) 16 if err != nil { 17 return err 18 } 19 20 logrus.Debugf("addEndpointNameResolution %s %s add_service:%t sAliases:%v tAliases:%v", eID, svcName, addService, serviceAliases, taskAliases) 21 22 // Add container resolution mappings 23 if err := c.addContainerNameResolution(nID, eID, containerName, taskAliases, ip, method); err != nil { 24 return err 25 } 26 27 serviceID := svcID 28 if serviceID == "" { 29 // This is the case of a normal container not part of a service 30 serviceID = eID 31 } 32 33 // Add endpoint IP to special "tasks.svc_name" so that the applications have access to DNS RR. 34 n.(*network).addSvcRecords(eID, "tasks."+svcName, serviceID, ip, nil, false, method) 35 for _, alias := range serviceAliases { 36 n.(*network).addSvcRecords(eID, "tasks."+alias, serviceID, ip, nil, false, method) 37 } 38 39 // Add service name to vip in DNS, if vip is valid. Otherwise resort to DNS RR 40 if len(vip) == 0 { 41 n.(*network).addSvcRecords(eID, svcName, serviceID, ip, nil, false, method) 42 for _, alias := range serviceAliases { 43 n.(*network).addSvcRecords(eID, alias, serviceID, ip, nil, false, method) 44 } 45 } 46 47 if addService && len(vip) != 0 { 48 n.(*network).addSvcRecords(eID, svcName, serviceID, vip, nil, false, method) 49 for _, alias := range serviceAliases { 50 n.(*network).addSvcRecords(eID, alias, serviceID, vip, nil, false, method) 51 } 52 } 53 54 return nil 55 } 56 57 func (c *Controller) addContainerNameResolution(nID, eID, containerName string, taskAliases []string, ip net.IP, method string) error { 58 n, err := c.NetworkByID(nID) 59 if err != nil { 60 return err 61 } 62 logrus.Debugf("addContainerNameResolution %s %s", eID, containerName) 63 64 // Add resolution for container name 65 n.(*network).addSvcRecords(eID, containerName, eID, ip, nil, true, method) 66 67 // Add resolution for taskaliases 68 for _, alias := range taskAliases { 69 n.(*network).addSvcRecords(eID, alias, eID, ip, nil, false, method) 70 } 71 72 return nil 73 } 74 75 func (c *Controller) deleteEndpointNameResolution(svcName, svcID, nID, eID, containerName string, vip net.IP, serviceAliases, taskAliases []string, ip net.IP, rmService, multipleEntries bool, method string) error { 76 n, err := c.NetworkByID(nID) 77 if err != nil { 78 return err 79 } 80 81 logrus.Debugf("deleteEndpointNameResolution %s %s rm_service:%t suppress:%t sAliases:%v tAliases:%v", eID, svcName, rmService, multipleEntries, serviceAliases, taskAliases) 82 83 // Delete container resolution mappings 84 if err := c.delContainerNameResolution(nID, eID, containerName, taskAliases, ip, method); err != nil { 85 logrus.WithError(err).Warn("Error delting container from resolver") 86 } 87 88 serviceID := svcID 89 if serviceID == "" { 90 // This is the case of a normal container not part of a service 91 serviceID = eID 92 } 93 94 // Delete the special "tasks.svc_name" backend record. 95 if !multipleEntries { 96 n.(*network).deleteSvcRecords(eID, "tasks."+svcName, serviceID, ip, nil, false, method) 97 for _, alias := range serviceAliases { 98 n.(*network).deleteSvcRecords(eID, "tasks."+alias, serviceID, ip, nil, false, method) 99 } 100 } 101 102 // If we are doing DNS RR delete the endpoint IP from DNS record right away. 103 if !multipleEntries && len(vip) == 0 { 104 n.(*network).deleteSvcRecords(eID, svcName, serviceID, ip, nil, false, method) 105 for _, alias := range serviceAliases { 106 n.(*network).deleteSvcRecords(eID, alias, serviceID, ip, nil, false, method) 107 } 108 } 109 110 // Remove the DNS record for VIP only if we are removing the service 111 if rmService && len(vip) != 0 && !multipleEntries { 112 n.(*network).deleteSvcRecords(eID, svcName, serviceID, vip, nil, false, method) 113 for _, alias := range serviceAliases { 114 n.(*network).deleteSvcRecords(eID, alias, serviceID, vip, nil, false, method) 115 } 116 } 117 118 return nil 119 } 120 121 func (c *Controller) delContainerNameResolution(nID, eID, containerName string, taskAliases []string, ip net.IP, method string) error { 122 n, err := c.NetworkByID(nID) 123 if err != nil { 124 return err 125 } 126 logrus.Debugf("delContainerNameResolution %s %s", eID, containerName) 127 128 // Delete resolution for container name 129 n.(*network).deleteSvcRecords(eID, containerName, eID, ip, nil, true, method) 130 131 // Delete resolution for taskaliases 132 for _, alias := range taskAliases { 133 n.(*network).deleteSvcRecords(eID, alias, eID, ip, nil, true, method) 134 } 135 136 return nil 137 } 138 139 func newService(name string, id string, ingressPorts []*PortConfig, serviceAliases []string) *service { 140 return &service{ 141 name: name, 142 id: id, 143 ingressPorts: ingressPorts, 144 loadBalancers: make(map[string]*loadBalancer), 145 aliases: serviceAliases, 146 } 147 } 148 149 func (c *Controller) getLBIndex(sid, nid string, ingressPorts []*PortConfig) int { 150 skey := serviceKey{ 151 id: sid, 152 ports: portConfigs(ingressPorts).String(), 153 } 154 c.mu.Lock() 155 s, ok := c.serviceBindings[skey] 156 c.mu.Unlock() 157 158 if !ok { 159 return 0 160 } 161 162 s.Lock() 163 lb := s.loadBalancers[nid] 164 s.Unlock() 165 166 return int(lb.fwMark) 167 } 168 169 // cleanupServiceDiscovery when the network is being deleted, erase all the associated service discovery records 170 func (c *Controller) cleanupServiceDiscovery(cleanupNID string) { 171 c.mu.Lock() 172 defer c.mu.Unlock() 173 if cleanupNID == "" { 174 logrus.Debugf("cleanupServiceDiscovery for all networks") 175 c.svcRecords = make(map[string]*svcInfo) 176 return 177 } 178 logrus.Debugf("cleanupServiceDiscovery for network:%s", cleanupNID) 179 delete(c.svcRecords, cleanupNID) 180 } 181 182 func (c *Controller) cleanupServiceBindings(cleanupNID string) { 183 var cleanupFuncs []func() 184 185 logrus.Debugf("cleanupServiceBindings for %s", cleanupNID) 186 c.mu.Lock() 187 services := make([]*service, 0, len(c.serviceBindings)) 188 for _, s := range c.serviceBindings { 189 services = append(services, s) 190 } 191 c.mu.Unlock() 192 193 for _, s := range services { 194 s.Lock() 195 // Skip the serviceBindings that got deleted 196 if s.deleted { 197 s.Unlock() 198 continue 199 } 200 for nid, lb := range s.loadBalancers { 201 if cleanupNID != "" && nid != cleanupNID { 202 continue 203 } 204 for eid, be := range lb.backEnds { 205 cleanupFuncs = append(cleanupFuncs, makeServiceCleanupFunc(c, s, nid, eid, lb.vip, be.ip)) 206 } 207 } 208 s.Unlock() 209 } 210 211 for _, f := range cleanupFuncs { 212 f() 213 } 214 } 215 216 func makeServiceCleanupFunc(c *Controller, s *service, nID, eID string, vip net.IP, ip net.IP) func() { 217 // ContainerName and taskAliases are not available here, this is still fine because the Service discovery 218 // cleanup already happened before. The only thing that rmServiceBinding is still doing here a part from the Load 219 // Balancer bookeeping, is to keep consistent the mapping of endpoint to IP. 220 return func() { 221 if err := c.rmServiceBinding(s.name, s.id, nID, eID, "", vip, s.ingressPorts, s.aliases, []string{}, ip, "cleanupServiceBindings", false, true); err != nil { 222 logrus.Errorf("Failed to remove service bindings for service %s network %s endpoint %s while cleanup: %v", s.id, nID, eID, err) 223 } 224 } 225 } 226 227 func (c *Controller) addServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases, taskAliases []string, ip net.IP, method string) error { 228 var addService bool 229 230 // Failure to lock the network ID on add can result in racing 231 // racing against network deletion resulting in inconsistent 232 // state in the c.serviceBindings map and it's sub-maps. Also, 233 // always lock network ID before services to avoid deadlock. 234 c.networkLocker.Lock(nID) 235 defer c.networkLocker.Unlock(nID) //nolint:errcheck 236 237 n, err := c.NetworkByID(nID) 238 if err != nil { 239 return err 240 } 241 242 skey := serviceKey{ 243 id: svcID, 244 ports: portConfigs(ingressPorts).String(), 245 } 246 247 var s *service 248 for { 249 c.mu.Lock() 250 var ok bool 251 s, ok = c.serviceBindings[skey] 252 if !ok { 253 // Create a new service if we are seeing this service 254 // for the first time. 255 s = newService(svcName, svcID, ingressPorts, serviceAliases) 256 c.serviceBindings[skey] = s 257 } 258 c.mu.Unlock() 259 s.Lock() 260 if !s.deleted { 261 // ok the object is good to be used 262 break 263 } 264 s.Unlock() 265 } 266 logrus.Debugf("addServiceBinding from %s START for %s %s p:%p nid:%s skey:%v", method, svcName, eID, s, nID, skey) 267 defer s.Unlock() 268 269 lb, ok := s.loadBalancers[nID] 270 if !ok { 271 // Create a new load balancer if we are seeing this 272 // network attachment on the service for the first 273 // time. 274 fwMarkCtrMu.Lock() 275 276 lb = &loadBalancer{ 277 vip: vip, 278 fwMark: fwMarkCtr, 279 backEnds: make(map[string]*lbBackend), 280 service: s, 281 } 282 283 fwMarkCtr++ 284 fwMarkCtrMu.Unlock() 285 286 s.loadBalancers[nID] = lb 287 addService = true 288 } 289 290 lb.backEnds[eID] = &lbBackend{ip, false} 291 292 ok, entries := s.assignIPToEndpoint(ip.String(), eID) 293 if !ok || entries > 1 { 294 setStr, b := s.printIPToEndpoint(ip.String()) 295 if len(setStr) > maxSetStringLen { 296 setStr = setStr[:maxSetStringLen] 297 } 298 logrus.Warnf("addServiceBinding %s possible transient state ok:%t entries:%d set:%t %s", eID, ok, entries, b, setStr) 299 } 300 301 // Add loadbalancer service and backend to the network 302 n.(*network).addLBBackend(ip, lb) 303 304 // Add the appropriate name resolutions 305 if err := c.addEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, addService, "addServiceBinding"); err != nil { 306 return err 307 } 308 309 logrus.Debugf("addServiceBinding from %s END for %s %s", method, svcName, eID) 310 311 return nil 312 } 313 314 func (c *Controller) rmServiceBinding(svcName, svcID, nID, eID, containerName string, vip net.IP, ingressPorts []*PortConfig, serviceAliases []string, taskAliases []string, ip net.IP, method string, deleteSvcRecords bool, fullRemove bool) error { 315 var rmService bool 316 317 skey := serviceKey{ 318 id: svcID, 319 ports: portConfigs(ingressPorts).String(), 320 } 321 322 c.mu.Lock() 323 s, ok := c.serviceBindings[skey] 324 c.mu.Unlock() 325 if !ok { 326 logrus.Warnf("rmServiceBinding %s %s %s aborted c.serviceBindings[skey] !ok", method, svcName, eID) 327 return nil 328 } 329 330 s.Lock() 331 defer s.Unlock() 332 logrus.Debugf("rmServiceBinding from %s START for %s %s p:%p nid:%s sKey:%v deleteSvc:%t", method, svcName, eID, s, nID, skey, deleteSvcRecords) 333 lb, ok := s.loadBalancers[nID] 334 if !ok { 335 logrus.Warnf("rmServiceBinding %s %s %s aborted s.loadBalancers[nid] !ok", method, svcName, eID) 336 return nil 337 } 338 339 be, ok := lb.backEnds[eID] 340 if !ok { 341 logrus.Warnf("rmServiceBinding %s %s %s aborted lb.backEnds[eid] && lb.disabled[eid] !ok", method, svcName, eID) 342 return nil 343 } 344 345 if fullRemove { 346 // delete regardless 347 delete(lb.backEnds, eID) 348 } else { 349 be.disabled = true 350 } 351 352 if len(lb.backEnds) == 0 { 353 // All the backends for this service have been 354 // removed. Time to remove the load balancer and also 355 // remove the service entry in IPVS. 356 rmService = true 357 358 delete(s.loadBalancers, nID) 359 logrus.Debugf("rmServiceBinding %s delete %s, p:%p in loadbalancers len:%d", eID, nID, lb, len(s.loadBalancers)) 360 } 361 362 ok, entries := s.removeIPToEndpoint(ip.String(), eID) 363 if !ok || entries > 0 { 364 setStr, b := s.printIPToEndpoint(ip.String()) 365 if len(setStr) > maxSetStringLen { 366 setStr = setStr[:maxSetStringLen] 367 } 368 logrus.Warnf("rmServiceBinding %s possible transient state ok:%t entries:%d set:%t %s", eID, ok, entries, b, setStr) 369 } 370 371 // Remove loadbalancer service(if needed) and backend in all 372 // sandboxes in the network only if the vip is valid. 373 if entries == 0 { 374 // The network may well have been deleted from the store (and 375 // dataplane) before the last of the service bindings. On Linux that's 376 // ok because removing the network sandbox from the dataplane 377 // implicitly cleans up all related dataplane state. 378 // On the Windows dataplane, VFP policylists must be removed 379 // independently of the network, and they must be removed before the HNS 380 // network. Otherwise, policylist removal fails with "network not 381 // found." On Windows cleanupServiceBindings must be called prior to 382 // removing the network from the store or dataplane. 383 n, err := c.NetworkByID(nID) 384 if err == nil { 385 n.(*network).rmLBBackend(ip, lb, rmService, fullRemove) 386 } 387 } 388 389 // Delete the name resolutions 390 if deleteSvcRecords { 391 if err := c.deleteEndpointNameResolution(svcName, svcID, nID, eID, containerName, vip, serviceAliases, taskAliases, ip, rmService, entries > 0, "rmServiceBinding"); err != nil { 392 return err 393 } 394 } 395 396 if len(s.loadBalancers) == 0 { 397 // All loadbalancers for the service removed. Time to 398 // remove the service itself. 399 c.mu.Lock() 400 401 // Mark the object as deleted so that the add won't use it wrongly 402 s.deleted = true 403 // NOTE The delete from the serviceBindings map has to be the last operation else we are allowing a race between this service 404 // that is getting deleted and a new service that will be created if the entry is not anymore there 405 delete(c.serviceBindings, skey) 406 c.mu.Unlock() 407 } 408 409 logrus.Debugf("rmServiceBinding from %s END for %s %s", method, svcName, eID) 410 return nil 411 }