github.com/rawahars/moby@v24.0.4+incompatible/libnetwork/store.go (about) 1 package libnetwork 2 3 import ( 4 "fmt" 5 "strings" 6 7 "github.com/docker/docker/libnetwork/datastore" 8 "github.com/docker/libkv/store/boltdb" 9 "github.com/sirupsen/logrus" 10 ) 11 12 func registerKVStores() { 13 boltdb.Register() 14 } 15 16 func (c *Controller) initStores() error { 17 registerKVStores() 18 19 c.mu.Lock() 20 defer c.mu.Unlock() 21 22 if c.cfg == nil { 23 return nil 24 } 25 var err error 26 c.store, err = datastore.NewDataStore(c.cfg.Scope) 27 if err != nil { 28 return err 29 } 30 31 c.startWatch() 32 return nil 33 } 34 35 func (c *Controller) closeStores() { 36 if store := c.store; store != nil { 37 store.Close() 38 } 39 } 40 41 func (c *Controller) getStore() datastore.DataStore { 42 c.mu.Lock() 43 defer c.mu.Unlock() 44 45 return c.store 46 } 47 48 func (c *Controller) getNetworkFromStore(nid string) (*network, error) { 49 for _, n := range c.getNetworksFromStore() { 50 if n.id == nid { 51 return n, nil 52 } 53 } 54 return nil, ErrNoSuchNetwork(nid) 55 } 56 57 func (c *Controller) getNetworks() ([]*network, error) { 58 var nl []*network 59 60 store := c.getStore() 61 if store == nil { 62 return nil, nil 63 } 64 65 kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix), 66 &network{ctrlr: c}) 67 if err != nil && err != datastore.ErrKeyNotFound { 68 return nil, fmt.Errorf("failed to get networks: %w", err) 69 } 70 71 for _, kvo := range kvol { 72 n := kvo.(*network) 73 n.ctrlr = c 74 75 ec := &endpointCnt{n: n} 76 err = store.GetObject(datastore.Key(ec.Key()...), ec) 77 if err != nil && !n.inDelete { 78 logrus.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err) 79 continue 80 } 81 82 n.epCnt = ec 83 if n.scope == "" { 84 n.scope = store.Scope() 85 } 86 nl = append(nl, n) 87 } 88 89 return nl, nil 90 } 91 92 func (c *Controller) getNetworksFromStore() []*network { // FIXME: unify with c.getNetworks() 93 var nl []*network 94 95 store := c.getStore() 96 kvol, err := store.List(datastore.Key(datastore.NetworkKeyPrefix), &network{ctrlr: c}) 97 if err != nil { 98 if err != datastore.ErrKeyNotFound { 99 logrus.Debugf("failed to get networks from store: %v", err) 100 } 101 return nil 102 } 103 104 kvep, err := store.Map(datastore.Key(epCntKeyPrefix), &endpointCnt{}) 105 if err != nil && err != datastore.ErrKeyNotFound { 106 logrus.Warnf("failed to get endpoint_count map from store: %v", err) 107 } 108 109 for _, kvo := range kvol { 110 n := kvo.(*network) 111 n.mu.Lock() 112 n.ctrlr = c 113 ec := &endpointCnt{n: n} 114 // Trim the leading & trailing "/" to make it consistent across all stores 115 if val, ok := kvep[strings.Trim(datastore.Key(ec.Key()...), "/")]; ok { 116 ec = val.(*endpointCnt) 117 ec.n = n 118 n.epCnt = ec 119 } 120 if n.scope == "" { 121 n.scope = store.Scope() 122 } 123 n.mu.Unlock() 124 nl = append(nl, n) 125 } 126 127 return nl 128 } 129 130 func (n *network) getEndpointFromStore(eid string) (*Endpoint, error) { 131 store := n.ctrlr.getStore() 132 ep := &Endpoint{id: eid, network: n} 133 err := store.GetObject(datastore.Key(ep.Key()...), ep) 134 if err != nil { 135 return nil, fmt.Errorf("could not find endpoint %s: %w", eid, err) 136 } 137 return ep, nil 138 } 139 140 func (n *network) getEndpointsFromStore() ([]*Endpoint, error) { 141 var epl []*Endpoint 142 143 tmp := Endpoint{network: n} 144 store := n.getController().getStore() 145 kvol, err := store.List(datastore.Key(tmp.KeyPrefix()...), &Endpoint{network: n}) 146 if err != nil { 147 if err != datastore.ErrKeyNotFound { 148 return nil, fmt.Errorf("failed to get endpoints for network %s scope %s: %w", 149 n.Name(), store.Scope(), err) 150 } 151 return nil, nil 152 } 153 154 for _, kvo := range kvol { 155 ep := kvo.(*Endpoint) 156 epl = append(epl, ep) 157 } 158 159 return epl, nil 160 } 161 162 func (c *Controller) updateToStore(kvObject datastore.KVObject) error { 163 cs := c.getStore() 164 if cs == nil { 165 return ErrDataStoreNotInitialized 166 } 167 168 if err := cs.PutObjectAtomic(kvObject); err != nil { 169 if err == datastore.ErrKeyModified { 170 return err 171 } 172 return fmt.Errorf("failed to update store for object type %T: %v", kvObject, err) 173 } 174 175 return nil 176 } 177 178 func (c *Controller) deleteFromStore(kvObject datastore.KVObject) error { 179 cs := c.getStore() 180 if cs == nil { 181 return ErrDataStoreNotInitialized 182 } 183 184 retry: 185 if err := cs.DeleteObjectAtomic(kvObject); err != nil { 186 if err == datastore.ErrKeyModified { 187 if err := cs.GetObject(datastore.Key(kvObject.Key()...), kvObject); err != nil { 188 return fmt.Errorf("could not update the kvobject to latest when trying to delete: %v", err) 189 } 190 logrus.Warnf("Error (%v) deleting object %v, retrying....", err, kvObject.Key()) 191 goto retry 192 } 193 return err 194 } 195 196 return nil 197 } 198 199 type netWatch struct { 200 localEps map[string]*Endpoint 201 remoteEps map[string]*Endpoint 202 stopCh chan struct{} 203 } 204 205 func (c *Controller) getLocalEps(nw *netWatch) []*Endpoint { 206 c.mu.Lock() 207 defer c.mu.Unlock() 208 209 var epl []*Endpoint 210 for _, ep := range nw.localEps { 211 epl = append(epl, ep) 212 } 213 214 return epl 215 } 216 217 func (c *Controller) watchSvcRecord(ep *Endpoint) { 218 c.watchCh <- ep 219 } 220 221 func (c *Controller) unWatchSvcRecord(ep *Endpoint) { 222 c.unWatchCh <- ep 223 } 224 225 func (c *Controller) networkWatchLoop(nw *netWatch, ep *Endpoint, ecCh <-chan datastore.KVObject) { 226 for { 227 select { 228 case <-nw.stopCh: 229 return 230 case o := <-ecCh: 231 ec := o.(*endpointCnt) 232 233 epl, err := ec.n.getEndpointsFromStore() 234 if err != nil { 235 logrus.WithError(err).Debug("error getting endpoints from store") 236 continue 237 } 238 239 c.mu.Lock() 240 var addEp []*Endpoint 241 242 delEpMap := make(map[string]*Endpoint) 243 renameEpMap := make(map[string]bool) 244 for k, v := range nw.remoteEps { 245 delEpMap[k] = v 246 } 247 248 for _, lEp := range epl { 249 if _, ok := nw.localEps[lEp.ID()]; ok { 250 continue 251 } 252 253 if ep, ok := nw.remoteEps[lEp.ID()]; ok { 254 // On a container rename EP ID will remain 255 // the same but the name will change. service 256 // records should reflect the change. 257 // Keep old EP entry in the delEpMap and add 258 // EP from the store (which has the new name) 259 // into the new list 260 if lEp.name == ep.name { 261 delete(delEpMap, lEp.ID()) 262 continue 263 } 264 renameEpMap[lEp.ID()] = true 265 } 266 nw.remoteEps[lEp.ID()] = lEp 267 addEp = append(addEp, lEp) 268 } 269 270 // EPs whose name are to be deleted from the svc records 271 // should also be removed from nw's remote EP list, except 272 // the ones that are getting renamed. 273 for _, lEp := range delEpMap { 274 if !renameEpMap[lEp.ID()] { 275 delete(nw.remoteEps, lEp.ID()) 276 } 277 } 278 c.mu.Unlock() 279 280 for _, lEp := range delEpMap { 281 ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), false) 282 } 283 for _, lEp := range addEp { 284 ep.getNetwork().updateSvcRecord(lEp, c.getLocalEps(nw), true) 285 } 286 } 287 } 288 } 289 290 func (c *Controller) processEndpointCreate(nmap map[string]*netWatch, ep *Endpoint) { 291 n := ep.getNetwork() 292 if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() { 293 return 294 } 295 296 networkID := n.ID() 297 endpointID := ep.ID() 298 299 c.mu.Lock() 300 nw, ok := nmap[networkID] 301 c.mu.Unlock() 302 303 if ok { 304 // Update the svc db for the local endpoint join right away 305 n.updateSvcRecord(ep, c.getLocalEps(nw), true) 306 307 c.mu.Lock() 308 nw.localEps[endpointID] = ep 309 310 // If we had learned that from the kv store remove it 311 // from remote ep list now that we know that this is 312 // indeed a local endpoint 313 delete(nw.remoteEps, endpointID) 314 c.mu.Unlock() 315 return 316 } 317 318 nw = &netWatch{ 319 localEps: make(map[string]*Endpoint), 320 remoteEps: make(map[string]*Endpoint), 321 } 322 323 // Update the svc db for the local endpoint join right away 324 // Do this before adding this ep to localEps so that we don't 325 // try to update this ep's container's svc records 326 n.updateSvcRecord(ep, c.getLocalEps(nw), true) 327 328 c.mu.Lock() 329 nw.localEps[endpointID] = ep 330 nmap[networkID] = nw 331 nw.stopCh = make(chan struct{}) 332 c.mu.Unlock() 333 334 store := c.getStore() 335 if store == nil { 336 return 337 } 338 339 if !store.Watchable() { 340 return 341 } 342 343 ch, err := store.Watch(n.getEpCnt(), nw.stopCh) 344 if err != nil { 345 logrus.Warnf("Error creating watch for network: %v", err) 346 return 347 } 348 349 go c.networkWatchLoop(nw, ep, ch) 350 } 351 352 func (c *Controller) processEndpointDelete(nmap map[string]*netWatch, ep *Endpoint) { 353 n := ep.getNetwork() 354 if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() { 355 return 356 } 357 358 networkID := n.ID() 359 endpointID := ep.ID() 360 361 c.mu.Lock() 362 nw, ok := nmap[networkID] 363 364 if ok { 365 delete(nw.localEps, endpointID) 366 c.mu.Unlock() 367 368 // Update the svc db about local endpoint leave right away 369 // Do this after we remove this ep from localEps so that we 370 // don't try to remove this svc record from this ep's container. 371 n.updateSvcRecord(ep, c.getLocalEps(nw), false) 372 373 c.mu.Lock() 374 if len(nw.localEps) == 0 { 375 close(nw.stopCh) 376 377 // This is the last container going away for the network. Destroy 378 // this network's svc db entry 379 delete(c.svcRecords, networkID) 380 381 delete(nmap, networkID) 382 } 383 } 384 c.mu.Unlock() 385 } 386 387 func (c *Controller) watchLoop() { 388 for { 389 select { 390 case ep := <-c.watchCh: 391 c.processEndpointCreate(c.nmap, ep) 392 case ep := <-c.unWatchCh: 393 c.processEndpointDelete(c.nmap, ep) 394 } 395 } 396 } 397 398 func (c *Controller) startWatch() { 399 if c.watchCh != nil { 400 return 401 } 402 c.watchCh = make(chan *Endpoint) 403 c.unWatchCh = make(chan *Endpoint) 404 c.nmap = make(map[string]*netWatch) 405 406 go c.watchLoop() 407 } 408 409 func (c *Controller) networkCleanup() { 410 for _, n := range c.getNetworksFromStore() { 411 if n.inDelete { 412 logrus.Infof("Removing stale network %s (%s)", n.Name(), n.ID()) 413 if err := n.delete(true, true); err != nil { 414 logrus.Debugf("Error while removing stale network: %v", err) 415 } 416 } 417 } 418 } 419 420 var populateSpecial NetworkWalker = func(nw Network) bool { 421 if n := nw.(*network); n.hasSpecialDriver() && !n.ConfigOnly() { 422 if err := n.getController().addNetwork(n); err != nil { 423 logrus.Warnf("Failed to populate network %q with driver %q", nw.Name(), nw.Type()) 424 } 425 } 426 return false 427 }