github.com/adityamillind98/moby@v23.0.0-rc.4+incompatible/libnetwork/drivers/overlay/overlay.go (about) 1 //go:build linux 2 // +build linux 3 4 package overlay 5 6 //go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf --gogo_out=import_path=github.com/docker/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto 7 8 import ( 9 "context" 10 "fmt" 11 "net" 12 "sync" 13 14 "github.com/docker/docker/libnetwork/datastore" 15 "github.com/docker/docker/libnetwork/discoverapi" 16 "github.com/docker/docker/libnetwork/driverapi" 17 "github.com/docker/docker/libnetwork/idm" 18 "github.com/docker/docker/libnetwork/netlabel" 19 "github.com/docker/docker/libnetwork/osl" 20 "github.com/docker/docker/libnetwork/types" 21 "github.com/hashicorp/serf/serf" 22 "github.com/sirupsen/logrus" 23 ) 24 25 const ( 26 networkType = "overlay" 27 vethPrefix = "veth" 28 vethLen = 7 29 vxlanIDStart = 256 30 vxlanIDEnd = (1 << 24) - 1 31 vxlanEncap = 50 32 secureOption = "encrypted" 33 ) 34 35 var initVxlanIdm = make(chan (bool), 1) 36 37 type driver struct { 38 eventCh chan serf.Event 39 notifyCh chan ovNotify 40 exitCh chan chan struct{} 41 bindAddress string 42 advertiseAddress string 43 neighIP string 44 config map[string]interface{} 45 peerDb peerNetworkMap 46 secMap *encrMap 47 serfInstance *serf.Serf 48 networks networkTable 49 store datastore.DataStore 50 localStore datastore.DataStore 51 vxlanIdm *idm.Idm 52 initOS sync.Once 53 joinOnce sync.Once 54 localJoinOnce sync.Once 55 keys []*key 56 peerOpCh chan *peerOperation 57 peerOpCancel context.CancelFunc 58 sync.Mutex 59 } 60 61 // Init registers a new instance of overlay driver 62 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { 63 c := driverapi.Capability{ 64 DataScope: datastore.GlobalScope, 65 ConnectivityScope: datastore.GlobalScope, 66 } 67 d := &driver{ 68 networks: networkTable{}, 69 peerDb: peerNetworkMap{ 70 mp: map[string]*peerMap{}, 71 }, 72 secMap: &encrMap{nodes: map[string][]*spi{}}, 73 config: config, 74 peerOpCh: make(chan *peerOperation), 75 } 76 77 // Launch the go routine for processing peer operations 78 ctx, cancel := context.WithCancel(context.Background()) 79 d.peerOpCancel = cancel 80 go d.peerOpRoutine(ctx, d.peerOpCh) 81 82 if data, ok := config[netlabel.GlobalKVClient]; ok { 83 var err error 84 dsc, ok := data.(discoverapi.DatastoreConfigData) 85 if !ok { 86 return types.InternalErrorf("incorrect data in datastore configuration: %v", data) 87 } 88 d.store, err = datastore.NewDataStoreFromConfig(dsc) 89 if err != nil { 90 return types.InternalErrorf("failed to initialize data store: %v", err) 91 } 92 } 93 94 if data, ok := config[netlabel.LocalKVClient]; ok { 95 var err error 96 dsc, ok := data.(discoverapi.DatastoreConfigData) 97 if !ok { 98 return types.InternalErrorf("incorrect data in datastore configuration: %v", data) 99 } 100 d.localStore, err = datastore.NewDataStoreFromConfig(dsc) 101 if err != nil { 102 return types.InternalErrorf("failed to initialize local data store: %v", err) 103 } 104 } 105 106 if err := d.restoreEndpoints(); err != nil { 107 logrus.Warnf("Failure during overlay endpoints restore: %v", err) 108 } 109 110 return dc.RegisterDriver(networkType, d, c) 111 } 112 113 // Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox 114 func (d *driver) restoreEndpoints() error { 115 if d.localStore == nil { 116 logrus.Warn("Cannot restore overlay endpoints because local datastore is missing") 117 return nil 118 } 119 kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{}) 120 if err != nil && err != datastore.ErrKeyNotFound { 121 return fmt.Errorf("failed to read overlay endpoint from store: %v", err) 122 } 123 124 if err == datastore.ErrKeyNotFound { 125 return nil 126 } 127 for _, kvo := range kvol { 128 ep := kvo.(*endpoint) 129 n := d.network(ep.nid) 130 if n == nil { 131 logrus.Debugf("Network (%.7s) not found for restored endpoint (%.7s)", ep.nid, ep.id) 132 logrus.Debugf("Deleting stale overlay endpoint (%.7s) from store", ep.id) 133 if err := d.deleteEndpointFromStore(ep); err != nil { 134 logrus.Debugf("Failed to delete stale overlay endpoint (%.7s) from store", ep.id) 135 } 136 continue 137 } 138 n.addEndpoint(ep) 139 140 s := n.getSubnetforIP(ep.addr) 141 if s == nil { 142 return fmt.Errorf("could not find subnet for endpoint %s", ep.id) 143 } 144 145 if err := n.joinSandbox(s, true, true); err != nil { 146 return fmt.Errorf("restore network sandbox failed: %v", err) 147 } 148 149 Ifaces := make(map[string][]osl.IfaceOption) 150 vethIfaceOption := make([]osl.IfaceOption, 1) 151 vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName)) 152 Ifaces["veth+veth"] = vethIfaceOption 153 154 err := n.sbox.Restore(Ifaces, nil, nil, nil) 155 if err != nil { 156 n.leaveSandbox() 157 return fmt.Errorf("failed to restore overlay sandbox: %v", err) 158 } 159 160 d.peerAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), false, false, true) 161 } 162 return nil 163 } 164 165 // Fini cleans up the driver resources 166 func Fini(drv driverapi.Driver) { 167 d := drv.(*driver) 168 169 // Notify the peer go routine to return 170 if d.peerOpCancel != nil { 171 d.peerOpCancel() 172 } 173 174 if d.exitCh != nil { 175 waitCh := make(chan struct{}) 176 177 d.exitCh <- waitCh 178 179 <-waitCh 180 } 181 } 182 183 func (d *driver) configure() error { 184 // Apply OS specific kernel configs if needed 185 d.initOS.Do(applyOStweaks) 186 187 if d.store == nil { 188 return nil 189 } 190 191 if d.vxlanIdm == nil { 192 return d.initializeVxlanIdm() 193 } 194 195 return nil 196 } 197 198 func (d *driver) initializeVxlanIdm() error { 199 var err error 200 201 initVxlanIdm <- true 202 defer func() { <-initVxlanIdm }() 203 204 if d.vxlanIdm != nil { 205 return nil 206 } 207 208 d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd) 209 if err != nil { 210 return fmt.Errorf("failed to initialize vxlan id manager: %v", err) 211 } 212 213 return nil 214 } 215 216 func (d *driver) Type() string { 217 return networkType 218 } 219 220 func (d *driver) IsBuiltIn() bool { 221 return true 222 } 223 224 func validateSelf(node string) error { 225 advIP := net.ParseIP(node) 226 if advIP == nil { 227 return fmt.Errorf("invalid self address (%s)", node) 228 } 229 230 addrs, err := net.InterfaceAddrs() 231 if err != nil { 232 return fmt.Errorf("Unable to get interface addresses %v", err) 233 } 234 for _, addr := range addrs { 235 ip, _, err := net.ParseCIDR(addr.String()) 236 if err == nil && ip.Equal(advIP) { 237 return nil 238 } 239 } 240 return fmt.Errorf("Multi-Host overlay networking requires cluster-advertise(%s) to be configured with a local ip-address that is reachable within the cluster", advIP.String()) 241 } 242 243 func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) { 244 if self && !d.isSerfAlive() { 245 d.Lock() 246 d.advertiseAddress = advertiseAddress 247 d.bindAddress = bindAddress 248 d.Unlock() 249 250 // If containers are already running on this network update the 251 // advertise address in the peerDB 252 d.localJoinOnce.Do(func() { 253 d.peerDBUpdateSelf() 254 }) 255 256 // If there is no cluster store there is no need to start serf. 257 if d.store != nil { 258 if err := validateSelf(advertiseAddress); err != nil { 259 logrus.Warn(err.Error()) 260 } 261 err := d.serfInit() 262 if err != nil { 263 logrus.Errorf("initializing serf instance failed: %v", err) 264 d.Lock() 265 d.advertiseAddress = "" 266 d.bindAddress = "" 267 d.Unlock() 268 return 269 } 270 } 271 } 272 273 d.Lock() 274 if !self { 275 d.neighIP = advertiseAddress 276 } 277 neighIP := d.neighIP 278 d.Unlock() 279 280 if d.serfInstance != nil && neighIP != "" { 281 var err error 282 d.joinOnce.Do(func() { 283 err = d.serfJoin(neighIP) 284 if err == nil { 285 d.pushLocalDb() 286 } 287 }) 288 if err != nil { 289 logrus.Errorf("joining serf neighbor %s failed: %v", advertiseAddress, err) 290 d.Lock() 291 d.joinOnce = sync.Once{} 292 d.Unlock() 293 return 294 } 295 } 296 } 297 298 func (d *driver) pushLocalEndpointEvent(action, nid, eid string) { 299 n := d.network(nid) 300 if n == nil { 301 logrus.Debugf("Error pushing local endpoint event for network %s", nid) 302 return 303 } 304 ep := n.endpoint(eid) 305 if ep == nil { 306 logrus.Debugf("Error pushing local endpoint event for ep %s / %s", nid, eid) 307 return 308 } 309 310 if !d.isSerfAlive() { 311 return 312 } 313 d.notifyCh <- ovNotify{ 314 action: "join", 315 nw: n, 316 ep: ep, 317 } 318 } 319 320 // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster 321 func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error { 322 var err error 323 switch dType { 324 case discoverapi.NodeDiscovery: 325 nodeData, ok := data.(discoverapi.NodeDiscoveryData) 326 if !ok || nodeData.Address == "" { 327 return fmt.Errorf("invalid discovery data") 328 } 329 d.nodeJoin(nodeData.Address, nodeData.BindAddress, nodeData.Self) 330 case discoverapi.DatastoreConfig: 331 if d.store != nil { 332 return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already") 333 } 334 dsc, ok := data.(discoverapi.DatastoreConfigData) 335 if !ok { 336 return types.InternalErrorf("incorrect data in datastore configuration: %v", data) 337 } 338 d.store, err = datastore.NewDataStoreFromConfig(dsc) 339 if err != nil { 340 return types.InternalErrorf("failed to initialize data store: %v", err) 341 } 342 case discoverapi.EncryptionKeysConfig: 343 encrData, ok := data.(discoverapi.DriverEncryptionConfig) 344 if !ok { 345 return fmt.Errorf("invalid encryption key notification data") 346 } 347 keys := make([]*key, 0, len(encrData.Keys)) 348 for i := 0; i < len(encrData.Keys); i++ { 349 k := &key{ 350 value: encrData.Keys[i], 351 tag: uint32(encrData.Tags[i]), 352 } 353 keys = append(keys, k) 354 } 355 if err := d.setKeys(keys); err != nil { 356 logrus.Warn(err) 357 } 358 case discoverapi.EncryptionKeysUpdate: 359 var newKey, delKey, priKey *key 360 encrData, ok := data.(discoverapi.DriverEncryptionUpdate) 361 if !ok { 362 return fmt.Errorf("invalid encryption key notification data") 363 } 364 if encrData.Key != nil { 365 newKey = &key{ 366 value: encrData.Key, 367 tag: uint32(encrData.Tag), 368 } 369 } 370 if encrData.Primary != nil { 371 priKey = &key{ 372 value: encrData.Primary, 373 tag: uint32(encrData.PrimaryTag), 374 } 375 } 376 if encrData.Prune != nil { 377 delKey = &key{ 378 value: encrData.Prune, 379 tag: uint32(encrData.PruneTag), 380 } 381 } 382 if err := d.updateKeys(newKey, priKey, delKey); err != nil { 383 return err 384 } 385 default: 386 } 387 return nil 388 } 389 390 // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster 391 func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error { 392 return nil 393 }