github.com/adityamillind98/moby@v23.0.0-rc.4+incompatible/libnetwork/drivers/overlay/overlay.go (about)

     1  //go:build linux
     2  // +build linux
     3  
     4  package overlay
     5  
     6  //go:generate protoc -I.:../../Godeps/_workspace/src/github.com/gogo/protobuf  --gogo_out=import_path=github.com/docker/docker/libnetwork/drivers/overlay,Mgogoproto/gogo.proto=github.com/gogo/protobuf/gogoproto:. overlay.proto
     7  
     8  import (
     9  	"context"
    10  	"fmt"
    11  	"net"
    12  	"sync"
    13  
    14  	"github.com/docker/docker/libnetwork/datastore"
    15  	"github.com/docker/docker/libnetwork/discoverapi"
    16  	"github.com/docker/docker/libnetwork/driverapi"
    17  	"github.com/docker/docker/libnetwork/idm"
    18  	"github.com/docker/docker/libnetwork/netlabel"
    19  	"github.com/docker/docker/libnetwork/osl"
    20  	"github.com/docker/docker/libnetwork/types"
    21  	"github.com/hashicorp/serf/serf"
    22  	"github.com/sirupsen/logrus"
    23  )
    24  
    25  const (
    26  	networkType  = "overlay"
    27  	vethPrefix   = "veth"
    28  	vethLen      = 7
    29  	vxlanIDStart = 256
    30  	vxlanIDEnd   = (1 << 24) - 1
    31  	vxlanEncap   = 50
    32  	secureOption = "encrypted"
    33  )
    34  
    35  var initVxlanIdm = make(chan (bool), 1)
    36  
    37  type driver struct {
    38  	eventCh          chan serf.Event
    39  	notifyCh         chan ovNotify
    40  	exitCh           chan chan struct{}
    41  	bindAddress      string
    42  	advertiseAddress string
    43  	neighIP          string
    44  	config           map[string]interface{}
    45  	peerDb           peerNetworkMap
    46  	secMap           *encrMap
    47  	serfInstance     *serf.Serf
    48  	networks         networkTable
    49  	store            datastore.DataStore
    50  	localStore       datastore.DataStore
    51  	vxlanIdm         *idm.Idm
    52  	initOS           sync.Once
    53  	joinOnce         sync.Once
    54  	localJoinOnce    sync.Once
    55  	keys             []*key
    56  	peerOpCh         chan *peerOperation
    57  	peerOpCancel     context.CancelFunc
    58  	sync.Mutex
    59  }
    60  
    61  // Init registers a new instance of overlay driver
    62  func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
    63  	c := driverapi.Capability{
    64  		DataScope:         datastore.GlobalScope,
    65  		ConnectivityScope: datastore.GlobalScope,
    66  	}
    67  	d := &driver{
    68  		networks: networkTable{},
    69  		peerDb: peerNetworkMap{
    70  			mp: map[string]*peerMap{},
    71  		},
    72  		secMap:   &encrMap{nodes: map[string][]*spi{}},
    73  		config:   config,
    74  		peerOpCh: make(chan *peerOperation),
    75  	}
    76  
    77  	// Launch the go routine for processing peer operations
    78  	ctx, cancel := context.WithCancel(context.Background())
    79  	d.peerOpCancel = cancel
    80  	go d.peerOpRoutine(ctx, d.peerOpCh)
    81  
    82  	if data, ok := config[netlabel.GlobalKVClient]; ok {
    83  		var err error
    84  		dsc, ok := data.(discoverapi.DatastoreConfigData)
    85  		if !ok {
    86  			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
    87  		}
    88  		d.store, err = datastore.NewDataStoreFromConfig(dsc)
    89  		if err != nil {
    90  			return types.InternalErrorf("failed to initialize data store: %v", err)
    91  		}
    92  	}
    93  
    94  	if data, ok := config[netlabel.LocalKVClient]; ok {
    95  		var err error
    96  		dsc, ok := data.(discoverapi.DatastoreConfigData)
    97  		if !ok {
    98  			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
    99  		}
   100  		d.localStore, err = datastore.NewDataStoreFromConfig(dsc)
   101  		if err != nil {
   102  			return types.InternalErrorf("failed to initialize local data store: %v", err)
   103  		}
   104  	}
   105  
   106  	if err := d.restoreEndpoints(); err != nil {
   107  		logrus.Warnf("Failure during overlay endpoints restore: %v", err)
   108  	}
   109  
   110  	return dc.RegisterDriver(networkType, d, c)
   111  }
   112  
   113  // Endpoints are stored in the local store. Restore them and reconstruct the overlay sandbox
   114  func (d *driver) restoreEndpoints() error {
   115  	if d.localStore == nil {
   116  		logrus.Warn("Cannot restore overlay endpoints because local datastore is missing")
   117  		return nil
   118  	}
   119  	kvol, err := d.localStore.List(datastore.Key(overlayEndpointPrefix), &endpoint{})
   120  	if err != nil && err != datastore.ErrKeyNotFound {
   121  		return fmt.Errorf("failed to read overlay endpoint from store: %v", err)
   122  	}
   123  
   124  	if err == datastore.ErrKeyNotFound {
   125  		return nil
   126  	}
   127  	for _, kvo := range kvol {
   128  		ep := kvo.(*endpoint)
   129  		n := d.network(ep.nid)
   130  		if n == nil {
   131  			logrus.Debugf("Network (%.7s) not found for restored endpoint (%.7s)", ep.nid, ep.id)
   132  			logrus.Debugf("Deleting stale overlay endpoint (%.7s) from store", ep.id)
   133  			if err := d.deleteEndpointFromStore(ep); err != nil {
   134  				logrus.Debugf("Failed to delete stale overlay endpoint (%.7s) from store", ep.id)
   135  			}
   136  			continue
   137  		}
   138  		n.addEndpoint(ep)
   139  
   140  		s := n.getSubnetforIP(ep.addr)
   141  		if s == nil {
   142  			return fmt.Errorf("could not find subnet for endpoint %s", ep.id)
   143  		}
   144  
   145  		if err := n.joinSandbox(s, true, true); err != nil {
   146  			return fmt.Errorf("restore network sandbox failed: %v", err)
   147  		}
   148  
   149  		Ifaces := make(map[string][]osl.IfaceOption)
   150  		vethIfaceOption := make([]osl.IfaceOption, 1)
   151  		vethIfaceOption = append(vethIfaceOption, n.sbox.InterfaceOptions().Master(s.brName))
   152  		Ifaces["veth+veth"] = vethIfaceOption
   153  
   154  		err := n.sbox.Restore(Ifaces, nil, nil, nil)
   155  		if err != nil {
   156  			n.leaveSandbox()
   157  			return fmt.Errorf("failed to restore overlay sandbox: %v", err)
   158  		}
   159  
   160  		d.peerAdd(ep.nid, ep.id, ep.addr.IP, ep.addr.Mask, ep.mac, net.ParseIP(d.advertiseAddress), false, false, true)
   161  	}
   162  	return nil
   163  }
   164  
   165  // Fini cleans up the driver resources
   166  func Fini(drv driverapi.Driver) {
   167  	d := drv.(*driver)
   168  
   169  	// Notify the peer go routine to return
   170  	if d.peerOpCancel != nil {
   171  		d.peerOpCancel()
   172  	}
   173  
   174  	if d.exitCh != nil {
   175  		waitCh := make(chan struct{})
   176  
   177  		d.exitCh <- waitCh
   178  
   179  		<-waitCh
   180  	}
   181  }
   182  
   183  func (d *driver) configure() error {
   184  	// Apply OS specific kernel configs if needed
   185  	d.initOS.Do(applyOStweaks)
   186  
   187  	if d.store == nil {
   188  		return nil
   189  	}
   190  
   191  	if d.vxlanIdm == nil {
   192  		return d.initializeVxlanIdm()
   193  	}
   194  
   195  	return nil
   196  }
   197  
   198  func (d *driver) initializeVxlanIdm() error {
   199  	var err error
   200  
   201  	initVxlanIdm <- true
   202  	defer func() { <-initVxlanIdm }()
   203  
   204  	if d.vxlanIdm != nil {
   205  		return nil
   206  	}
   207  
   208  	d.vxlanIdm, err = idm.New(d.store, "vxlan-id", vxlanIDStart, vxlanIDEnd)
   209  	if err != nil {
   210  		return fmt.Errorf("failed to initialize vxlan id manager: %v", err)
   211  	}
   212  
   213  	return nil
   214  }
   215  
   216  func (d *driver) Type() string {
   217  	return networkType
   218  }
   219  
   220  func (d *driver) IsBuiltIn() bool {
   221  	return true
   222  }
   223  
   224  func validateSelf(node string) error {
   225  	advIP := net.ParseIP(node)
   226  	if advIP == nil {
   227  		return fmt.Errorf("invalid self address (%s)", node)
   228  	}
   229  
   230  	addrs, err := net.InterfaceAddrs()
   231  	if err != nil {
   232  		return fmt.Errorf("Unable to get interface addresses %v", err)
   233  	}
   234  	for _, addr := range addrs {
   235  		ip, _, err := net.ParseCIDR(addr.String())
   236  		if err == nil && ip.Equal(advIP) {
   237  			return nil
   238  		}
   239  	}
   240  	return fmt.Errorf("Multi-Host overlay networking requires cluster-advertise(%s) to be configured with a local ip-address that is reachable within the cluster", advIP.String())
   241  }
   242  
   243  func (d *driver) nodeJoin(advertiseAddress, bindAddress string, self bool) {
   244  	if self && !d.isSerfAlive() {
   245  		d.Lock()
   246  		d.advertiseAddress = advertiseAddress
   247  		d.bindAddress = bindAddress
   248  		d.Unlock()
   249  
   250  		// If containers are already running on this network update the
   251  		// advertise address in the peerDB
   252  		d.localJoinOnce.Do(func() {
   253  			d.peerDBUpdateSelf()
   254  		})
   255  
   256  		// If there is no cluster store there is no need to start serf.
   257  		if d.store != nil {
   258  			if err := validateSelf(advertiseAddress); err != nil {
   259  				logrus.Warn(err.Error())
   260  			}
   261  			err := d.serfInit()
   262  			if err != nil {
   263  				logrus.Errorf("initializing serf instance failed: %v", err)
   264  				d.Lock()
   265  				d.advertiseAddress = ""
   266  				d.bindAddress = ""
   267  				d.Unlock()
   268  				return
   269  			}
   270  		}
   271  	}
   272  
   273  	d.Lock()
   274  	if !self {
   275  		d.neighIP = advertiseAddress
   276  	}
   277  	neighIP := d.neighIP
   278  	d.Unlock()
   279  
   280  	if d.serfInstance != nil && neighIP != "" {
   281  		var err error
   282  		d.joinOnce.Do(func() {
   283  			err = d.serfJoin(neighIP)
   284  			if err == nil {
   285  				d.pushLocalDb()
   286  			}
   287  		})
   288  		if err != nil {
   289  			logrus.Errorf("joining serf neighbor %s failed: %v", advertiseAddress, err)
   290  			d.Lock()
   291  			d.joinOnce = sync.Once{}
   292  			d.Unlock()
   293  			return
   294  		}
   295  	}
   296  }
   297  
   298  func (d *driver) pushLocalEndpointEvent(action, nid, eid string) {
   299  	n := d.network(nid)
   300  	if n == nil {
   301  		logrus.Debugf("Error pushing local endpoint event for network %s", nid)
   302  		return
   303  	}
   304  	ep := n.endpoint(eid)
   305  	if ep == nil {
   306  		logrus.Debugf("Error pushing local endpoint event for ep %s / %s", nid, eid)
   307  		return
   308  	}
   309  
   310  	if !d.isSerfAlive() {
   311  		return
   312  	}
   313  	d.notifyCh <- ovNotify{
   314  		action: "join",
   315  		nw:     n,
   316  		ep:     ep,
   317  	}
   318  }
   319  
   320  // DiscoverNew is a notification for a new discovery event, such as a new node joining a cluster
   321  func (d *driver) DiscoverNew(dType discoverapi.DiscoveryType, data interface{}) error {
   322  	var err error
   323  	switch dType {
   324  	case discoverapi.NodeDiscovery:
   325  		nodeData, ok := data.(discoverapi.NodeDiscoveryData)
   326  		if !ok || nodeData.Address == "" {
   327  			return fmt.Errorf("invalid discovery data")
   328  		}
   329  		d.nodeJoin(nodeData.Address, nodeData.BindAddress, nodeData.Self)
   330  	case discoverapi.DatastoreConfig:
   331  		if d.store != nil {
   332  			return types.ForbiddenErrorf("cannot accept datastore configuration: Overlay driver has a datastore configured already")
   333  		}
   334  		dsc, ok := data.(discoverapi.DatastoreConfigData)
   335  		if !ok {
   336  			return types.InternalErrorf("incorrect data in datastore configuration: %v", data)
   337  		}
   338  		d.store, err = datastore.NewDataStoreFromConfig(dsc)
   339  		if err != nil {
   340  			return types.InternalErrorf("failed to initialize data store: %v", err)
   341  		}
   342  	case discoverapi.EncryptionKeysConfig:
   343  		encrData, ok := data.(discoverapi.DriverEncryptionConfig)
   344  		if !ok {
   345  			return fmt.Errorf("invalid encryption key notification data")
   346  		}
   347  		keys := make([]*key, 0, len(encrData.Keys))
   348  		for i := 0; i < len(encrData.Keys); i++ {
   349  			k := &key{
   350  				value: encrData.Keys[i],
   351  				tag:   uint32(encrData.Tags[i]),
   352  			}
   353  			keys = append(keys, k)
   354  		}
   355  		if err := d.setKeys(keys); err != nil {
   356  			logrus.Warn(err)
   357  		}
   358  	case discoverapi.EncryptionKeysUpdate:
   359  		var newKey, delKey, priKey *key
   360  		encrData, ok := data.(discoverapi.DriverEncryptionUpdate)
   361  		if !ok {
   362  			return fmt.Errorf("invalid encryption key notification data")
   363  		}
   364  		if encrData.Key != nil {
   365  			newKey = &key{
   366  				value: encrData.Key,
   367  				tag:   uint32(encrData.Tag),
   368  			}
   369  		}
   370  		if encrData.Primary != nil {
   371  			priKey = &key{
   372  				value: encrData.Primary,
   373  				tag:   uint32(encrData.PrimaryTag),
   374  			}
   375  		}
   376  		if encrData.Prune != nil {
   377  			delKey = &key{
   378  				value: encrData.Prune,
   379  				tag:   uint32(encrData.PruneTag),
   380  			}
   381  		}
   382  		if err := d.updateKeys(newKey, priKey, delKey); err != nil {
   383  			return err
   384  		}
   385  	default:
   386  	}
   387  	return nil
   388  }
   389  
   390  // DiscoverDelete is a notification for a discovery delete event, such as a node leaving a cluster
   391  func (d *driver) DiscoverDelete(dType discoverapi.DiscoveryType, data interface{}) error {
   392  	return nil
   393  }