github.com/zhuohuang-hust/src-cbuild@v0.0.0-20230105071821-c7aab3e7c840/mergeCode/libnetwork/drivers/overlay/ov_network.go (about)

     1  package overlay
     2  
     3  import (
     4  	"encoding/json"
     5  	"fmt"
     6  	"net"
     7  	"os"
     8  	"path/filepath"
     9  	"strconv"
    10  	"strings"
    11  	"sync"
    12  	"syscall"
    13  
    14  	"github.com/Sirupsen/logrus"
    15  	"github.com/docker/libnetwork/datastore"
    16  	"github.com/docker/libnetwork/driverapi"
    17  	"github.com/docker/libnetwork/netlabel"
    18  	"github.com/docker/libnetwork/netutils"
    19  	"github.com/docker/libnetwork/ns"
    20  	"github.com/docker/libnetwork/osl"
    21  	"github.com/docker/libnetwork/resolvconf"
    22  	"github.com/docker/libnetwork/types"
    23  	"github.com/vishvananda/netlink"
    24  	"github.com/vishvananda/netlink/nl"
    25  	"github.com/vishvananda/netns"
    26  )
    27  
    28  var (
    29  	hostMode    bool
    30  	networkOnce sync.Once
    31  	networkMu   sync.Mutex
    32  	vniTbl      = make(map[uint32]string)
    33  )
    34  
    35  type networkTable map[string]*network
    36  
    37  type subnet struct {
    38  	once      *sync.Once
    39  	vxlanName string
    40  	brName    string
    41  	vni       uint32
    42  	initErr   error
    43  	subnetIP  *net.IPNet
    44  	gwIP      *net.IPNet
    45  }
    46  
    47  type subnetJSON struct {
    48  	SubnetIP string
    49  	GwIP     string
    50  	Vni      uint32
    51  }
    52  
    53  type network struct {
    54  	id        string
    55  	dbIndex   uint64
    56  	dbExists  bool
    57  	sbox      osl.Sandbox
    58  	endpoints endpointTable
    59  	driver    *driver
    60  	joinCnt   int
    61  	once      *sync.Once
    62  	initEpoch int
    63  	initErr   error
    64  	subnets   []*subnet
    65  	secure    bool
    66  	mtu       int
    67  	sync.Mutex
    68  }
    69  
    70  func (d *driver) NetworkAllocate(id string, option map[string]string, ipV4Data, ipV6Data []driverapi.IPAMData) (map[string]string, error) {
    71  	return nil, types.NotImplementedErrorf("not implemented")
    72  }
    73  
    74  func (d *driver) NetworkFree(id string) error {
    75  	return types.NotImplementedErrorf("not implemented")
    76  }
    77  
    78  func (d *driver) CreateNetwork(id string, option map[string]interface{}, nInfo driverapi.NetworkInfo, ipV4Data, ipV6Data []driverapi.IPAMData) error {
    79  	if id == "" {
    80  		return fmt.Errorf("invalid network id")
    81  	}
    82  	if len(ipV4Data) == 0 || ipV4Data[0].Pool.String() == "0.0.0.0/0" {
    83  		return types.BadRequestErrorf("ipv4 pool is empty")
    84  	}
    85  
    86  	// Since we perform lazy configuration make sure we try
    87  	// configuring the driver when we enter CreateNetwork
    88  	if err := d.configure(); err != nil {
    89  		return err
    90  	}
    91  
    92  	n := &network{
    93  		id:        id,
    94  		driver:    d,
    95  		endpoints: endpointTable{},
    96  		once:      &sync.Once{},
    97  		subnets:   []*subnet{},
    98  	}
    99  
   100  	vnis := make([]uint32, 0, len(ipV4Data))
   101  	if gval, ok := option[netlabel.GenericData]; ok {
   102  		optMap := gval.(map[string]string)
   103  		if val, ok := optMap[netlabel.OverlayVxlanIDList]; ok {
   104  			logrus.Debugf("overlay: Received vxlan IDs: %s", val)
   105  			vniStrings := strings.Split(val, ",")
   106  			for _, vniStr := range vniStrings {
   107  				vni, err := strconv.Atoi(vniStr)
   108  				if err != nil {
   109  					return fmt.Errorf("invalid vxlan id value %q passed", vniStr)
   110  				}
   111  
   112  				vnis = append(vnis, uint32(vni))
   113  			}
   114  		}
   115  		if _, ok := optMap[secureOption]; ok {
   116  			n.secure = true
   117  		}
   118  		if val, ok := optMap[netlabel.DriverMTU]; ok {
   119  			var err error
   120  			if n.mtu, err = strconv.Atoi(val); err != nil {
   121  				return fmt.Errorf("failed to parse %v: %v", val, err)
   122  			}
   123  			if n.mtu < 0 {
   124  				return fmt.Errorf("invalid MTU value: %v", n.mtu)
   125  			}
   126  		}
   127  	}
   128  
   129  	// If we are getting vnis from libnetwork, either we get for
   130  	// all subnets or none.
   131  	if len(vnis) != 0 && len(vnis) < len(ipV4Data) {
   132  		return fmt.Errorf("insufficient vnis(%d) passed to overlay", len(vnis))
   133  	}
   134  
   135  	for i, ipd := range ipV4Data {
   136  		s := &subnet{
   137  			subnetIP: ipd.Pool,
   138  			gwIP:     ipd.Gateway,
   139  			once:     &sync.Once{},
   140  		}
   141  
   142  		if len(vnis) != 0 {
   143  			s.vni = vnis[i]
   144  		}
   145  
   146  		n.subnets = append(n.subnets, s)
   147  	}
   148  
   149  	if err := n.writeToStore(); err != nil {
   150  		return fmt.Errorf("failed to update data store for network %v: %v", n.id, err)
   151  	}
   152  
   153  	// Make sure no rule is on the way from any stale secure network
   154  	if !n.secure {
   155  		for _, vni := range vnis {
   156  			programMangle(vni, false)
   157  		}
   158  	}
   159  
   160  	if nInfo != nil {
   161  		if err := nInfo.TableEventRegister(ovPeerTable); err != nil {
   162  			return err
   163  		}
   164  	}
   165  
   166  	d.addNetwork(n)
   167  	return nil
   168  }
   169  
   170  func (d *driver) DeleteNetwork(nid string) error {
   171  	if nid == "" {
   172  		return fmt.Errorf("invalid network id")
   173  	}
   174  
   175  	// Make sure driver resources are initialized before proceeding
   176  	if err := d.configure(); err != nil {
   177  		return err
   178  	}
   179  
   180  	n := d.network(nid)
   181  	if n == nil {
   182  		return fmt.Errorf("could not find network with id %s", nid)
   183  	}
   184  
   185  	d.deleteNetwork(nid)
   186  
   187  	vnis, err := n.releaseVxlanID()
   188  	if err != nil {
   189  		return err
   190  	}
   191  
   192  	if n.secure {
   193  		for _, vni := range vnis {
   194  			programMangle(vni, false)
   195  		}
   196  	}
   197  
   198  	return nil
   199  }
   200  
   201  func (d *driver) ProgramExternalConnectivity(nid, eid string, options map[string]interface{}) error {
   202  	return nil
   203  }
   204  
   205  func (d *driver) RevokeExternalConnectivity(nid, eid string) error {
   206  	return nil
   207  }
   208  
   209  func (n *network) incEndpointCount() {
   210  	n.Lock()
   211  	defer n.Unlock()
   212  	n.joinCnt++
   213  }
   214  
   215  func (n *network) joinSandbox(restore bool) error {
   216  	// If there is a race between two go routines here only one will win
   217  	// the other will wait.
   218  	n.once.Do(func() {
   219  		// save the error status of initSandbox in n.initErr so that
   220  		// all the racing go routines are able to know the status.
   221  		n.initErr = n.initSandbox(restore)
   222  	})
   223  
   224  	return n.initErr
   225  }
   226  
   227  func (n *network) joinSubnetSandbox(s *subnet, restore bool) error {
   228  	s.once.Do(func() {
   229  		s.initErr = n.initSubnetSandbox(s, restore)
   230  	})
   231  	return s.initErr
   232  }
   233  
   234  func (n *network) leaveSandbox() {
   235  	n.Lock()
   236  	defer n.Unlock()
   237  	n.joinCnt--
   238  	if n.joinCnt != 0 {
   239  		return
   240  	}
   241  
   242  	// We are about to destroy sandbox since the container is leaving the network
   243  	// Reinitialize the once variable so that we will be able to trigger one time
   244  	// sandbox initialization(again) when another container joins subsequently.
   245  	n.once = &sync.Once{}
   246  	for _, s := range n.subnets {
   247  		s.once = &sync.Once{}
   248  	}
   249  
   250  	n.destroySandbox()
   251  }
   252  
   253  // to be called while holding network lock
   254  func (n *network) destroySandbox() {
   255  	if n.sbox != nil {
   256  		for _, iface := range n.sbox.Info().Interfaces() {
   257  			if err := iface.Remove(); err != nil {
   258  				logrus.Debugf("Remove interface %s failed: %v", iface.SrcName(), err)
   259  			}
   260  		}
   261  
   262  		for _, s := range n.subnets {
   263  			if hostMode {
   264  				if err := removeFilters(n.id[:12], s.brName); err != nil {
   265  					logrus.Warnf("Could not remove overlay filters: %v", err)
   266  				}
   267  			}
   268  
   269  			if s.vxlanName != "" {
   270  				err := deleteInterface(s.vxlanName)
   271  				if err != nil {
   272  					logrus.Warnf("could not cleanup sandbox properly: %v", err)
   273  				}
   274  			}
   275  		}
   276  
   277  		if hostMode {
   278  			if err := removeNetworkChain(n.id[:12]); err != nil {
   279  				logrus.Warnf("could not remove network chain: %v", err)
   280  			}
   281  		}
   282  
   283  		n.sbox.Destroy()
   284  		n.sbox = nil
   285  	}
   286  }
   287  
   288  func populateVNITbl() {
   289  	filepath.Walk(filepath.Dir(osl.GenerateKey("walk")),
   290  		func(path string, info os.FileInfo, err error) error {
   291  			_, fname := filepath.Split(path)
   292  
   293  			if len(strings.Split(fname, "-")) <= 1 {
   294  				return nil
   295  			}
   296  
   297  			ns, err := netns.GetFromPath(path)
   298  			if err != nil {
   299  				logrus.Errorf("Could not open namespace path %s during vni population: %v", path, err)
   300  				return nil
   301  			}
   302  			defer ns.Close()
   303  
   304  			nlh, err := netlink.NewHandleAt(ns, syscall.NETLINK_ROUTE)
   305  			if err != nil {
   306  				logrus.Errorf("Could not open netlink handle during vni population for ns %s: %v", path, err)
   307  				return nil
   308  			}
   309  			defer nlh.Delete()
   310  
   311  			links, err := nlh.LinkList()
   312  			if err != nil {
   313  				logrus.Errorf("Failed to list interfaces during vni population for ns %s: %v", path, err)
   314  				return nil
   315  			}
   316  
   317  			for _, l := range links {
   318  				if l.Type() == "vxlan" {
   319  					vniTbl[uint32(l.(*netlink.Vxlan).VxlanId)] = path
   320  				}
   321  			}
   322  
   323  			return nil
   324  		})
   325  }
   326  
   327  func networkOnceInit() {
   328  	populateVNITbl()
   329  
   330  	if os.Getenv("_OVERLAY_HOST_MODE") != "" {
   331  		hostMode = true
   332  		return
   333  	}
   334  
   335  	err := createVxlan("testvxlan", 1, 0)
   336  	if err != nil {
   337  		logrus.Errorf("Failed to create testvxlan interface: %v", err)
   338  		return
   339  	}
   340  
   341  	defer deleteInterface("testvxlan")
   342  
   343  	path := "/proc/self/ns/net"
   344  	hNs, err := netns.GetFromPath(path)
   345  	if err != nil {
   346  		logrus.Errorf("Failed to get network namespace from path %s while setting host mode: %v", path, err)
   347  		return
   348  	}
   349  	defer hNs.Close()
   350  
   351  	nlh := ns.NlHandle()
   352  
   353  	iface, err := nlh.LinkByName("testvxlan")
   354  	if err != nil {
   355  		logrus.Errorf("Failed to get link testvxlan while setting host mode: %v", err)
   356  		return
   357  	}
   358  
   359  	// If we are not able to move the vxlan interface to a namespace
   360  	// then fallback to host mode
   361  	if err := nlh.LinkSetNsFd(iface, int(hNs)); err != nil {
   362  		hostMode = true
   363  	}
   364  }
   365  
   366  func (n *network) generateVxlanName(s *subnet) string {
   367  	id := n.id
   368  	if len(n.id) > 5 {
   369  		id = n.id[:5]
   370  	}
   371  
   372  	return "vx-" + fmt.Sprintf("%06x", n.vxlanID(s)) + "-" + id
   373  }
   374  
   375  func (n *network) generateBridgeName(s *subnet) string {
   376  	id := n.id
   377  	if len(n.id) > 5 {
   378  		id = n.id[:5]
   379  	}
   380  
   381  	return n.getBridgeNamePrefix(s) + "-" + id
   382  }
   383  
   384  func (n *network) getBridgeNamePrefix(s *subnet) string {
   385  	return "ov-" + fmt.Sprintf("%06x", n.vxlanID(s))
   386  }
   387  
   388  func isOverlap(nw *net.IPNet) bool {
   389  	var nameservers []string
   390  
   391  	if rc, err := resolvconf.Get(); err == nil {
   392  		nameservers = resolvconf.GetNameserversAsCIDR(rc.Content)
   393  	}
   394  
   395  	if err := netutils.CheckNameserverOverlaps(nameservers, nw); err != nil {
   396  		return true
   397  	}
   398  
   399  	if err := netutils.CheckRouteOverlaps(nw); err != nil {
   400  		return true
   401  	}
   402  
   403  	return false
   404  }
   405  
   406  func (n *network) restoreSubnetSandbox(s *subnet, brName, vxlanName string) error {
   407  	sbox := n.sandbox()
   408  
   409  	// restore overlay osl sandbox
   410  	Ifaces := make(map[string][]osl.IfaceOption)
   411  	brIfaceOption := make([]osl.IfaceOption, 2)
   412  	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Address(s.gwIP))
   413  	brIfaceOption = append(brIfaceOption, sbox.InterfaceOptions().Bridge(true))
   414  	Ifaces[fmt.Sprintf("%s+%s", brName, "br")] = brIfaceOption
   415  
   416  	err := sbox.Restore(Ifaces, nil, nil, nil)
   417  	if err != nil {
   418  		return err
   419  	}
   420  
   421  	Ifaces = make(map[string][]osl.IfaceOption)
   422  	vxlanIfaceOption := make([]osl.IfaceOption, 1)
   423  	vxlanIfaceOption = append(vxlanIfaceOption, sbox.InterfaceOptions().Master(brName))
   424  	Ifaces[fmt.Sprintf("%s+%s", vxlanName, "vxlan")] = vxlanIfaceOption
   425  	err = sbox.Restore(Ifaces, nil, nil, nil)
   426  	if err != nil {
   427  		return err
   428  	}
   429  	return nil
   430  }
   431  
   432  func (n *network) setupSubnetSandbox(s *subnet, brName, vxlanName string) error {
   433  
   434  	if hostMode {
   435  		// Try to delete stale bridge interface if it exists
   436  		if err := deleteInterface(brName); err != nil {
   437  			deleteInterfaceBySubnet(n.getBridgeNamePrefix(s), s)
   438  		}
   439  		// Try to delete the vxlan interface by vni if already present
   440  		deleteVxlanByVNI("", n.vxlanID(s))
   441  
   442  		if isOverlap(s.subnetIP) {
   443  			return fmt.Errorf("overlay subnet %s has conflicts in the host while running in host mode", s.subnetIP.String())
   444  		}
   445  	}
   446  
   447  	if !hostMode {
   448  		// Try to find this subnet's vni is being used in some
   449  		// other namespace by looking at vniTbl that we just
   450  		// populated in the once init. If a hit is found then
   451  		// it must a stale namespace from previous
   452  		// life. Destroy it completely and reclaim resourced.
   453  		networkMu.Lock()
   454  		path, ok := vniTbl[n.vxlanID(s)]
   455  		networkMu.Unlock()
   456  
   457  		if ok {
   458  			deleteVxlanByVNI(path, n.vxlanID(s))
   459  			if err := syscall.Unmount(path, syscall.MNT_FORCE); err != nil {
   460  				logrus.Errorf("unmount of %s failed: %v", path, err)
   461  			}
   462  			os.Remove(path)
   463  
   464  			networkMu.Lock()
   465  			delete(vniTbl, n.vxlanID(s))
   466  			networkMu.Unlock()
   467  		}
   468  	}
   469  
   470  	// create a bridge and vxlan device for this subnet and move it to the sandbox
   471  	sbox := n.sandbox()
   472  
   473  	if err := sbox.AddInterface(brName, "br",
   474  		sbox.InterfaceOptions().Address(s.gwIP),
   475  		sbox.InterfaceOptions().Bridge(true)); err != nil {
   476  		return fmt.Errorf("bridge creation in sandbox failed for subnet %q: %v", s.subnetIP.String(), err)
   477  	}
   478  
   479  	err := createVxlan(vxlanName, n.vxlanID(s), n.maxMTU())
   480  	if err != nil {
   481  		return err
   482  	}
   483  
   484  	if err := sbox.AddInterface(vxlanName, "vxlan",
   485  		sbox.InterfaceOptions().Master(brName)); err != nil {
   486  		return fmt.Errorf("vxlan interface creation failed for subnet %q: %v", s.subnetIP.String(), err)
   487  	}
   488  
   489  	if hostMode {
   490  		if err := addFilters(n.id[:12], brName); err != nil {
   491  			return err
   492  		}
   493  	}
   494  
   495  	return nil
   496  }
   497  
   498  func (n *network) initSubnetSandbox(s *subnet, restore bool) error {
   499  	brName := n.generateBridgeName(s)
   500  	vxlanName := n.generateVxlanName(s)
   501  
   502  	if restore {
   503  		if err := n.restoreSubnetSandbox(s, brName, vxlanName); err != nil {
   504  			return err
   505  		}
   506  	} else {
   507  		if err := n.setupSubnetSandbox(s, brName, vxlanName); err != nil {
   508  			return err
   509  		}
   510  	}
   511  
   512  	n.Lock()
   513  	s.vxlanName = vxlanName
   514  	s.brName = brName
   515  	n.Unlock()
   516  
   517  	return nil
   518  }
   519  
   520  func (n *network) cleanupStaleSandboxes() {
   521  	filepath.Walk(filepath.Dir(osl.GenerateKey("walk")),
   522  		func(path string, info os.FileInfo, err error) error {
   523  			_, fname := filepath.Split(path)
   524  
   525  			pList := strings.Split(fname, "-")
   526  			if len(pList) <= 1 {
   527  				return nil
   528  			}
   529  
   530  			pattern := pList[1]
   531  			if strings.Contains(n.id, pattern) {
   532  				// Delete all vnis
   533  				deleteVxlanByVNI(path, 0)
   534  				syscall.Unmount(path, syscall.MNT_DETACH)
   535  				os.Remove(path)
   536  
   537  				// Now that we have destroyed this
   538  				// sandbox, remove all references to
   539  				// it in vniTbl so that we don't
   540  				// inadvertently destroy the sandbox
   541  				// created in this life.
   542  				networkMu.Lock()
   543  				for vni, tblPath := range vniTbl {
   544  					if tblPath == path {
   545  						delete(vniTbl, vni)
   546  					}
   547  				}
   548  				networkMu.Unlock()
   549  			}
   550  
   551  			return nil
   552  		})
   553  }
   554  
   555  func (n *network) initSandbox(restore bool) error {
   556  	n.Lock()
   557  	n.initEpoch++
   558  	n.Unlock()
   559  
   560  	networkOnce.Do(networkOnceInit)
   561  
   562  	if !restore {
   563  		if hostMode {
   564  			if err := addNetworkChain(n.id[:12]); err != nil {
   565  				return err
   566  			}
   567  		}
   568  
   569  		// If there are any stale sandboxes related to this network
   570  		// from previous daemon life clean it up here
   571  		n.cleanupStaleSandboxes()
   572  	}
   573  
   574  	// In the restore case network sandbox already exist; but we don't know
   575  	// what epoch number it was created with. It has to be retrieved by
   576  	// searching the net namespaces.
   577  	key := ""
   578  	if restore {
   579  		key = osl.GenerateKey("-" + n.id)
   580  	} else {
   581  		key = osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch) + n.id)
   582  	}
   583  
   584  	sbox, err := osl.NewSandbox(key, !hostMode, restore)
   585  	if err != nil {
   586  		return fmt.Errorf("could not get network sandbox (oper %t): %v", restore, err)
   587  	}
   588  
   589  	n.setSandbox(sbox)
   590  
   591  	if !restore {
   592  		n.driver.peerDbUpdateSandbox(n.id)
   593  	}
   594  
   595  	var nlSock *nl.NetlinkSocket
   596  	sbox.InvokeFunc(func() {
   597  		nlSock, err = nl.Subscribe(syscall.NETLINK_ROUTE, syscall.RTNLGRP_NEIGH)
   598  		if err != nil {
   599  			err = fmt.Errorf("failed to subscribe to neighbor group netlink messages")
   600  		}
   601  	})
   602  
   603  	if nlSock != nil {
   604  		go n.watchMiss(nlSock)
   605  	}
   606  
   607  	return nil
   608  }
   609  
   610  func (n *network) watchMiss(nlSock *nl.NetlinkSocket) {
   611  	for {
   612  		msgs, err := nlSock.Receive()
   613  		if err != nil {
   614  			logrus.Errorf("Failed to receive from netlink: %v ", err)
   615  			continue
   616  		}
   617  
   618  		for _, msg := range msgs {
   619  			if msg.Header.Type != syscall.RTM_GETNEIGH && msg.Header.Type != syscall.RTM_NEWNEIGH {
   620  				continue
   621  			}
   622  
   623  			neigh, err := netlink.NeighDeserialize(msg.Data)
   624  			if err != nil {
   625  				logrus.Errorf("Failed to deserialize netlink ndmsg: %v", err)
   626  				continue
   627  			}
   628  
   629  			if neigh.IP.To4() == nil {
   630  				continue
   631  			}
   632  
   633  			// Not any of the network's subnets. Ignore.
   634  			if !n.contains(neigh.IP) {
   635  				continue
   636  			}
   637  
   638  			logrus.Debugf("miss notification for dest IP, %v", neigh.IP.String())
   639  
   640  			if neigh.State&(netlink.NUD_STALE|netlink.NUD_INCOMPLETE) == 0 {
   641  				continue
   642  			}
   643  
   644  			if !n.driver.isSerfAlive() {
   645  				continue
   646  			}
   647  
   648  			mac, IPmask, vtep, err := n.driver.resolvePeer(n.id, neigh.IP)
   649  			if err != nil {
   650  				logrus.Errorf("could not resolve peer %q: %v", neigh.IP, err)
   651  				continue
   652  			}
   653  
   654  			if err := n.driver.peerAdd(n.id, "dummy", neigh.IP, IPmask, mac, vtep, true); err != nil {
   655  				logrus.Errorf("could not add neighbor entry for missed peer %q: %v", neigh.IP, err)
   656  			}
   657  		}
   658  	}
   659  }
   660  
   661  func (d *driver) addNetwork(n *network) {
   662  	d.Lock()
   663  	d.networks[n.id] = n
   664  	d.Unlock()
   665  }
   666  
   667  func (d *driver) deleteNetwork(nid string) {
   668  	d.Lock()
   669  	delete(d.networks, nid)
   670  	d.Unlock()
   671  }
   672  
   673  func (d *driver) network(nid string) *network {
   674  	d.Lock()
   675  	n, ok := d.networks[nid]
   676  	d.Unlock()
   677  	if !ok {
   678  		n = d.getNetworkFromStore(nid)
   679  		if n != nil {
   680  			n.driver = d
   681  			n.endpoints = endpointTable{}
   682  			n.once = &sync.Once{}
   683  			d.Lock()
   684  			d.networks[nid] = n
   685  			d.Unlock()
   686  		}
   687  	}
   688  
   689  	return n
   690  }
   691  
   692  func (d *driver) getNetworkFromStore(nid string) *network {
   693  	if d.store == nil {
   694  		return nil
   695  	}
   696  
   697  	n := &network{id: nid}
   698  	if err := d.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
   699  		return nil
   700  	}
   701  
   702  	return n
   703  }
   704  
   705  func (n *network) sandbox() osl.Sandbox {
   706  	n.Lock()
   707  	defer n.Unlock()
   708  
   709  	return n.sbox
   710  }
   711  
   712  func (n *network) setSandbox(sbox osl.Sandbox) {
   713  	n.Lock()
   714  	n.sbox = sbox
   715  	n.Unlock()
   716  }
   717  
   718  func (n *network) vxlanID(s *subnet) uint32 {
   719  	n.Lock()
   720  	defer n.Unlock()
   721  
   722  	return s.vni
   723  }
   724  
   725  func (n *network) setVxlanID(s *subnet, vni uint32) {
   726  	n.Lock()
   727  	s.vni = vni
   728  	n.Unlock()
   729  }
   730  
   731  func (n *network) Key() []string {
   732  	return []string{"overlay", "network", n.id}
   733  }
   734  
   735  func (n *network) KeyPrefix() []string {
   736  	return []string{"overlay", "network"}
   737  }
   738  
   739  func (n *network) Value() []byte {
   740  	m := map[string]interface{}{}
   741  
   742  	netJSON := []*subnetJSON{}
   743  
   744  	for _, s := range n.subnets {
   745  		sj := &subnetJSON{
   746  			SubnetIP: s.subnetIP.String(),
   747  			GwIP:     s.gwIP.String(),
   748  			Vni:      s.vni,
   749  		}
   750  		netJSON = append(netJSON, sj)
   751  	}
   752  
   753  	b, err := json.Marshal(netJSON)
   754  	if err != nil {
   755  		return []byte{}
   756  	}
   757  
   758  	m["secure"] = n.secure
   759  	m["subnets"] = netJSON
   760  	m["mtu"] = n.mtu
   761  	b, err = json.Marshal(m)
   762  	if err != nil {
   763  		return []byte{}
   764  	}
   765  
   766  	return b
   767  }
   768  
   769  func (n *network) Index() uint64 {
   770  	return n.dbIndex
   771  }
   772  
   773  func (n *network) SetIndex(index uint64) {
   774  	n.dbIndex = index
   775  	n.dbExists = true
   776  }
   777  
   778  func (n *network) Exists() bool {
   779  	return n.dbExists
   780  }
   781  
   782  func (n *network) Skip() bool {
   783  	return false
   784  }
   785  
   786  func (n *network) SetValue(value []byte) error {
   787  	var (
   788  		m       map[string]interface{}
   789  		newNet  bool
   790  		isMap   = true
   791  		netJSON = []*subnetJSON{}
   792  	)
   793  
   794  	if err := json.Unmarshal(value, &m); err != nil {
   795  		err := json.Unmarshal(value, &netJSON)
   796  		if err != nil {
   797  			return err
   798  		}
   799  		isMap = false
   800  	}
   801  
   802  	if len(n.subnets) == 0 {
   803  		newNet = true
   804  	}
   805  
   806  	if isMap {
   807  		if val, ok := m["secure"]; ok {
   808  			n.secure = val.(bool)
   809  		}
   810  		if val, ok := m["mtu"]; ok {
   811  			n.mtu = int(val.(float64))
   812  		}
   813  		bytes, err := json.Marshal(m["subnets"])
   814  		if err != nil {
   815  			return err
   816  		}
   817  		if err := json.Unmarshal(bytes, &netJSON); err != nil {
   818  			return err
   819  		}
   820  	}
   821  
   822  	for _, sj := range netJSON {
   823  		subnetIPstr := sj.SubnetIP
   824  		gwIPstr := sj.GwIP
   825  		vni := sj.Vni
   826  
   827  		subnetIP, _ := types.ParseCIDR(subnetIPstr)
   828  		gwIP, _ := types.ParseCIDR(gwIPstr)
   829  
   830  		if newNet {
   831  			s := &subnet{
   832  				subnetIP: subnetIP,
   833  				gwIP:     gwIP,
   834  				vni:      vni,
   835  				once:     &sync.Once{},
   836  			}
   837  			n.subnets = append(n.subnets, s)
   838  		} else {
   839  			sNet := n.getMatchingSubnet(subnetIP)
   840  			if sNet != nil {
   841  				sNet.vni = vni
   842  			}
   843  		}
   844  	}
   845  	return nil
   846  }
   847  
   848  func (n *network) DataScope() string {
   849  	return datastore.GlobalScope
   850  }
   851  
   852  func (n *network) writeToStore() error {
   853  	if n.driver.store == nil {
   854  		return nil
   855  	}
   856  
   857  	return n.driver.store.PutObjectAtomic(n)
   858  }
   859  
   860  func (n *network) releaseVxlanID() ([]uint32, error) {
   861  	if len(n.subnets) == 0 {
   862  		return nil, nil
   863  	}
   864  
   865  	if n.driver.store != nil {
   866  		if err := n.driver.store.DeleteObjectAtomic(n); err != nil {
   867  			if err == datastore.ErrKeyModified || err == datastore.ErrKeyNotFound {
   868  				// In both the above cases we can safely assume that the key has been removed by some other
   869  				// instance and so simply get out of here
   870  				return nil, nil
   871  			}
   872  
   873  			return nil, fmt.Errorf("failed to delete network to vxlan id map: %v", err)
   874  		}
   875  	}
   876  	var vnis []uint32
   877  	for _, s := range n.subnets {
   878  		if n.driver.vxlanIdm != nil {
   879  			vni := n.vxlanID(s)
   880  			vnis = append(vnis, vni)
   881  			n.driver.vxlanIdm.Release(uint64(vni))
   882  		}
   883  
   884  		n.setVxlanID(s, 0)
   885  	}
   886  
   887  	return vnis, nil
   888  }
   889  
   890  func (n *network) obtainVxlanID(s *subnet) error {
   891  	//return if the subnet already has a vxlan id assigned
   892  	if s.vni != 0 {
   893  		return nil
   894  	}
   895  
   896  	if n.driver.store == nil {
   897  		return fmt.Errorf("no valid vxlan id and no datastore configured, cannot obtain vxlan id")
   898  	}
   899  
   900  	for {
   901  		if err := n.driver.store.GetObject(datastore.Key(n.Key()...), n); err != nil {
   902  			return fmt.Errorf("getting network %q from datastore failed %v", n.id, err)
   903  		}
   904  
   905  		if s.vni == 0 {
   906  			vxlanID, err := n.driver.vxlanIdm.GetID()
   907  			if err != nil {
   908  				return fmt.Errorf("failed to allocate vxlan id: %v", err)
   909  			}
   910  
   911  			n.setVxlanID(s, uint32(vxlanID))
   912  			if err := n.writeToStore(); err != nil {
   913  				n.driver.vxlanIdm.Release(uint64(n.vxlanID(s)))
   914  				n.setVxlanID(s, 0)
   915  				if err == datastore.ErrKeyModified {
   916  					continue
   917  				}
   918  				return fmt.Errorf("network %q failed to update data store: %v", n.id, err)
   919  			}
   920  			return nil
   921  		}
   922  		return nil
   923  	}
   924  }
   925  
   926  // contains return true if the passed ip belongs to one the network's
   927  // subnets
   928  func (n *network) contains(ip net.IP) bool {
   929  	for _, s := range n.subnets {
   930  		if s.subnetIP.Contains(ip) {
   931  			return true
   932  		}
   933  	}
   934  
   935  	return false
   936  }
   937  
   938  // getSubnetforIP returns the subnet to which the given IP belongs
   939  func (n *network) getSubnetforIP(ip *net.IPNet) *subnet {
   940  	for _, s := range n.subnets {
   941  		// first check if the mask lengths are the same
   942  		i, _ := s.subnetIP.Mask.Size()
   943  		j, _ := ip.Mask.Size()
   944  		if i != j {
   945  			continue
   946  		}
   947  		if s.subnetIP.Contains(ip.IP) {
   948  			return s
   949  		}
   950  	}
   951  	return nil
   952  }
   953  
   954  // getMatchingSubnet return the network's subnet that matches the input
   955  func (n *network) getMatchingSubnet(ip *net.IPNet) *subnet {
   956  	if ip == nil {
   957  		return nil
   958  	}
   959  	for _, s := range n.subnets {
   960  		// first check if the mask lengths are the same
   961  		i, _ := s.subnetIP.Mask.Size()
   962  		j, _ := ip.Mask.Size()
   963  		if i != j {
   964  			continue
   965  		}
   966  		if s.subnetIP.IP.Equal(ip.IP) {
   967  			return s
   968  		}
   969  	}
   970  	return nil
   971  }