github.com/noironetworks/cilium-net@v1.6.12/cilium-health/launch/endpoint.go (about)

     1  // Copyright 2017-2019 Authors of Cilium
     2  //
     3  // Licensed under the Apache License, Version 2.0 (the "License");
     4  // you may not use this file except in compliance with the License.
     5  // You may obtain a copy of the License at
     6  //
     7  //     http://www.apache.org/licenses/LICENSE-2.0
     8  //
     9  // Unless required by applicable law or agreed to in writing, software
    10  // distributed under the License is distributed on an "AS IS" BASIS,
    11  // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12  // See the License for the specific language governing permissions and
    13  // limitations under the License.
    14  
    15  package launch
    16  
    17  import (
    18  	"context"
    19  	"fmt"
    20  	"net"
    21  	"os"
    22  	"os/exec"
    23  	"path/filepath"
    24  	"strings"
    25  	"time"
    26  
    27  	"github.com/cilium/cilium/api/v1/models"
    28  	"github.com/cilium/cilium/pkg/datapath/linux/route"
    29  	"github.com/cilium/cilium/pkg/defaults"
    30  	"github.com/cilium/cilium/pkg/endpoint"
    31  	"github.com/cilium/cilium/pkg/endpoint/connector"
    32  	"github.com/cilium/cilium/pkg/endpoint/regeneration"
    33  	"github.com/cilium/cilium/pkg/endpointmanager"
    34  	healthDefaults "github.com/cilium/cilium/pkg/health/defaults"
    35  	"github.com/cilium/cilium/pkg/health/probe"
    36  	"github.com/cilium/cilium/pkg/labels"
    37  	"github.com/cilium/cilium/pkg/launcher"
    38  	"github.com/cilium/cilium/pkg/logging/logfields"
    39  	"github.com/cilium/cilium/pkg/metrics"
    40  	"github.com/cilium/cilium/pkg/mtu"
    41  	"github.com/cilium/cilium/pkg/netns"
    42  	"github.com/cilium/cilium/pkg/node"
    43  	"github.com/cilium/cilium/pkg/option"
    44  	"github.com/cilium/cilium/pkg/pidfile"
    45  	"github.com/cilium/cilium/pkg/sysctl"
    46  
    47  	"github.com/containernetworking/plugins/pkg/ns"
    48  	"github.com/vishvananda/netlink"
    49  	"golang.org/x/sys/unix"
    50  )
    51  
    52  const (
    53  	ciliumHealth = "cilium-health"
    54  	netNSName    = "cilium-health"
    55  	binaryName   = "cilium-health-responder"
    56  )
    57  
    58  var (
    59  	// vethName is the host-side veth link device name for cilium-health EP
    60  	// (veth mode only).
    61  	vethName = "lxc_health"
    62  
    63  	// legacyVethName is the host-side cilium-health EP device name used in
    64  	// older Cilium versions. Used for removal only.
    65  	legacyVethName = "cilium_health"
    66  
    67  	// epIfaceName is the endpoint-side link device name for cilium-health.
    68  	epIfaceName = "cilium"
    69  
    70  	// PidfilePath
    71  	PidfilePath = "health-endpoint.pid"
    72  
    73  	// LaunchTime is the expected time within which the health endpoint
    74  	// should be able to be successfully run and its BPF program attached.
    75  	LaunchTime = 30 * time.Second
    76  )
    77  
    78  func configureHealthRouting(netns, dev string, addressing *models.NodeAddressing, mtuConfig mtu.Configuration) error {
    79  	routes := []route.Route{}
    80  
    81  	if option.Config.EnableIPv4 {
    82  		v4Routes, err := connector.IPv4Routes(addressing, mtuConfig.GetRouteMTU())
    83  		if err == nil {
    84  			routes = append(routes, v4Routes...)
    85  		} else {
    86  			log.Debugf("Couldn't get IPv4 routes for health routing")
    87  		}
    88  	}
    89  
    90  	if option.Config.EnableIPv6 {
    91  		v6Routes, err := connector.IPv6Routes(addressing, mtuConfig.GetRouteMTU())
    92  		if err != nil {
    93  			return fmt.Errorf("Failed to get IPv6 routes")
    94  		}
    95  		routes = append(routes, v6Routes...)
    96  	}
    97  
    98  	prog := "ip"
    99  	args := []string{"netns", "exec", netns, "bash", "-c"}
   100  	routeCmds := []string{}
   101  	for _, rt := range routes {
   102  		cmd := strings.Join(rt.ToIPCommand(dev), " ")
   103  		log.WithField("netns", netns).WithField("command", cmd).Debug("Adding route")
   104  		routeCmds = append(routeCmds, cmd)
   105  	}
   106  	cmd := strings.Join(routeCmds, " && ")
   107  	args = append(args, cmd)
   108  
   109  	log.Debugf("Running \"%s %+v\"", prog, args)
   110  	out, err := exec.Command(prog, args...).CombinedOutput()
   111  	if err == nil && len(out) > 0 {
   112  		log.Warn(out)
   113  	}
   114  
   115  	return err
   116  }
   117  
   118  func configureHealthInterface(netNS ns.NetNS, ifName string, ip4Addr, ip6Addr *net.IPNet) error {
   119  	return netNS.Do(func(_ ns.NetNS) error {
   120  		link, err := netlink.LinkByName(ifName)
   121  		if err != nil {
   122  			return err
   123  		}
   124  
   125  		if ip6Addr == nil {
   126  			name := fmt.Sprintf("net.ipv6.conf.%s.disable_ipv6", ifName)
   127  			// Ignore the error; if IPv6 is completely disabled
   128  			// then it's okay if we can't write the sysctl.
   129  			_ = sysctl.Write(name, "1")
   130  		} else {
   131  			if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip6Addr}); err != nil {
   132  				return err
   133  			}
   134  		}
   135  
   136  		if ip4Addr != nil {
   137  			if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip4Addr}); err != nil {
   138  				return err
   139  			}
   140  		}
   141  
   142  		if err = netlink.LinkSetUp(link); err != nil {
   143  			return err
   144  		}
   145  
   146  		lo, err := netlink.LinkByName("lo")
   147  		if err != nil {
   148  			return err
   149  		}
   150  
   151  		if err = netlink.LinkSetUp(lo); err != nil {
   152  			return err
   153  		}
   154  
   155  		return nil
   156  	})
   157  }
   158  
   159  // Client wraps a client to a specific cilium-health endpoint instance, to
   160  // provide convenience methods such as PingEndpoint().
   161  type Client struct {
   162  	host string
   163  }
   164  
   165  // PingEndpoint attempts to make an API ping request to the local cilium-health
   166  // endpoint, and returns whether this was successful.
   167  func (c *Client) PingEndpoint() error {
   168  	return probe.GetHello(c.host)
   169  }
   170  
   171  // KillEndpoint attempts to kill any existing cilium-health endpoint if it
   172  // exists.
   173  //
   174  // This is intended to be invoked in multiple situations:
   175  // * The health endpoint has never been run before
   176  // * The health endpoint was run during a previous run of the Cilium agent
   177  // * The health endpoint crashed during the current run of the Cilium agent
   178  //   and needs to be cleaned up before it is restarted.
   179  func KillEndpoint() {
   180  	path := filepath.Join(option.Config.StateDir, PidfilePath)
   181  	scopedLog := log.WithField(logfields.PIDFile, path)
   182  	scopedLog.Debug("Killing old health endpoint process")
   183  	pid, err := pidfile.Kill(path)
   184  	if err != nil {
   185  		scopedLog.WithError(err).Warning("Failed to kill cilium-health-responder")
   186  	} else if pid != 0 {
   187  		scopedLog.WithField(logfields.PID, pid).Debug("Killed endpoint process")
   188  	}
   189  }
   190  
   191  // CleanupEndpoint cleans up remaining resources associated with the health
   192  // endpoint.
   193  //
   194  // This is expected to be called after the process is killed and the endpoint
   195  // is removed from the endpointmanager.
   196  func CleanupEndpoint() {
   197  	// Removes the interfaces used for the endpoint process, followed by the
   198  	// deletion of the health namespace itself. The removal of the interfaces
   199  	// is needed, because network namespace removal does not always trigger the
   200  	// deletion of associated interfaces immediately (e.g. when a process in the
   201  	// namespace marked for deletion has not yet been terminated).
   202  	switch option.Config.DatapathMode {
   203  	case option.DatapathModeVeth:
   204  		for _, iface := range []string{legacyVethName, vethName} {
   205  			scopedLog := log.WithField(logfields.Veth, iface)
   206  			if link, err := netlink.LinkByName(iface); err == nil {
   207  				err = netlink.LinkDel(link)
   208  				if err != nil {
   209  					scopedLog.WithError(err).Info("Couldn't delete cilium-health veth device")
   210  				}
   211  			} else {
   212  				scopedLog.WithError(err).Debug("Didn't find existing device")
   213  			}
   214  		}
   215  	case option.DatapathModeIpvlan:
   216  		if err := netns.RemoveIfFromNetNSWithNameIfBothExist(netNSName, epIfaceName); err != nil {
   217  			log.WithError(err).WithField(logfields.Ipvlan, epIfaceName).
   218  				Info("Couldn't delete cilium-health ipvlan slave device")
   219  		}
   220  	}
   221  
   222  	if err := netns.RemoveNetNSWithName(netNSName); err != nil {
   223  		log.WithError(err).Debug("Unable to remove cilium-health namespace")
   224  	}
   225  }
   226  
   227  // LaunchAsEndpoint launches the cilium-health agent in a nested network
   228  // namespace and attaches it to Cilium the same way as any other endpoint,
   229  // but with special reserved labels.
   230  //
   231  // CleanupEndpoint() must be called before calling LaunchAsEndpoint() to ensure
   232  // cleanup of prior cilium-health endpoint instances.
   233  func LaunchAsEndpoint(baseCtx context.Context, owner regeneration.Owner, n *node.Node, mtuConfig mtu.Configuration) (*Client, error) {
   234  	var (
   235  		cmd  = launcher.Launcher{}
   236  		info = &models.EndpointChangeRequest{
   237  			ContainerName: ciliumHealth,
   238  			State:         models.EndpointStateWaitingForIdentity,
   239  			Addressing:    &models.AddressPair{},
   240  		}
   241  		healthIP               net.IP
   242  		ip4Address, ip6Address *net.IPNet
   243  	)
   244  
   245  	if n.IPv6HealthIP != nil {
   246  		healthIP = n.IPv6HealthIP
   247  		info.Addressing.IPV6 = healthIP.String()
   248  		ip6Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv6Mask}
   249  	}
   250  	if n.IPv4HealthIP != nil {
   251  		healthIP = n.IPv4HealthIP
   252  		info.Addressing.IPV4 = healthIP.String()
   253  		ip4Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv4Mask}
   254  	}
   255  
   256  	if option.Config.EnableEndpointRoutes {
   257  		dpConfig := &models.EndpointDatapathConfiguration{
   258  			InstallEndpointRoute: true,
   259  			RequireEgressProg:    true,
   260  		}
   261  		info.DatapathConfiguration = dpConfig
   262  	}
   263  
   264  	netNS, err := netns.ReplaceNetNSWithName(netNSName)
   265  	if err != nil {
   266  		return nil, err
   267  	}
   268  
   269  	switch option.Config.DatapathMode {
   270  	case option.DatapathModeVeth:
   271  		_, epLink, err := connector.SetupVethWithNames(vethName, epIfaceName, mtuConfig.GetDeviceMTU(), info)
   272  		if err != nil {
   273  			return nil, fmt.Errorf("Error while creating veth: %s", err)
   274  		}
   275  
   276  		if err = netlink.LinkSetNsFd(*epLink, int(netNS.Fd())); err != nil {
   277  			return nil, fmt.Errorf("failed to move device %q to health namespace: %s", epIfaceName, err)
   278  		}
   279  
   280  	case option.DatapathModeIpvlan:
   281  		mapFD, err := connector.CreateAndSetupIpvlanSlave("",
   282  			epIfaceName, netNS, mtuConfig.GetDeviceMTU(),
   283  			option.Config.Ipvlan.MasterDeviceIndex,
   284  			option.Config.Ipvlan.OperationMode, info)
   285  		if err != nil {
   286  			if errDel := netns.RemoveNetNSWithName(netNSName); errDel != nil {
   287  				log.WithError(errDel).WithField(logfields.NetNSName, netNSName).
   288  					Warning("Unable to remove network namespace")
   289  			}
   290  			return nil, err
   291  		}
   292  		defer unix.Close(mapFD)
   293  
   294  	}
   295  
   296  	if err = configureHealthInterface(netNS, epIfaceName, ip4Address, ip6Address); err != nil {
   297  		return nil, fmt.Errorf("failed configure health interface %q: %s", epIfaceName, err)
   298  	}
   299  
   300  	pidfile := filepath.Join(option.Config.StateDir, PidfilePath)
   301  	prog := "ip"
   302  	args := []string{"netns", "exec", netNSName, binaryName, "--pidfile", pidfile}
   303  	cmd.SetTarget(prog)
   304  	cmd.SetArgs(args)
   305  	log.Infof("Spawning health endpoint with command %q %q", prog, args)
   306  	if err := cmd.Run(); err != nil {
   307  		return nil, err
   308  	}
   309  
   310  	// Create the endpoint
   311  	ep, err := endpoint.NewEndpointFromChangeModel(owner, info)
   312  	if err != nil {
   313  		return nil, fmt.Errorf("Error while creating endpoint model: %s", err)
   314  	}
   315  
   316  	// Wait until the cilium-health endpoint is running before setting up routes
   317  	deadline := time.Now().Add(1 * time.Minute)
   318  	for {
   319  		if _, err := os.Stat(pidfile); err == nil {
   320  			log.WithField("pidfile", pidfile).Debug("cilium-health agent running")
   321  			break
   322  		} else if time.Now().After(deadline) {
   323  			return nil, fmt.Errorf("Endpoint failed to run: %s", err)
   324  		} else {
   325  			time.Sleep(1 * time.Second)
   326  		}
   327  	}
   328  
   329  	// Set up the endpoint routes
   330  	hostAddressing := node.GetNodeAddressing()
   331  	if err = configureHealthRouting(info.ContainerName, epIfaceName, hostAddressing, mtuConfig); err != nil {
   332  		return nil, fmt.Errorf("Error while configuring routes: %s", err)
   333  	}
   334  
   335  	if err := endpointmanager.AddEndpoint(owner, ep, "Create cilium-health endpoint"); err != nil {
   336  		return nil, fmt.Errorf("Error while adding endpoint: %s", err)
   337  	}
   338  
   339  	if err := ep.LockAlive(); err != nil {
   340  		return nil, err
   341  	}
   342  	ep.PinDatapathMap()
   343  	ep.Unlock()
   344  
   345  	// Give the endpoint a security identity
   346  	ctx, cancel := context.WithTimeout(baseCtx, LaunchTime)
   347  	defer cancel()
   348  	ep.UpdateLabels(ctx, labels.LabelHealth, nil, true)
   349  
   350  	// Initialize the health client to talk to this instance.
   351  	client := &Client{host: "http://" + net.JoinHostPort(healthIP.String(), fmt.Sprintf("%d", healthDefaults.HTTPPathPort))}
   352  	metrics.SubprocessStart.WithLabelValues(ciliumHealth).Inc()
   353  
   354  	return client, nil
   355  }