github.com/noironetworks/cilium-net@v1.6.12/cilium-health/launch/endpoint.go (about) 1 // Copyright 2017-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package launch 16 17 import ( 18 "context" 19 "fmt" 20 "net" 21 "os" 22 "os/exec" 23 "path/filepath" 24 "strings" 25 "time" 26 27 "github.com/cilium/cilium/api/v1/models" 28 "github.com/cilium/cilium/pkg/datapath/linux/route" 29 "github.com/cilium/cilium/pkg/defaults" 30 "github.com/cilium/cilium/pkg/endpoint" 31 "github.com/cilium/cilium/pkg/endpoint/connector" 32 "github.com/cilium/cilium/pkg/endpoint/regeneration" 33 "github.com/cilium/cilium/pkg/endpointmanager" 34 healthDefaults "github.com/cilium/cilium/pkg/health/defaults" 35 "github.com/cilium/cilium/pkg/health/probe" 36 "github.com/cilium/cilium/pkg/labels" 37 "github.com/cilium/cilium/pkg/launcher" 38 "github.com/cilium/cilium/pkg/logging/logfields" 39 "github.com/cilium/cilium/pkg/metrics" 40 "github.com/cilium/cilium/pkg/mtu" 41 "github.com/cilium/cilium/pkg/netns" 42 "github.com/cilium/cilium/pkg/node" 43 "github.com/cilium/cilium/pkg/option" 44 "github.com/cilium/cilium/pkg/pidfile" 45 "github.com/cilium/cilium/pkg/sysctl" 46 47 "github.com/containernetworking/plugins/pkg/ns" 48 "github.com/vishvananda/netlink" 49 "golang.org/x/sys/unix" 50 ) 51 52 const ( 53 ciliumHealth = "cilium-health" 54 netNSName = "cilium-health" 55 binaryName = "cilium-health-responder" 56 ) 57 58 var ( 59 // vethName is the host-side veth link device name for cilium-health EP 60 // (veth mode only). 61 vethName = "lxc_health" 62 63 // legacyVethName is the host-side cilium-health EP device name used in 64 // older Cilium versions. Used for removal only. 65 legacyVethName = "cilium_health" 66 67 // epIfaceName is the endpoint-side link device name for cilium-health. 68 epIfaceName = "cilium" 69 70 // PidfilePath 71 PidfilePath = "health-endpoint.pid" 72 73 // LaunchTime is the expected time within which the health endpoint 74 // should be able to be successfully run and its BPF program attached. 75 LaunchTime = 30 * time.Second 76 ) 77 78 func configureHealthRouting(netns, dev string, addressing *models.NodeAddressing, mtuConfig mtu.Configuration) error { 79 routes := []route.Route{} 80 81 if option.Config.EnableIPv4 { 82 v4Routes, err := connector.IPv4Routes(addressing, mtuConfig.GetRouteMTU()) 83 if err == nil { 84 routes = append(routes, v4Routes...) 85 } else { 86 log.Debugf("Couldn't get IPv4 routes for health routing") 87 } 88 } 89 90 if option.Config.EnableIPv6 { 91 v6Routes, err := connector.IPv6Routes(addressing, mtuConfig.GetRouteMTU()) 92 if err != nil { 93 return fmt.Errorf("Failed to get IPv6 routes") 94 } 95 routes = append(routes, v6Routes...) 96 } 97 98 prog := "ip" 99 args := []string{"netns", "exec", netns, "bash", "-c"} 100 routeCmds := []string{} 101 for _, rt := range routes { 102 cmd := strings.Join(rt.ToIPCommand(dev), " ") 103 log.WithField("netns", netns).WithField("command", cmd).Debug("Adding route") 104 routeCmds = append(routeCmds, cmd) 105 } 106 cmd := strings.Join(routeCmds, " && ") 107 args = append(args, cmd) 108 109 log.Debugf("Running \"%s %+v\"", prog, args) 110 out, err := exec.Command(prog, args...).CombinedOutput() 111 if err == nil && len(out) > 0 { 112 log.Warn(out) 113 } 114 115 return err 116 } 117 118 func configureHealthInterface(netNS ns.NetNS, ifName string, ip4Addr, ip6Addr *net.IPNet) error { 119 return netNS.Do(func(_ ns.NetNS) error { 120 link, err := netlink.LinkByName(ifName) 121 if err != nil { 122 return err 123 } 124 125 if ip6Addr == nil { 126 name := fmt.Sprintf("net.ipv6.conf.%s.disable_ipv6", ifName) 127 // Ignore the error; if IPv6 is completely disabled 128 // then it's okay if we can't write the sysctl. 129 _ = sysctl.Write(name, "1") 130 } else { 131 if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip6Addr}); err != nil { 132 return err 133 } 134 } 135 136 if ip4Addr != nil { 137 if err = netlink.AddrAdd(link, &netlink.Addr{IPNet: ip4Addr}); err != nil { 138 return err 139 } 140 } 141 142 if err = netlink.LinkSetUp(link); err != nil { 143 return err 144 } 145 146 lo, err := netlink.LinkByName("lo") 147 if err != nil { 148 return err 149 } 150 151 if err = netlink.LinkSetUp(lo); err != nil { 152 return err 153 } 154 155 return nil 156 }) 157 } 158 159 // Client wraps a client to a specific cilium-health endpoint instance, to 160 // provide convenience methods such as PingEndpoint(). 161 type Client struct { 162 host string 163 } 164 165 // PingEndpoint attempts to make an API ping request to the local cilium-health 166 // endpoint, and returns whether this was successful. 167 func (c *Client) PingEndpoint() error { 168 return probe.GetHello(c.host) 169 } 170 171 // KillEndpoint attempts to kill any existing cilium-health endpoint if it 172 // exists. 173 // 174 // This is intended to be invoked in multiple situations: 175 // * The health endpoint has never been run before 176 // * The health endpoint was run during a previous run of the Cilium agent 177 // * The health endpoint crashed during the current run of the Cilium agent 178 // and needs to be cleaned up before it is restarted. 179 func KillEndpoint() { 180 path := filepath.Join(option.Config.StateDir, PidfilePath) 181 scopedLog := log.WithField(logfields.PIDFile, path) 182 scopedLog.Debug("Killing old health endpoint process") 183 pid, err := pidfile.Kill(path) 184 if err != nil { 185 scopedLog.WithError(err).Warning("Failed to kill cilium-health-responder") 186 } else if pid != 0 { 187 scopedLog.WithField(logfields.PID, pid).Debug("Killed endpoint process") 188 } 189 } 190 191 // CleanupEndpoint cleans up remaining resources associated with the health 192 // endpoint. 193 // 194 // This is expected to be called after the process is killed and the endpoint 195 // is removed from the endpointmanager. 196 func CleanupEndpoint() { 197 // Removes the interfaces used for the endpoint process, followed by the 198 // deletion of the health namespace itself. The removal of the interfaces 199 // is needed, because network namespace removal does not always trigger the 200 // deletion of associated interfaces immediately (e.g. when a process in the 201 // namespace marked for deletion has not yet been terminated). 202 switch option.Config.DatapathMode { 203 case option.DatapathModeVeth: 204 for _, iface := range []string{legacyVethName, vethName} { 205 scopedLog := log.WithField(logfields.Veth, iface) 206 if link, err := netlink.LinkByName(iface); err == nil { 207 err = netlink.LinkDel(link) 208 if err != nil { 209 scopedLog.WithError(err).Info("Couldn't delete cilium-health veth device") 210 } 211 } else { 212 scopedLog.WithError(err).Debug("Didn't find existing device") 213 } 214 } 215 case option.DatapathModeIpvlan: 216 if err := netns.RemoveIfFromNetNSWithNameIfBothExist(netNSName, epIfaceName); err != nil { 217 log.WithError(err).WithField(logfields.Ipvlan, epIfaceName). 218 Info("Couldn't delete cilium-health ipvlan slave device") 219 } 220 } 221 222 if err := netns.RemoveNetNSWithName(netNSName); err != nil { 223 log.WithError(err).Debug("Unable to remove cilium-health namespace") 224 } 225 } 226 227 // LaunchAsEndpoint launches the cilium-health agent in a nested network 228 // namespace and attaches it to Cilium the same way as any other endpoint, 229 // but with special reserved labels. 230 // 231 // CleanupEndpoint() must be called before calling LaunchAsEndpoint() to ensure 232 // cleanup of prior cilium-health endpoint instances. 233 func LaunchAsEndpoint(baseCtx context.Context, owner regeneration.Owner, n *node.Node, mtuConfig mtu.Configuration) (*Client, error) { 234 var ( 235 cmd = launcher.Launcher{} 236 info = &models.EndpointChangeRequest{ 237 ContainerName: ciliumHealth, 238 State: models.EndpointStateWaitingForIdentity, 239 Addressing: &models.AddressPair{}, 240 } 241 healthIP net.IP 242 ip4Address, ip6Address *net.IPNet 243 ) 244 245 if n.IPv6HealthIP != nil { 246 healthIP = n.IPv6HealthIP 247 info.Addressing.IPV6 = healthIP.String() 248 ip6Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv6Mask} 249 } 250 if n.IPv4HealthIP != nil { 251 healthIP = n.IPv4HealthIP 252 info.Addressing.IPV4 = healthIP.String() 253 ip4Address = &net.IPNet{IP: healthIP, Mask: defaults.ContainerIPv4Mask} 254 } 255 256 if option.Config.EnableEndpointRoutes { 257 dpConfig := &models.EndpointDatapathConfiguration{ 258 InstallEndpointRoute: true, 259 RequireEgressProg: true, 260 } 261 info.DatapathConfiguration = dpConfig 262 } 263 264 netNS, err := netns.ReplaceNetNSWithName(netNSName) 265 if err != nil { 266 return nil, err 267 } 268 269 switch option.Config.DatapathMode { 270 case option.DatapathModeVeth: 271 _, epLink, err := connector.SetupVethWithNames(vethName, epIfaceName, mtuConfig.GetDeviceMTU(), info) 272 if err != nil { 273 return nil, fmt.Errorf("Error while creating veth: %s", err) 274 } 275 276 if err = netlink.LinkSetNsFd(*epLink, int(netNS.Fd())); err != nil { 277 return nil, fmt.Errorf("failed to move device %q to health namespace: %s", epIfaceName, err) 278 } 279 280 case option.DatapathModeIpvlan: 281 mapFD, err := connector.CreateAndSetupIpvlanSlave("", 282 epIfaceName, netNS, mtuConfig.GetDeviceMTU(), 283 option.Config.Ipvlan.MasterDeviceIndex, 284 option.Config.Ipvlan.OperationMode, info) 285 if err != nil { 286 if errDel := netns.RemoveNetNSWithName(netNSName); errDel != nil { 287 log.WithError(errDel).WithField(logfields.NetNSName, netNSName). 288 Warning("Unable to remove network namespace") 289 } 290 return nil, err 291 } 292 defer unix.Close(mapFD) 293 294 } 295 296 if err = configureHealthInterface(netNS, epIfaceName, ip4Address, ip6Address); err != nil { 297 return nil, fmt.Errorf("failed configure health interface %q: %s", epIfaceName, err) 298 } 299 300 pidfile := filepath.Join(option.Config.StateDir, PidfilePath) 301 prog := "ip" 302 args := []string{"netns", "exec", netNSName, binaryName, "--pidfile", pidfile} 303 cmd.SetTarget(prog) 304 cmd.SetArgs(args) 305 log.Infof("Spawning health endpoint with command %q %q", prog, args) 306 if err := cmd.Run(); err != nil { 307 return nil, err 308 } 309 310 // Create the endpoint 311 ep, err := endpoint.NewEndpointFromChangeModel(owner, info) 312 if err != nil { 313 return nil, fmt.Errorf("Error while creating endpoint model: %s", err) 314 } 315 316 // Wait until the cilium-health endpoint is running before setting up routes 317 deadline := time.Now().Add(1 * time.Minute) 318 for { 319 if _, err := os.Stat(pidfile); err == nil { 320 log.WithField("pidfile", pidfile).Debug("cilium-health agent running") 321 break 322 } else if time.Now().After(deadline) { 323 return nil, fmt.Errorf("Endpoint failed to run: %s", err) 324 } else { 325 time.Sleep(1 * time.Second) 326 } 327 } 328 329 // Set up the endpoint routes 330 hostAddressing := node.GetNodeAddressing() 331 if err = configureHealthRouting(info.ContainerName, epIfaceName, hostAddressing, mtuConfig); err != nil { 332 return nil, fmt.Errorf("Error while configuring routes: %s", err) 333 } 334 335 if err := endpointmanager.AddEndpoint(owner, ep, "Create cilium-health endpoint"); err != nil { 336 return nil, fmt.Errorf("Error while adding endpoint: %s", err) 337 } 338 339 if err := ep.LockAlive(); err != nil { 340 return nil, err 341 } 342 ep.PinDatapathMap() 343 ep.Unlock() 344 345 // Give the endpoint a security identity 346 ctx, cancel := context.WithTimeout(baseCtx, LaunchTime) 347 defer cancel() 348 ep.UpdateLabels(ctx, labels.LabelHealth, nil, true) 349 350 // Initialize the health client to talk to this instance. 351 client := &Client{host: "http://" + net.JoinHostPort(healthIP.String(), fmt.Sprintf("%d", healthDefaults.HTTPPathPort))} 352 metrics.SubprocessStart.WithLabelValues(ciliumHealth).Inc() 353 354 return client, nil 355 }