github.com/looshlee/cilium@v1.6.12/daemon/datapath.go (about) 1 // Copyright 2016-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "bufio" 19 "context" 20 "fmt" 21 "os" 22 "path/filepath" 23 "strings" 24 25 "github.com/vishvananda/netlink" 26 27 "github.com/cilium/cilium/common" 28 "github.com/cilium/cilium/pkg/bpf" 29 "github.com/cilium/cilium/pkg/cgroups" 30 "github.com/cilium/cilium/pkg/command/exec" 31 "github.com/cilium/cilium/pkg/datapath/alignchecker" 32 "github.com/cilium/cilium/pkg/datapath/loader" 33 "github.com/cilium/cilium/pkg/datapath/prefilter" 34 "github.com/cilium/cilium/pkg/defaults" 35 "github.com/cilium/cilium/pkg/logging/logfields" 36 "github.com/cilium/cilium/pkg/node" 37 "github.com/cilium/cilium/pkg/option" 38 ) 39 40 func (d *Daemon) compileBase() error { 41 var args []string 42 var mode string 43 var ret error 44 45 args = make([]string, initArgMax) 46 47 // Lock so that endpoints cannot be built while we are compile base programs. 48 d.compilationMutex.Lock() 49 defer d.compilationMutex.Unlock() 50 51 if err := d.writeNetdevHeader("./"); err != nil { 52 log.WithError(err).Warn("Unable to write netdev header") 53 return err 54 } 55 loader.Init(d.datapath, &d.nodeDiscovery.LocalConfig) 56 57 scopedLog := log.WithField(logfields.XDPDevice, option.Config.DevicePreFilter) 58 if option.Config.DevicePreFilter != "undefined" { 59 if err := prefilter.ProbePreFilter(option.Config.DevicePreFilter, option.Config.ModePreFilter); err != nil { 60 scopedLog.WithError(err).Warn("Turning off prefilter") 61 option.Config.DevicePreFilter = "undefined" 62 } 63 } 64 if option.Config.DevicePreFilter != "undefined" { 65 if d.preFilter, ret = prefilter.NewPreFilter(); ret != nil { 66 scopedLog.WithError(ret).Warn("Unable to init prefilter") 67 return ret 68 } 69 70 if err := d.writePreFilterHeader("./"); err != nil { 71 scopedLog.WithError(err).Warn("Unable to write prefilter header") 72 return err 73 } 74 75 args[initArgDevicePreFilter] = option.Config.DevicePreFilter 76 args[initArgModePreFilter] = option.Config.ModePreFilter 77 } 78 79 args[initArgLib] = option.Config.BpfDir 80 args[initArgRundir] = option.Config.StateDir 81 args[initArgCgroupRoot] = cgroups.GetCgroupRoot() 82 args[initArgBpffsRoot] = bpf.GetMapRoot() 83 84 if option.Config.EnableIPv4 { 85 args[initArgIPv4NodeIP] = node.GetInternalIPv4().String() 86 } else { 87 args[initArgIPv4NodeIP] = "<nil>" 88 } 89 90 if option.Config.EnableIPv6 { 91 args[initArgIPv6NodeIP] = node.GetIPv6().String() 92 } else { 93 args[initArgIPv6NodeIP] = "<nil>" 94 } 95 96 args[initArgMTU] = fmt.Sprintf("%d", d.mtuConfig.GetDeviceMTU()) 97 98 if option.Config.EnableIPSec { 99 args[initArgIPSec] = "true" 100 } else { 101 args[initArgIPSec] = "false" 102 } 103 104 if !option.Config.InstallIptRules && option.Config.Masquerade { 105 args[initArgMasquerade] = "true" 106 } else { 107 args[initArgMasquerade] = "false" 108 } 109 110 if option.Config.EnableHostReachableServices { 111 args[initArgHostReachableServices] = "true" 112 if option.Config.EnableHostServicesUDP { 113 args[initArgHostReachableServicesUDP] = "true" 114 } else { 115 args[initArgHostReachableServicesUDP] = "false" 116 } 117 } else { 118 args[initArgHostReachableServices] = "false" 119 args[initArgHostReachableServicesUDP] = "false" 120 } 121 122 if option.Config.EncryptInterface != "" { 123 args[initArgEncryptInterface] = option.Config.EncryptInterface 124 } 125 126 if option.Config.Device != "undefined" { 127 _, err := netlink.LinkByName(option.Config.Device) 128 if err != nil { 129 log.WithError(err).WithField("device", option.Config.Device).Warn("Link does not exist") 130 return err 131 } 132 133 if option.Config.IsLBEnabled() { 134 if option.Config.Device != option.Config.LBInterface { 135 //FIXME: allow different interfaces 136 return fmt.Errorf("Unable to have an interface for LB mode different than snooping interface") 137 } 138 if err := d.setHostAddresses(); err != nil { 139 return err 140 } 141 mode = "lb" 142 } else { 143 if option.Config.DatapathMode == option.DatapathModeIpvlan { 144 mode = "ipvlan" 145 } else { 146 mode = "direct" 147 } 148 } 149 150 args[initArgMode] = mode 151 if option.Config.EnableNodePort && 152 strings.ToLower(option.Config.Tunnel) != "disabled" { 153 args[initArgMode] = option.Config.Tunnel 154 } 155 args[initArgDevice] = option.Config.Device 156 } else { 157 if option.Config.IsLBEnabled() && strings.ToLower(option.Config.Tunnel) != "disabled" { 158 //FIXME: allow LBMode in tunnel 159 return fmt.Errorf("Unable to run LB mode with tunnel mode") 160 } 161 162 args[initArgMode] = option.Config.Tunnel 163 164 if option.Config.IsFlannelMasterDeviceSet() { 165 args[initArgMode] = "flannel" 166 args[initArgDevice] = option.Config.FlannelMasterDevice 167 } 168 } 169 170 if option.Config.EnableEndpointRoutes == true { 171 args[initArgMode] = "routed" 172 } 173 174 if option.Config.EnableNodePort { 175 args[initArgNodePort] = "true" 176 } 177 178 log.Info("Setting up base BPF datapath") 179 180 prog := filepath.Join(option.Config.BpfDir, "init.sh") 181 ctx, cancel := context.WithTimeout(context.Background(), defaults.ExecTimeout) 182 defer cancel() 183 cmd := exec.CommandContext(ctx, prog, args...) 184 cmd.Env = bpf.Environment() 185 if _, err := cmd.CombinedOutput(log, true); err != nil { 186 return err 187 } 188 189 if canDisableDwarfRelocations { 190 // Validate alignments of C and Go equivalent structs 191 if err := alignchecker.CheckStructAlignments(defaults.AlignCheckerName); err != nil { 192 log.WithError(err).Fatal("C and Go structs alignment check failed") 193 } 194 } else { 195 log.Warning("Cannot check matching of C and Go common struct alignments due to old LLVM/clang version") 196 } 197 198 if !option.Config.IsFlannelMasterDeviceSet() { 199 d.ipam.ReserveLocalRoutes() 200 } 201 202 if err := d.datapath.Node().NodeConfigurationChanged(d.nodeDiscovery.LocalConfig); err != nil { 203 return err 204 } 205 206 if option.Config.InstallIptRules { 207 if err := d.iptablesManager.TransientRulesStart(option.Config.HostDevice); err != nil { 208 return err 209 } 210 } 211 // Always remove masquerade rule and then re-add it if required 212 d.iptablesManager.RemoveRules() 213 if option.Config.InstallIptRules { 214 err := d.iptablesManager.InstallRules(option.Config.HostDevice) 215 d.iptablesManager.TransientRulesEnd(false) 216 if err != nil { 217 return err 218 } 219 } 220 // Reinstall proxy rules for any running proxies 221 if d.l7Proxy != nil { 222 d.l7Proxy.ReinstallRules() 223 } 224 225 log.Info("Setting sysctl net.core.bpf_jit_enable=1") 226 log.Info("Setting sysctl net.ipv4.conf.all.rp_filter=0") 227 log.Info("Setting sysctl net.ipv6.conf.all.disable_ipv6=0") 228 229 return nil 230 } 231 232 func (d *Daemon) createNodeConfigHeaderfile() error { 233 nodeConfigPath := option.Config.GetNodeConfigPath() 234 f, err := os.Create(nodeConfigPath) 235 if err != nil { 236 log.WithError(err).WithField(logfields.Path, nodeConfigPath).Fatal("Failed to create node configuration file") 237 return err 238 } 239 defer f.Close() 240 241 if err = d.datapath.WriteNodeConfig(f, &d.nodeDiscovery.LocalConfig); err != nil { 242 log.WithError(err).WithField(logfields.Path, nodeConfigPath).Fatal("Failed to write node configuration file") 243 return err 244 } 245 return nil 246 } 247 248 func deleteHostDevice() { 249 link, err := netlink.LinkByName(option.Config.HostDevice) 250 if err != nil { 251 log.WithError(err).Warningf("Unable to lookup host device %s. No old cilium_host interface exists", option.Config.HostDevice) 252 return 253 } 254 255 if err := netlink.LinkDel(link); err != nil { 256 log.WithError(err).Errorf("Unable to delete host device %s to change allocation CIDR", option.Config.HostDevice) 257 } 258 } 259 260 // listFilterIfs returns a map of interfaces based on the given filter. 261 // The filter should take a link and, if found, return the index of that 262 // interface, if not found return -1. 263 func listFilterIfs(filter func(netlink.Link) int) (map[int]netlink.Link, error) { 264 ifs, err := netlink.LinkList() 265 if err != nil { 266 return nil, err 267 } 268 vethLXCIdxs := map[int]netlink.Link{} 269 for _, intf := range ifs { 270 if idx := filter(intf); idx != -1 { 271 vethLXCIdxs[idx] = intf 272 } 273 } 274 return vethLXCIdxs, nil 275 } 276 277 // clearCiliumVeths checks all veths created by cilium and removes all that 278 // are considered a leftover from failed attempts to connect the container. 279 func (d *Daemon) clearCiliumVeths() error { 280 log.Info("Removing stale endpoint interfaces") 281 282 leftVeths, err := listFilterIfs(func(intf netlink.Link) int { 283 // Filter by veth and return the index of the interface. 284 if intf.Type() == "veth" { 285 return intf.Attrs().Index 286 } 287 return -1 288 }) 289 290 if err != nil { 291 return fmt.Errorf("unable to retrieve host network interfaces: %s", err) 292 } 293 294 for _, v := range leftVeths { 295 peerIndex := v.Attrs().ParentIndex 296 parentVeth, found := leftVeths[peerIndex] 297 if found && peerIndex != 0 && strings.HasPrefix(parentVeth.Attrs().Name, "lxc") { 298 err := netlink.LinkDel(v) 299 if err != nil { 300 log.WithError(err).Warningf("Unable to delete stale veth device %s", v.Attrs().Name) 301 } 302 } 303 } 304 return nil 305 } 306 307 // Must be called with option.Config.EnablePolicyMU locked. 308 func (d *Daemon) writePreFilterHeader(dir string) error { 309 headerPath := filepath.Join(dir, common.PreFilterHeaderFileName) 310 log.WithField(logfields.Path, headerPath).Debug("writing configuration") 311 f, err := os.Create(headerPath) 312 if err != nil { 313 return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err) 314 315 } 316 defer f.Close() 317 fw := bufio.NewWriter(f) 318 fmt.Fprint(fw, "/*\n") 319 fmt.Fprintf(fw, " * XDP device: %s\n", option.Config.DevicePreFilter) 320 fmt.Fprintf(fw, " * XDP mode: %s\n", option.Config.ModePreFilter) 321 fmt.Fprint(fw, " */\n\n") 322 d.preFilter.WriteConfig(fw) 323 return fw.Flush() 324 } 325 326 func (d *Daemon) writeNetdevHeader(dir string) error { 327 headerPath := filepath.Join(dir, common.NetdevHeaderFileName) 328 log.WithField(logfields.Path, headerPath).Debug("writing configuration") 329 330 f, err := os.Create(headerPath) 331 if err != nil { 332 return fmt.Errorf("failed to open file %s for writing: %s", headerPath, err) 333 334 } 335 defer f.Close() 336 337 if err := d.datapath.WriteNetdevConfig(f, d); err != nil { 338 return err 339 } 340 return nil 341 }