github.com/kubearmor/cilium@v1.6.12/daemon/daemon_main.go (about) 1 // Copyright 2016-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "context" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "net" 23 "os" 24 "os/exec" 25 "path" 26 "path/filepath" 27 "regexp" 28 "strings" 29 "time" 30 31 "github.com/cilium/cilium/api/v1/server" 32 "github.com/cilium/cilium/api/v1/server/restapi" 33 "github.com/cilium/cilium/common" 34 _ "github.com/cilium/cilium/pkg/alignchecker" 35 "github.com/cilium/cilium/pkg/bpf" 36 "github.com/cilium/cilium/pkg/cgroups" 37 "github.com/cilium/cilium/pkg/cleanup" 38 "github.com/cilium/cilium/pkg/components" 39 "github.com/cilium/cilium/pkg/controller" 40 "github.com/cilium/cilium/pkg/datapath/iptables" 41 linuxdatapath "github.com/cilium/cilium/pkg/datapath/linux" 42 "github.com/cilium/cilium/pkg/datapath/loader" 43 "github.com/cilium/cilium/pkg/datapath/maps" 44 "github.com/cilium/cilium/pkg/defaults" 45 "github.com/cilium/cilium/pkg/endpointmanager" 46 "github.com/cilium/cilium/pkg/envoy" 47 "github.com/cilium/cilium/pkg/flowdebug" 48 "github.com/cilium/cilium/pkg/identity" 49 "github.com/cilium/cilium/pkg/identity/cache" 50 "github.com/cilium/cilium/pkg/k8s" 51 "github.com/cilium/cilium/pkg/k8s/endpointsynchronizer" 52 "github.com/cilium/cilium/pkg/kvstore" 53 "github.com/cilium/cilium/pkg/labels" 54 "github.com/cilium/cilium/pkg/loadinfo" 55 "github.com/cilium/cilium/pkg/logging" 56 "github.com/cilium/cilium/pkg/logging/logfields" 57 ipcachemap "github.com/cilium/cilium/pkg/maps/ipcache" 58 "github.com/cilium/cilium/pkg/metrics" 59 monitorAPI "github.com/cilium/cilium/pkg/monitor/api" 60 "github.com/cilium/cilium/pkg/node" 61 "github.com/cilium/cilium/pkg/option" 62 "github.com/cilium/cilium/pkg/pidfile" 63 "github.com/cilium/cilium/pkg/policy" 64 "github.com/cilium/cilium/pkg/pprof" 65 "github.com/cilium/cilium/pkg/service" 66 "github.com/cilium/cilium/pkg/version" 67 "github.com/cilium/cilium/pkg/versioncheck" 68 "github.com/cilium/cilium/pkg/workloads" 69 70 "github.com/go-openapi/loads" 71 gops "github.com/google/gops/agent" 72 go_version "github.com/hashicorp/go-version" 73 "github.com/jessevdk/go-flags" 74 "github.com/sirupsen/logrus" 75 "github.com/spf13/cobra" 76 "github.com/spf13/viper" 77 "github.com/vishvananda/netlink" 78 "golang.org/x/sys/unix" 79 "google.golang.org/grpc" 80 ) 81 82 var ( 83 minKernelVer = versioncheck.MustCompile(">= 4.8.0") 84 minClangVer = versioncheck.MustCompile(">= 3.8.0") 85 86 recKernelVer = versioncheck.MustCompile(">= 4.9.0") 87 recClangVer = versioncheck.MustCompile(">= 3.9.0") 88 89 // LLVM/clang version which supports `-mattr=dwarfris` 90 dwarfrisClangVer = versioncheck.MustCompile(">= 7.0.0") 91 canDisableDwarfRelocations bool 92 ) 93 94 const ( 95 // list of supported verbose debug groups 96 argDebugVerboseFlow = "flow" 97 argDebugVerboseKvstore = "kvstore" 98 argDebugVerboseEnvoy = "envoy" 99 100 apiTimeout = 60 * time.Second 101 daemonSubsys = "daemon" 102 103 // fatalSleep is the duration Cilium should sleep before existing in case 104 // of a log.Fatal is issued or a CLI flag is specified but does not exist. 105 fatalSleep = 2 * time.Second 106 ) 107 108 var ( 109 log = logging.DefaultLogger.WithField(logfields.LogSubsys, daemonSubsys) 110 111 bootstrapTimestamp = time.Now() 112 113 // RootCmd represents the base command when called without any subcommands 114 RootCmd = &cobra.Command{ 115 Use: "cilium-agent", 116 Short: "Run the cilium agent", 117 Run: func(cmd *cobra.Command, args []string) { 118 bootstrapStats.earlyInit.Start() 119 initEnv(cmd) 120 bootstrapStats.earlyInit.End(true) 121 runDaemon() 122 }, 123 } 124 125 bootstrapStats = bootstrapStatistics{} 126 ) 127 128 func init() { 129 RootCmd.SetFlagErrorFunc(func(_ *cobra.Command, e error) error { 130 time.Sleep(fatalSleep) 131 return e 132 }) 133 logrus.RegisterExitHandler(func() { 134 time.Sleep(fatalSleep) 135 }, 136 ) 137 } 138 139 func daemonMain() { 140 bootstrapStats.overall.Start() 141 142 // Open socket for using gops to get stacktraces of the agent. 143 if err := gops.Listen(gops.Options{}); err != nil { 144 errorString := fmt.Sprintf("unable to start gops: %s", err) 145 fmt.Println(errorString) 146 os.Exit(-1) 147 } 148 interruptCh := registerSigHandler() 149 if err := RootCmd.Execute(); err != nil { 150 fmt.Println(err) 151 os.Exit(-1) 152 } 153 <-interruptCh 154 os.Exit(0) 155 } 156 157 func parseKernelVersion(ver string) (*go_version.Version, error) { 158 verStrs := strings.Split(ver, ".") 159 switch { 160 case len(verStrs) < 2: 161 return nil, fmt.Errorf("unable to get kernel version from %q", ver) 162 case len(verStrs) < 3: 163 verStrs = append(verStrs, "0") 164 } 165 // We are assuming the kernel version will be something as: 166 // 4.9.17-040917-generic 167 168 // If verStrs is []string{ "4", "9", "17-040917-generic" } 169 // then we need to retrieve patch number. 170 patch := regexp.MustCompilePOSIX(`^[0-9]+`).FindString(verStrs[2]) 171 if patch == "" { 172 verStrs[2] = "0" 173 } else { 174 verStrs[2] = patch 175 } 176 return go_version.NewVersion(strings.Join(verStrs[:3], ".")) 177 } 178 179 func getKernelVersion() (*go_version.Version, error) { 180 var unameBuf unix.Utsname 181 if err := unix.Uname(&unameBuf); err != nil { 182 log.WithError(err).Fatal("kernel version: NOT OK") 183 } 184 return parseKernelVersion(string(unameBuf.Release[:])) 185 } 186 187 func getClangVersion(filePath string) (*go_version.Version, error) { 188 verOut, err := exec.Command(filePath, "--version").CombinedOutput() 189 if err != nil { 190 log.WithError(err).Fatal("clang version: NOT OK") 191 } 192 res := regexp.MustCompile(`(clang version )([^ ]*)`).FindStringSubmatch(string(verOut)) 193 if len(res) != 3 { 194 log.Fatalf("clang version: NOT OK: unable to get clang's version "+ 195 "from: %q", string(verOut)) 196 } 197 // at this point res is []string{"clang", "version", "maj.min.patch"} 198 verStrs := strings.Split(res[2], ".") 199 if len(verStrs) < 3 { 200 return nil, fmt.Errorf("unable to get clang version from %q", string(verOut)) 201 } 202 v := strings.Join(verStrs[:3], ".") 203 // Handle Ubuntu versioning by removing the dash and everything after. 204 // F. ex. `4.0.0-1ubuntu1~16 -> 4.0.0` and `3.8.0-2ubuntu4 -> 3.8.0`. 205 v = strings.Split(v, "-")[0] 206 return go_version.NewVersion(v) 207 } 208 209 func checkBPFLogs(logType string, fatal bool) { 210 bpfLogFile := logType + ".log" 211 bpfLogPath := filepath.Join(option.Config.StateDir, bpfLogFile) 212 213 if _, err := os.Stat(bpfLogPath); os.IsNotExist(err) { 214 log.Infof("%s check: OK!", logType) 215 } else if err == nil { 216 bpfFeaturesLog, err := ioutil.ReadFile(bpfLogPath) 217 if err != nil { 218 log.WithError(err).WithField(logfields.Path, bpfLogPath).Fatalf("%s check: NOT OK. Unable to read", logType) 219 } 220 printer := log.Debugf 221 if fatal { 222 printer = log.Errorf 223 printer("%s check: NOT OK", logType) 224 } else { 225 printer("%s check: Some features may be limited:", logType) 226 } 227 lines := strings.Trim(string(bpfFeaturesLog), "\n") 228 for _, line := range strings.Split(lines, "\n") { 229 printer(line) 230 } 231 if fatal { 232 log.Fatalf("%s check failed.", logType) 233 } 234 } else { 235 log.WithError(err).WithField(logfields.Path, bpfLogPath).Fatalf("%s check: NOT OK. Unable to read", logType) 236 } 237 } 238 239 func checkMinRequirements() { 240 kernelVersion, err := getKernelVersion() 241 if err != nil { 242 log.WithError(err).Fatal("kernel version: NOT OK") 243 } 244 if !minKernelVer.Check(kernelVersion) { 245 log.Fatalf("kernel version: NOT OK: minimal supported kernel "+ 246 "version is %s; kernel version that is running is: %s", minKernelVer, kernelVersion) 247 } 248 249 if option.Config.EnableIPv6 { 250 if _, err := os.Stat("/proc/net/if_inet6"); os.IsNotExist(err) { 251 log.Fatalf("kernel: ipv6 is enabled in agent but ipv6 is either disabled or not compiled in the kernel") 252 } 253 } 254 255 if filePath, err := exec.LookPath("clang"); err != nil { 256 log.WithError(err).Fatal("clang: NOT OK") 257 } else { 258 clangVersion, err := getClangVersion(filePath) 259 if err != nil { 260 log.WithError(err).Fatal("clang: NOT OK") 261 } 262 if !minClangVer.Check(clangVersion) { 263 log.Fatalf("clang version: NOT OK: minimal supported clang "+ 264 "version is %s; clang version that is running is: %s", minClangVer, clangVersion) 265 } 266 //clang >= 3.9 / kernel < 4.9 - does not work 267 if recClangVer.Check(clangVersion) && !recKernelVer.Check(kernelVersion) { 268 log.Fatalf("clang (%s) and kernel (%s) version: NOT OK: please upgrade "+ 269 "your kernel version to at least %s", 270 clangVersion, kernelVersion, recKernelVer) 271 } 272 canDisableDwarfRelocations = dwarfrisClangVer.Check(clangVersion) 273 log.Infof("clang (%s) and kernel (%s) versions: OK!", clangVersion, kernelVersion) 274 } 275 276 if filePath, err := exec.LookPath("llc"); err != nil { 277 log.WithError(err).Fatal("llc: NOT OK") 278 } else { 279 lccVersion, err := exec.Command(filePath, "--version").CombinedOutput() 280 if err == nil { 281 if strings.Contains(strings.ToLower(string(lccVersion)), "debug") { 282 log.Warn("llc version was compiled in debug mode, expect higher latency!") 283 } 284 } 285 // /usr/include/gnu/stubs-32.h is installed by 'glibc-devel.i686' in fedora 286 // /usr/include/sys/cdefs.h is installed by 'libc6-dev-i386' in ubuntu 287 // both files exist on both systems but cdefs.h already exists in fedora 288 // without 'glibc-devel.i686' so we check for 'stubs-32.h first. 289 if _, err := os.Stat("/usr/include/gnu/stubs-32.h"); os.IsNotExist(err) { 290 log.Fatal("linking environment: NOT OK, please make sure you have 'glibc-devel.i686' if you use fedora system or 'libc6-dev-i386' if you use ubuntu system") 291 } 292 if _, err := os.Stat("/usr/include/sys/cdefs.h"); os.IsNotExist(err) { 293 log.Fatal("linking environment: NOT OK, please make sure you have 'libc6-dev-i386' in your ubuntu system") 294 } 295 log.Info("linking environment: OK!") 296 } 297 298 globalsDir := option.Config.GetGlobalsDir() 299 if err := os.MkdirAll(globalsDir, defaults.StateDirRights); err != nil { 300 log.WithError(err).WithField(logfields.Path, globalsDir).Fatal("Could not create runtime directory") 301 } 302 if err := os.Chdir(option.Config.LibDir); err != nil { 303 log.WithError(err).WithField(logfields.Path, option.Config.LibDir).Fatal("Could not change to runtime directory") 304 } 305 probeScript := filepath.Join(option.Config.BpfDir, "run_probes.sh") 306 if err := exec.Command(probeScript, option.Config.BpfDir, option.Config.StateDir).Run(); err != nil { 307 log.WithError(err).Fatal("BPF Verifier: NOT OK. Unable to run checker for bpf_features") 308 } 309 featuresFilePath := filepath.Join(globalsDir, "bpf_features.h") 310 if _, err := os.Stat(featuresFilePath); os.IsNotExist(err) { 311 log.WithError(err).WithField(logfields.Path, globalsDir).Fatal("BPF Verifier: NOT OK. Unable to read bpf_features.h") 312 } 313 314 checkBPFLogs("bpf_requirements", true) 315 checkBPFLogs("bpf_features", false) 316 bpf.ReadFeatureProbes(featuresFilePath) 317 } 318 319 func skipInit(basePath string) bool { 320 switch basePath { 321 case components.CiliumAgentName, components.CiliumDaemonTestName: 322 return false 323 default: 324 return true 325 } 326 } 327 328 func init() { 329 if skipInit(path.Base(os.Args[0])) { 330 log.Debug("Skipping preparation of cilium-agent environment") 331 return 332 } 333 334 cobra.OnInitialize(initConfig) 335 336 // Reset the help function to also exit, as we block elsewhere in interrupts 337 // and would not exit when called with -h. 338 oldHelpFunc := RootCmd.HelpFunc() 339 RootCmd.SetHelpFunc(func(c *cobra.Command, a []string) { 340 oldHelpFunc(c, a) 341 os.Exit(0) 342 }) 343 344 flags := RootCmd.Flags() 345 346 // Validators 347 option.Config.FixedIdentityMappingValidator = option.Validator(func(val string) (string, error) { 348 vals := strings.Split(val, "=") 349 if len(vals) != 2 { 350 return "", fmt.Errorf(`invalid fixed identity: expecting "<numeric-identity>=<identity-name>" got %q`, val) 351 } 352 ni, err := identity.ParseNumericIdentity(vals[0]) 353 if err != nil { 354 return "", fmt.Errorf(`invalid numeric identity %q: %s`, val, err) 355 } 356 if !identity.IsUserReservedIdentity(ni) { 357 return "", fmt.Errorf(`invalid numeric identity %q: valid numeric identity is between %d and %d`, 358 val, identity.UserReservedNumericIdentity.Uint32(), identity.MinimalNumericIdentity.Uint32()) 359 } 360 lblStr := vals[1] 361 lbl := labels.ParseLabel(lblStr) 362 if lbl.IsReservedSource() { 363 return "", fmt.Errorf(`invalid source %q for label: %s`, labels.LabelSourceReserved, lblStr) 364 } 365 return val, nil 366 }) 367 368 // Env bindings 369 flags.String(option.AccessLog, "", "Path to access log of supported L7 requests observed") 370 option.BindEnv(option.AccessLog) 371 372 flags.StringSlice(option.AgentLabels, []string{}, "Additional labels to identify this agent") 373 option.BindEnv(option.AgentLabels) 374 375 flags.String(option.AllowLocalhost, option.AllowLocalhostAuto, "Policy when to allow local stack to reach local endpoints { auto | always | policy }") 376 option.BindEnv(option.AllowLocalhost) 377 378 flags.Bool(option.AnnotateK8sNode, defaults.AnnotateK8sNode, "Annotate Kubernetes node") 379 option.BindEnv(option.AnnotateK8sNode) 380 381 flags.Bool(option.BlacklistConflictingRoutes, defaults.BlacklistConflictingRoutes, "Don't blacklist IP allocations conflicting with local non-cilium routes") 382 option.BindEnv(option.BlacklistConflictingRoutes) 383 384 flags.Bool(option.AutoCreateCiliumNodeResource, defaults.AutoCreateCiliumNodeResource, "Automatically create CiliumNode resource for own node on startup") 385 option.BindEnv(option.AutoCreateCiliumNodeResource) 386 387 flags.String(option.BPFRoot, "", "Path to BPF filesystem") 388 option.BindEnv(option.BPFRoot) 389 390 flags.String(option.CGroupRoot, "", "Path to Cgroup2 filesystem") 391 option.BindEnv(option.CGroupRoot) 392 393 flags.Bool(option.BPFCompileDebugName, false, "Enable debugging of the BPF compilation process") 394 option.BindEnv(option.BPFCompileDebugName) 395 396 flags.Bool(option.SockopsEnableName, defaults.SockopsEnable, "Enable sockops when kernel supported") 397 option.BindEnv(option.SockopsEnableName) 398 399 flags.Int(option.ClusterIDName, 0, "Unique identifier of the cluster") 400 option.BindEnv(option.ClusterIDName) 401 402 flags.String(option.ClusterName, defaults.ClusterName, "Name of the cluster") 403 option.BindEnv(option.ClusterName) 404 405 flags.String(option.ClusterMeshConfigName, "", "Path to the ClusterMesh configuration directory") 406 option.BindEnv(option.ClusterMeshConfigName) 407 408 flags.String(option.ConfigFile, "", `Configuration file (default "$HOME/ciliumd.yaml")`) 409 option.BindEnv(option.ConfigFile) 410 411 flags.String(option.ConfigDir, "", `Configuration directory that contains a file for each option`) 412 option.BindEnv(option.ConfigDir) 413 414 flags.Uint(option.ConntrackGarbageCollectorIntervalDeprecated, 0, "Garbage collection interval for the connection tracking table (in seconds)") 415 flags.MarkDeprecated(option.ConntrackGarbageCollectorIntervalDeprecated, fmt.Sprintf("please use --%s", option.ConntrackGCInterval)) 416 option.BindEnv(option.ConntrackGarbageCollectorIntervalDeprecated) 417 418 flags.Duration(option.ConntrackGCInterval, time.Duration(0), "Overwrite the connection-tracking garbage collection interval") 419 option.BindEnv(option.ConntrackGCInterval) 420 421 flags.StringSlice(option.ContainerRuntime, option.ContainerRuntimeAuto, `Sets the container runtime(s) used by Cilium { containerd | crio | docker | none | auto } ( "auto" uses the container runtime found in the order: "docker", "containerd", "crio" )`) 422 option.BindEnv(option.ContainerRuntime) 423 424 flags.Var(option.NewNamedMapOptions(option.ContainerRuntimeEndpoint, &option.Config.ContainerRuntimeEndpoint, nil), 425 option.ContainerRuntimeEndpoint, `Container runtime(s) endpoint(s). (default: `+workloads.GetDefaultEPOptsStringWithPrefix("--container-runtime-endpoint=")+`)`) 426 option.BindEnv(option.ContainerRuntimeEndpoint) 427 428 flags.BoolP(option.DebugArg, "D", false, "Enable debugging mode") 429 option.BindEnv(option.DebugArg) 430 431 flags.StringSlice(option.DebugVerbose, []string{}, "List of enabled verbose debug groups") 432 option.BindEnv(option.DebugVerbose) 433 434 flags.StringP(option.Device, "d", "undefined", "Device facing cluster/external network for direct L3 (non-overlay mode)") 435 option.BindEnv(option.Device) 436 437 flags.String(option.DatapathMode, defaults.DatapathMode, "Datapath mode name") 438 option.BindEnv(option.DatapathMode) 439 440 flags.StringP(option.IpvlanMasterDevice, "", "undefined", "Device facing external network acting as ipvlan master") 441 option.BindEnv(option.IpvlanMasterDevice) 442 443 flags.Bool(option.DisableConntrack, false, "Disable connection tracking") 444 option.BindEnv(option.DisableConntrack) 445 446 flags.Bool(option.LegacyDisableIPv4Name, false, "Disable IPv4 mode") 447 flags.MarkHidden(option.LegacyDisableIPv4Name) 448 option.BindEnv(option.LegacyDisableIPv4Name) 449 450 flags.Bool(option.EnableEndpointRoutes, defaults.EnableEndpointRoutes, "Use per endpoint routes instead of routing via cilium_host") 451 option.BindEnv(option.EnableEndpointRoutes) 452 453 flags.Bool(option.EnableHealthChecking, defaults.EnableHealthChecking, "Enable connectivity health checking") 454 option.BindEnv(option.EnableHealthChecking) 455 456 flags.Bool(option.EnableEndpointHealthChecking, defaults.EnableEndpointHealthChecking, "Enable connectivity health checking between virtual endpoints") 457 option.BindEnv(option.EnableEndpointHealthChecking) 458 459 flags.Bool(option.EnableIPv4Name, defaults.EnableIPv4, "Enable IPv4 support") 460 option.BindEnv(option.EnableIPv4Name) 461 462 flags.Bool(option.EnableIPv6Name, defaults.EnableIPv6, "Enable IPv6 support") 463 option.BindEnv(option.EnableIPv6Name) 464 465 flags.String(option.EncryptInterface, "", "Transparent encryption interface") 466 option.BindEnv(option.EncryptInterface) 467 468 flags.Bool(option.EncryptNode, defaults.EncryptNode, "Enables encrypting traffic from non-Cilium pods and host networking") 469 option.BindEnv(option.EncryptNode) 470 471 flags.StringSlice(option.IPv4PodSubnets, []string{}, "List of IPv4 pod subnets to preconfigure for encryption") 472 option.BindEnv(option.IPv4PodSubnets) 473 474 flags.StringSlice(option.IPv6PodSubnets, []string{}, "List of IPv6 pod subnets to preconfigure for encryption") 475 option.BindEnv(option.IPv6PodSubnets) 476 477 flags.String(option.EndpointInterfaceNamePrefix, defaults.EndpointInterfaceNamePrefix, "Prefix of interface name shared by all endpoints") 478 option.BindEnv(option.EndpointInterfaceNamePrefix) 479 480 flags.StringSlice(option.ExcludeLocalAddress, []string{}, "Exclude CIDR from being recognized as local address") 481 option.BindEnv(option.ExcludeLocalAddress) 482 483 flags.Bool(option.DisableCiliumEndpointCRDName, false, "Disable use of CiliumEndpoint CRD") 484 option.BindEnv(option.DisableCiliumEndpointCRDName) 485 486 flags.Bool(option.DisableK8sServices, false, "Disable east-west K8s load balancing by cilium") 487 option.BindEnv(option.DisableK8sServices) 488 489 flags.String(option.EgressMasqueradeInterfaces, "", "Limit egress masquerading to interface selector") 490 option.BindEnv(option.EgressMasqueradeInterfaces) 491 492 flags.Bool(option.EnableHostReachableServices, false, "Enable reachability of services for host applications (beta)") 493 option.BindEnv(option.EnableHostReachableServices) 494 495 flags.StringSlice(option.HostReachableServicesProtos, []string{option.HostServicesTCP, option.HostServicesUDP}, "Only enable reachability of services for host applications for specific protocols") 496 option.BindEnv(option.HostReachableServicesProtos) 497 498 flags.Bool(option.DeprecatedEnableLegacyServices, false, "Enable legacy (prior-v1.5) services") 499 flags.MarkDeprecated(option.DeprecatedEnableLegacyServices, "this option is deprecated as of v1.6") 500 option.BindEnv(option.DeprecatedEnableLegacyServices) 501 502 flags.StringP(option.Docker, "e", workloads.GetRuntimeDefaultOpt(workloads.Docker, "endpoint"), "Path to docker runtime socket (DEPRECATED: use container-runtime-endpoint instead)") 503 option.BindEnv(option.Docker) 504 505 flags.Bool(option.EnableAutoDirectRoutingName, defaults.EnableAutoDirectRouting, "Enable automatic L2 routing between nodes") 506 option.BindEnv(option.EnableAutoDirectRoutingName) 507 508 flags.Bool(option.EnableXTSocketFallbackName, defaults.EnableXTSocketFallback, "Enable fallback for missing xt_socket module") 509 option.BindEnv(option.EnableXTSocketFallbackName) 510 511 flags.String(option.EnablePolicy, option.DefaultEnforcement, "Enable policy enforcement") 512 option.BindEnv(option.EnablePolicy) 513 514 flags.Bool(option.EnableL7Proxy, defaults.EnableL7Proxy, "Enable L7 proxy for L7 policy enforcement") 515 option.BindEnv(option.EnableL7Proxy) 516 517 flags.Bool(option.EnableTracing, false, "Enable tracing while determining policy (debugging)") 518 option.BindEnv(option.EnableTracing) 519 520 flags.String(option.EnvoyLog, "", "Path to a separate Envoy log file, if any") 521 option.BindEnv(option.EnvoyLog) 522 523 flags.Bool(option.EnableIPSecName, defaults.EnableIPSec, "Enable IPSec support") 524 option.BindEnv(option.EnableIPSecName) 525 526 flags.String(option.IPSecKeyFileName, "", "Path to IPSec key file") 527 option.BindEnv(option.IPSecKeyFileName) 528 529 flags.Bool(option.ForceLocalPolicyEvalAtSource, defaults.ForceLocalPolicyEvalAtSource, "Force policy evaluation of all local communication at the source endpoint") 530 option.BindEnv(option.ForceLocalPolicyEvalAtSource) 531 532 flags.String(option.HTTP403Message, "", "Message returned in proxy L7 403 body") 533 flags.MarkHidden(option.HTTP403Message) 534 option.BindEnv(option.HTTP403Message) 535 536 flags.Uint(option.HTTPRequestTimeout, 60*60, "Time after which a forwarded HTTP request is considered failed unless completed (in seconds); Use 0 for unlimited") 537 option.BindEnv(option.HTTPRequestTimeout) 538 539 flags.Uint(option.HTTPIdleTimeout, 0, "Time after which a non-gRPC HTTP stream is considered failed unless traffic in the stream has been processed (in seconds); defaults to 0 (unlimited)") 540 option.BindEnv(option.HTTPIdleTimeout) 541 542 flags.Uint(option.HTTPMaxGRPCTimeout, 0, "Time after which a forwarded gRPC request is considered failed unless completed (in seconds). A \"grpc-timeout\" header may override this with a shorter value; defaults to 0 (unlimited)") 543 option.BindEnv(option.HTTPMaxGRPCTimeout) 544 545 flags.Uint(option.HTTPRetryCount, 3, "Number of retries performed after a forwarded request attempt fails") 546 option.BindEnv(option.HTTPRetryCount) 547 548 flags.Uint(option.HTTPRetryTimeout, 0, "Time after which a forwarded but uncompleted request is retried (connection failures are retried immediately); defaults to 0 (never)") 549 option.BindEnv(option.HTTPRetryTimeout) 550 551 flags.Uint(option.ProxyConnectTimeout, 1, "Time after which a TCP connect attempt is considered failed unless completed (in seconds)") 552 option.BindEnv(option.ProxyConnectTimeout) 553 554 flags.Bool(option.DisableEnvoyVersionCheck, false, "Do not perform Envoy binary version check on startup") 555 flags.MarkHidden(option.DisableEnvoyVersionCheck) 556 // Disable version check if Envoy build is disabled 557 option.BindEnvWithLegacyEnvFallback(option.DisableEnvoyVersionCheck, "CILIUM_DISABLE_ENVOY_BUILD") 558 559 flags.Var(option.NewNamedMapOptions(option.FixedIdentityMapping, &option.Config.FixedIdentityMapping, option.Config.FixedIdentityMappingValidator), 560 option.FixedIdentityMapping, "Key-value for the fixed identity mapping which allows to use reserved label for fixed identities") 561 option.BindEnv(option.FixedIdentityMapping) 562 563 flags.Duration(option.IdentityChangeGracePeriod, defaults.IdentityChangeGracePeriod, "Time to wait before using new identity on endpoint identity change") 564 option.BindEnv(option.IdentityChangeGracePeriod) 565 566 flags.String(option.IdentityAllocationMode, option.IdentityAllocationModeKVstore, "Method to use for identity allocation") 567 option.BindEnv(option.IdentityAllocationMode) 568 569 flags.String(option.IPAM, "", "Backend to use for IPAM") 570 option.BindEnv(option.IPAM) 571 572 flags.Int(option.IPv4ClusterCIDRMaskSize, 8, "Mask size for the cluster wide CIDR") 573 option.BindEnv(option.IPv4ClusterCIDRMaskSize) 574 575 flags.String(option.IPv4Range, AutoCIDR, "Per-node IPv4 endpoint prefix, e.g. 10.16.0.0/16") 576 option.BindEnv(option.IPv4Range) 577 578 flags.String(option.IPv6Range, AutoCIDR, "Per-node IPv6 endpoint prefix, must be /96, e.g. fd02:1:1::/96") 579 option.BindEnv(option.IPv6Range) 580 581 flags.String(option.IPv6ClusterAllocCIDRName, defaults.IPv6ClusterAllocCIDR, "IPv6 /64 CIDR used to allocate per node endpoint /96 CIDR") 582 option.BindEnv(option.IPv6ClusterAllocCIDRName) 583 584 flags.String(option.IPv4ServiceRange, AutoCIDR, "Kubernetes IPv4 services CIDR if not inside cluster prefix") 585 option.BindEnv(option.IPv4ServiceRange) 586 587 flags.String(option.IPv6ServiceRange, AutoCIDR, "Kubernetes IPv6 services CIDR if not inside cluster prefix") 588 option.BindEnv(option.IPv6ServiceRange) 589 590 flags.Bool(option.K8sEventHandover, defaults.K8sEventHandover, "Enable k8s event handover to kvstore for improved scalability") 591 option.BindEnv(option.K8sEventHandover) 592 593 flags.String(option.K8sAPIServer, "", "Kubernetes api address server (for https use --k8s-kubeconfig-path instead)") 594 option.BindEnv(option.K8sAPIServer) 595 596 flags.String(option.K8sKubeConfigPath, "", "Absolute path of the kubernetes kubeconfig file") 597 option.BindEnv(option.K8sKubeConfigPath) 598 599 flags.String(option.K8sNamespaceName, "", "Name of the Kubernetes namespace in which Cilium is deployed in") 600 flags.MarkHidden(option.K8sNamespaceName) 601 option.BindEnv(option.K8sNamespaceName) 602 603 flags.Bool(option.K8sRequireIPv4PodCIDRName, false, "Require IPv4 PodCIDR to be specified in node resource") 604 option.BindEnv(option.K8sRequireIPv4PodCIDRName) 605 606 flags.Bool(option.K8sRequireIPv6PodCIDRName, false, "Require IPv6 PodCIDR to be specified in node resource") 607 option.BindEnv(option.K8sRequireIPv6PodCIDRName) 608 609 flags.Uint(option.K8sServiceCacheSize, defaults.K8sServiceCacheSize, "Cilium service cache size for kubernetes") 610 option.BindEnv(option.K8sServiceCacheSize) 611 flags.MarkHidden(option.K8sServiceCacheSize) 612 613 flags.Bool(option.K8sForceJSONPatch, false, "When set uses JSON Patch to update CNP and CEP status in kube-apiserver") 614 option.BindEnv(option.K8sForceJSONPatch) 615 flags.MarkHidden(option.K8sForceJSONPatch) 616 617 flags.String(option.K8sWatcherEndpointSelector, defaults.K8sWatcherEndpointSelector, "K8s endpoint watcher will watch for these k8s endpoints") 618 option.BindEnv(option.K8sWatcherEndpointSelector) 619 620 flags.Bool(option.KeepConfig, false, "When restoring state, keeps containers' configuration in place") 621 option.BindEnv(option.KeepConfig) 622 623 flags.Bool(option.KeepBPFTemplates, false, "Do not restore BPF template files from binary") 624 option.BindEnv(option.KeepBPFTemplates) 625 626 flags.String(option.KVStore, "", "Key-value store type") 627 option.BindEnv(option.KVStore) 628 629 flags.Duration(option.KVstoreLeaseTTL, defaults.KVstoreLeaseTTL, "Time-to-live for the KVstore lease.") 630 flags.MarkHidden(option.KVstoreLeaseTTL) 631 option.BindEnv(option.KVstoreLeaseTTL) 632 633 flags.Duration(option.KVstorePeriodicSync, defaults.KVstorePeriodicSync, "Periodic KVstore synchronization interval") 634 option.BindEnv(option.KVstorePeriodicSync) 635 636 flags.Duration(option.KVstoreConnectivityTimeout, defaults.KVstoreConnectivityTimeout, "Time after which an incomplete kvstore operation is considered failed") 637 option.BindEnv(option.KVstoreConnectivityTimeout) 638 639 flags.Duration(option.IPAllocationTimeout, defaults.IPAllocationTimeout, "Time after which an incomplete CIDR allocation is considered failed") 640 option.BindEnv(option.IPAllocationTimeout) 641 642 flags.Var(option.NewNamedMapOptions(option.KVStoreOpt, &option.Config.KVStoreOpt, nil), 643 option.KVStoreOpt, "Key-value store options") 644 option.BindEnv(option.KVStoreOpt) 645 646 flags.Duration(option.K8sSyncTimeoutName, defaults.K8sSyncTimeout, "Timeout for synchronizing k8s resources before exiting") 647 flags.MarkHidden(option.K8sSyncTimeoutName) 648 option.BindEnv(option.K8sSyncTimeoutName) 649 650 flags.Uint(option.K8sWatcherQueueSize, 1024, "Queue size used to serialize each k8s event type") 651 option.BindEnv(option.K8sWatcherQueueSize) 652 653 flags.String(option.LabelPrefixFile, "", "Valid label prefixes file path") 654 option.BindEnv(option.LabelPrefixFile) 655 656 flags.StringSlice(option.Labels, []string{}, "List of label prefixes used to determine identity of an endpoint") 657 option.BindEnv(option.Labels) 658 659 flags.String(option.LB, "", "Enables load balancer mode where load balancer bpf program is attached to the given interface") 660 option.BindEnv(option.LB) 661 662 flags.Bool(option.EnableNodePort, false, "Enable NodePort type services by Cilium (beta)") 663 option.BindEnv(option.EnableNodePort) 664 665 flags.StringSlice(option.NodePortRange, []string{fmt.Sprintf("%d", option.NodePortMinDefault), fmt.Sprintf("%d", option.NodePortMaxDefault)}, fmt.Sprintf("Set the min/max NodePort port range")) 666 option.BindEnv(option.NodePortRange) 667 668 flags.String(option.LibDir, defaults.LibraryPath, "Directory path to store runtime build environment") 669 option.BindEnv(option.LibDir) 670 671 flags.StringSlice(option.LogDriver, []string{}, "Logging endpoints to use for example syslog") 672 option.BindEnv(option.LogDriver) 673 674 flags.Var(option.NewNamedMapOptions(option.LogOpt, &option.Config.LogOpt, nil), 675 option.LogOpt, "Log driver options for cilium") 676 option.BindEnv(option.LogOpt) 677 678 flags.Bool(option.LogSystemLoadConfigName, false, "Enable periodic logging of system load") 679 option.BindEnv(option.LogSystemLoadConfigName) 680 681 flags.String(option.LoopbackIPv4, defaults.LoopbackIPv4, "IPv4 address for service loopback SNAT") 682 option.BindEnv(option.LoopbackIPv4) 683 684 flags.String(option.NAT46Range, defaults.DefaultNAT46Prefix, "IPv6 prefix to map IPv4 addresses to") 685 option.BindEnv(option.NAT46Range) 686 687 flags.Bool(option.Masquerade, true, "Masquerade packets from endpoints leaving the host") 688 option.BindEnv(option.Masquerade) 689 690 flags.Bool(option.InstallIptRules, true, "Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading)") 691 option.BindEnv(option.InstallIptRules) 692 693 flags.Duration(option.IPTablesLockTimeout, 5*time.Second, "Time to pass to each iptables invocation to wait for xtables lock acquisition") 694 option.BindEnv(option.IPTablesLockTimeout) 695 696 flags.Int(option.MaxCtrlIntervalName, 0, "Maximum interval (in seconds) between controller runs. Zero is no limit.") 697 flags.MarkHidden(option.MaxCtrlIntervalName) 698 option.BindEnv(option.MaxCtrlIntervalName) 699 700 flags.StringSlice(option.Metrics, []string{}, "Metrics that should be enabled or disabled from the default metric list. (+metric_foo to enable metric_foo , -metric_bar to disable metric_bar)") 701 option.BindEnv(option.Metrics) 702 703 flags.String(option.MonitorAggregationName, "None", 704 "Level of monitor aggregation for traces from the datapath") 705 option.BindEnvWithLegacyEnvFallback(option.MonitorAggregationName, "CILIUM_MONITOR_AGGREGATION_LEVEL") 706 707 flags.Int(option.MonitorQueueSizeName, 0, "Size of the event queue when reading monitor events") 708 option.BindEnv(option.MonitorQueueSizeName) 709 710 flags.Int(option.MTUName, 0, "Overwrite auto-detected MTU of underlying network") 711 option.BindEnv(option.MTUName) 712 713 flags.Bool(option.PrependIptablesChainsName, true, "Prepend custom iptables chains instead of appending") 714 option.BindEnvWithLegacyEnvFallback(option.PrependIptablesChainsName, "CILIUM_PREPEND_IPTABLES_CHAIN") 715 716 flags.String(option.IPv6NodeAddr, "auto", "IPv6 address of node") 717 option.BindEnv(option.IPv6NodeAddr) 718 719 flags.String(option.IPv4NodeAddr, "auto", "IPv4 address of node") 720 option.BindEnv(option.IPv4NodeAddr) 721 722 flags.String(option.ReadCNIConfiguration, "", "Read to the CNI configuration at specified path to extract per node configuration") 723 option.BindEnv(option.ReadCNIConfiguration) 724 725 flags.Bool(option.Restore, true, "Restores state, if possible, from previous daemon") 726 option.BindEnv(option.Restore) 727 728 flags.Bool(option.SidecarHTTPProxy, false, "Disable host HTTP proxy, assuming proxies in sidecar containers") 729 flags.MarkHidden(option.SidecarHTTPProxy) 730 option.BindEnv(option.SidecarHTTPProxy) 731 732 flags.String(option.SidecarIstioProxyImage, k8s.DefaultSidecarIstioProxyImageRegexp, 733 "Regular expression matching compatible Istio sidecar istio-proxy container image names") 734 option.BindEnv(option.SidecarIstioProxyImage) 735 736 flags.Bool(option.SingleClusterRouteName, false, 737 "Use a single cluster route instead of per node routes") 738 option.BindEnv(option.SingleClusterRouteName) 739 740 flags.String(option.SocketPath, defaults.SockPath, "Sets daemon's socket path to listen for connections") 741 option.BindEnv(option.SocketPath) 742 743 flags.String(option.StateDir, defaults.RuntimePath, "Directory path to store runtime state") 744 option.BindEnv(option.StateDir) 745 746 flags.StringP(option.TunnelName, "t", "", fmt.Sprintf("Tunnel mode {%s} (default \"vxlan\" for the \"veth\" datapath mode)", option.GetTunnelModes())) 747 option.BindEnv(option.TunnelName) 748 749 flags.Int(option.TracePayloadlen, 128, "Length of payload to capture when tracing") 750 option.BindEnv(option.TracePayloadlen) 751 752 flags.Bool(option.Version, false, "Print version information") 753 option.BindEnv(option.Version) 754 755 flags.String(option.FlannelMasterDevice, "", 756 "Installs a BPF program to allow for policy enforcement in the given network interface. "+ 757 "Allows to run Cilium on top of other CNI plugins that provide networking, "+ 758 "e.g. flannel, where for flannel, this value should be set with 'cni0'. [EXPERIMENTAL]") 759 option.BindEnv(option.FlannelMasterDevice) 760 761 flags.Bool(option.FlannelUninstallOnExit, false, fmt.Sprintf("When used along the %s "+ 762 "flag, it cleans up all BPF programs installed when Cilium agent is terminated.", option.FlannelMasterDevice)) 763 option.BindEnv(option.FlannelUninstallOnExit) 764 765 flags.Bool(option.FlannelManageExistingContainers, false, 766 fmt.Sprintf("Installs a BPF program to allow for policy enforcement in already running containers managed by Flannel."+ 767 " Require Cilium to be running in the hostPID.")) 768 option.BindEnv(option.FlannelManageExistingContainers) 769 770 flags.Bool(option.PProf, false, "Enable serving the pprof debugging API") 771 option.BindEnv(option.PProf) 772 773 flags.String(option.PrefilterDevice, "undefined", "Device facing external network for XDP prefiltering") 774 option.BindEnv(option.PrefilterDevice) 775 776 flags.String(option.PrefilterMode, option.ModePreFilterNative, "Prefilter mode { "+option.ModePreFilterNative+" | "+option.ModePreFilterGeneric+" } (default: "+option.ModePreFilterNative+")") 777 option.BindEnv(option.PrefilterMode) 778 779 flags.Bool(option.PreAllocateMapsName, defaults.PreAllocateMaps, "Enable BPF map pre-allocation") 780 option.BindEnv(option.PreAllocateMapsName) 781 782 // We expect only one of the possible variables to be filled. The evaluation order is: 783 // --prometheus-serve-addr, CILIUM_PROMETHEUS_SERVE_ADDR, then PROMETHEUS_SERVE_ADDR 784 // The second environment variable (without the CILIUM_ prefix) is here to 785 // handle the case where someone uses a new image with an older spec, and the 786 // older spec used the older variable name. 787 flags.String(option.PrometheusServeAddr, "", "IP:Port on which to serve prometheus metrics (pass \":Port\" to bind on all interfaces, \"\" is off)") 788 option.BindEnvWithLegacyEnvFallback(option.PrometheusServeAddr, "PROMETHEUS_SERVE_ADDR") 789 790 flags.Int(option.CTMapEntriesGlobalTCPName, option.CTMapEntriesGlobalTCPDefault, "Maximum number of entries in TCP CT table") 791 option.BindEnvWithLegacyEnvFallback(option.CTMapEntriesGlobalTCPName, "CILIUM_GLOBAL_CT_MAX_TCP") 792 793 flags.Int(option.CTMapEntriesGlobalAnyName, option.CTMapEntriesGlobalAnyDefault, "Maximum number of entries in non-TCP CT table") 794 option.BindEnvWithLegacyEnvFallback(option.CTMapEntriesGlobalAnyName, "CILIUM_GLOBAL_CT_MAX_ANY") 795 796 flags.Duration(option.CTMapEntriesTimeoutTCPName, 21600*time.Second, "Timeout for established entries in TCP CT table") 797 option.BindEnv(option.CTMapEntriesTimeoutTCPName) 798 799 flags.Duration(option.CTMapEntriesTimeoutAnyName, 60*time.Second, "Timeout for entries in non-TCP CT table") 800 option.BindEnv(option.CTMapEntriesTimeoutAnyName) 801 802 flags.Duration(option.CTMapEntriesTimeoutSVCTCPName, 21600*time.Second, "Timeout for established service entries in TCP CT table") 803 option.BindEnv(option.CTMapEntriesTimeoutSVCTCPName) 804 805 flags.Duration(option.CTMapEntriesTimeoutSVCAnyName, 60*time.Second, "Timeout for service entries in non-TCP CT table") 806 option.BindEnv(option.CTMapEntriesTimeoutSVCAnyName) 807 808 flags.Duration(option.CTMapEntriesTimeoutSYNName, 60*time.Second, "Establishment timeout for entries in TCP CT table") 809 option.BindEnv(option.CTMapEntriesTimeoutSYNName) 810 811 flags.Duration(option.CTMapEntriesTimeoutFINName, 10*time.Second, "Teardown timeout for entries in TCP CT table") 812 option.BindEnv(option.CTMapEntriesTimeoutFINName) 813 814 flags.Int(option.NATMapEntriesGlobalName, option.NATMapEntriesGlobalDefault, "Maximum number of entries for the global BPF NAT table") 815 option.BindEnv(option.NATMapEntriesGlobalName) 816 817 flags.Int(option.PolicyMapEntriesName, defaults.PolicyMapEntries, "Maximum number of entries in endpoint policy map (per endpoint)") 818 option.BindEnv(option.PolicyMapEntriesName) 819 820 flags.String(option.CMDRef, "", "Path to cmdref output directory") 821 flags.MarkHidden(option.CMDRef) 822 option.BindEnv(option.CMDRef) 823 824 flags.Int(option.ToFQDNsMinTTL, 0, fmt.Sprintf("The minimum time, in seconds, to use DNS data for toFQDNs policies. (default %d when --tofqdns-enable-poller, %d otherwise)", defaults.ToFQDNsMinTTLPoller, defaults.ToFQDNsMinTTL)) 825 option.BindEnv(option.ToFQDNsMinTTL) 826 827 flags.Int(option.ToFQDNsProxyPort, 0, "Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port.") 828 option.BindEnv(option.ToFQDNsProxyPort) 829 830 flags.Bool(option.ToFQDNsEnablePoller, false, "Enable proactive polling of DNS names in toFQDNs.matchName rules.") 831 option.BindEnv(option.ToFQDNsEnablePoller) 832 833 flags.Bool(option.ToFQDNsEnablePollerEvents, true, "Emit DNS responses seen by the DNS poller as Monitor events, if the poller is enabled.") 834 option.BindEnv(option.ToFQDNsEnablePollerEvents) 835 836 flags.StringVar(&option.Config.FQDNRejectResponse, option.FQDNRejectResponseCode, option.FQDNProxyDenyWithRefused, fmt.Sprintf("DNS response code for rejecting DNS requests, available options are '%v'", option.FQDNRejectOptions)) 837 option.BindEnv(option.FQDNRejectResponseCode) 838 839 flags.Int(option.ToFQDNsMaxIPsPerHost, defaults.ToFQDNsMaxIPsPerHost, "Maximum number of IPs to maintain per FQDN name for each endpoint") 840 option.BindEnv(option.ToFQDNsMaxIPsPerHost) 841 842 flags.DurationVar(&option.Config.FQDNProxyResponseMaxDelay, option.FQDNProxyResponseMaxDelay, 50*time.Millisecond, "The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information.") 843 option.BindEnv(option.FQDNProxyResponseMaxDelay) 844 845 flags.String(option.ToFQDNsPreCache, defaults.ToFQDNsPreCache, "DNS cache data at this path is preloaded on agent startup") 846 option.BindEnv(option.ToFQDNsPreCache) 847 848 flags.Int(option.PolicyQueueSize, defaults.PolicyQueueSize, "size of queues for policy-related events") 849 option.BindEnv(option.PolicyQueueSize) 850 851 flags.Int(option.EndpointQueueSize, defaults.EndpointQueueSize, "size of EventQueue per-endpoint") 852 option.BindEnv(option.EndpointQueueSize) 853 854 flags.Bool(option.SelectiveRegeneration, true, "only regenerate endpoints which need to be regenerated upon policy changes") 855 flags.MarkHidden(option.SelectiveRegeneration) 856 option.BindEnv(option.SelectiveRegeneration) 857 858 flags.Bool(option.SkipCRDCreation, false, "Skip Kubernetes Custom Resource Definitions creations") 859 option.BindEnv(option.SkipCRDCreation) 860 861 flags.String(option.WriteCNIConfigurationWhenReady, "", fmt.Sprintf("Write the CNI configuration as specified via --%s to path when agent is ready", option.ReadCNIConfiguration)) 862 option.BindEnv(option.WriteCNIConfigurationWhenReady) 863 864 flags.Duration(option.PolicyTriggerInterval, defaults.PolicyTriggerInterval, "Time between triggers of policy updates (regenerations for all endpoints)") 865 flags.MarkHidden(option.PolicyTriggerInterval) 866 option.BindEnv(option.PolicyTriggerInterval) 867 868 flags.Bool(option.DisableCNPStatusUpdates, false, "Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with `cnp-node-status-gc=false` in cilium-operator)") 869 option.BindEnv(option.DisableCNPStatusUpdates) 870 871 viper.BindPFlags(flags) 872 } 873 874 // RestoreExecPermissions restores file permissions to 0740 of all files inside 875 // `searchDir` with the given regex `patterns`. 876 func RestoreExecPermissions(searchDir string, patterns ...string) error { 877 fileList := []string{} 878 err := filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error { 879 for _, pattern := range patterns { 880 if regexp.MustCompile(pattern).MatchString(f.Name()) { 881 fileList = append(fileList, path) 882 break 883 } 884 } 885 return nil 886 }) 887 for _, fileToChange := range fileList { 888 // Changing files permissions to -rwx:r--:---, we are only 889 // adding executable permission to the owner and keeping the 890 // same permissions stored by go-bindata. 891 if err := os.Chmod(fileToChange, os.FileMode(0740)); err != nil { 892 return err 893 } 894 } 895 return err 896 } 897 898 // initConfig reads in config file and ENV variables if set. 899 func initConfig() { 900 if viper.GetBool("version") { 901 fmt.Printf("Cilium %s\n", version.Version) 902 os.Exit(0) 903 } 904 905 if option.Config.CMDRefDir != "" { 906 return 907 } 908 909 option.Config.ConfigFile = viper.GetString(option.ConfigFile) // enable ability to specify config file via flag 910 option.Config.ConfigDir = viper.GetString(option.ConfigDir) 911 viper.SetEnvPrefix("cilium") 912 913 if option.Config.ConfigDir != "" { 914 if _, err := os.Stat(option.Config.ConfigDir); os.IsNotExist(err) { 915 log.Fatalf("Non-existent configuration directory %s", option.Config.ConfigDir) 916 } 917 918 if m, err := option.ReadDirConfig(option.Config.ConfigDir); err != nil { 919 log.Fatalf("Unable to read configuration directory %s: %s", option.Config.ConfigDir, err) 920 } else { 921 // replace deprecated fields with new fields 922 option.ReplaceDeprecatedFields(m) 923 err := option.MergeConfig(m) 924 if err != nil { 925 log.Fatalf("Unable to merge configuration: %s", err) 926 } 927 } 928 } 929 930 if option.Config.ConfigFile != "" { 931 viper.SetConfigFile(option.Config.ConfigFile) 932 } else { 933 viper.SetConfigName("ciliumd") // name of config file (without extension) 934 viper.AddConfigPath("$HOME") // adding home directory as first search path 935 } 936 937 // If a config file is found, read it in. 938 if err := viper.ReadInConfig(); err == nil { 939 log.WithField(logfields.Path, viper.ConfigFileUsed()). 940 Info("Using config from file") 941 } else if option.Config.ConfigFile != "" { 942 log.WithField(logfields.Path, option.Config.ConfigFile). 943 Fatal("Error reading config file") 944 } else { 945 log.WithField(logfields.Reason, err).Info("Skipped reading configuration file") 946 } 947 } 948 949 func initEnv(cmd *cobra.Command) { 950 // Prepopulate option.Config with options from CLI. 951 option.Config.Populate() 952 953 // add hooks after setting up metrics in the option.Confog 954 logging.DefaultLogger.Hooks.Add(metrics.NewLoggingHook(components.CiliumAgentName)) 955 956 // Logging should always be bootstrapped first. Do not add any code above this! 957 logging.SetupLogging(option.Config.LogDriver, option.Config.LogOpt, "cilium-agent", option.Config.Debug) 958 959 if option.Config.CMDRefDir != "" { 960 genMarkdown(cmd) 961 } 962 963 option.LogRegisteredOptions(log) 964 965 for _, grp := range option.Config.DebugVerbose { 966 switch grp { 967 case argDebugVerboseFlow: 968 log.Debugf("Enabling flow debug") 969 flowdebug.Enable() 970 case argDebugVerboseKvstore: 971 kvstore.EnableTracing() 972 case argDebugVerboseEnvoy: 973 log.Debugf("Enabling Envoy tracing") 974 envoy.EnableTracing() 975 default: 976 log.Warningf("Unknown verbose debug group: %s", grp) 977 } 978 } 979 980 common.RequireRootPrivilege("cilium-agent") 981 982 log.Info(" _ _ _") 983 log.Info(" ___|_| |_|_ _ _____") 984 log.Info("| _| | | | | | |") 985 log.Info("|___|_|_|_|___|_|_|_|") 986 log.Infof("Cilium %s", version.Version) 987 988 if option.Config.LogSystemLoadConfig { 989 loadinfo.StartBackgroundLogger() 990 } 991 992 if option.Config.DisableEnvoyVersionCheck { 993 log.Info("Envoy version check disabled") 994 } else { 995 envoyVersion := envoy.GetEnvoyVersion() 996 log.Infof("%s", envoyVersion) 997 998 envoyVersionArray := strings.Fields(envoyVersion) 999 if len(envoyVersionArray) < 3 { 1000 log.Fatal("Truncated Envoy version string, cannot verify version match.") 1001 } 1002 // Make sure Envoy version matches ours 1003 if !strings.HasPrefix(envoyVersionArray[2], envoy.RequiredEnvoyVersionSHA) { 1004 log.Fatalf("Envoy version %s does not match with required version %s ,aborting.", 1005 envoyVersionArray[2], envoy.RequiredEnvoyVersionSHA) 1006 } 1007 } 1008 1009 // This check is here instead of in DaemonConfig.Populate (invoked at the 1010 // start of this function as option.Config.Populate) to avoid an import loop. 1011 if option.Config.IdentityAllocationMode == option.IdentityAllocationModeCRD && !k8s.IsEnabled() { 1012 log.Fatal("CRD Identity allocation mode requires k8s to be configured.") 1013 } 1014 1015 if option.Config.PProf { 1016 pprof.Enable() 1017 } 1018 1019 if option.Config.PreAllocateMaps { 1020 bpf.EnableMapPreAllocation() 1021 } 1022 1023 scopedLog := log.WithFields(logrus.Fields{ 1024 logfields.Path + ".RunDir": option.Config.RunDir, 1025 logfields.Path + ".LibDir": option.Config.LibDir, 1026 }) 1027 1028 if option.Config.LBInterface != "" { 1029 service.EnableGlobalServiceID(true) 1030 } 1031 1032 option.Config.BpfDir = filepath.Join(option.Config.LibDir, defaults.BpfDir) 1033 scopedLog = scopedLog.WithField(logfields.Path+".BPFDir", defaults.BpfDir) 1034 if err := os.MkdirAll(option.Config.RunDir, defaults.RuntimePathRights); err != nil { 1035 scopedLog.WithError(err).Fatal("Could not create runtime directory") 1036 } 1037 1038 option.Config.StateDir = filepath.Join(option.Config.RunDir, defaults.StateDir) 1039 scopedLog = scopedLog.WithField(logfields.Path+".StateDir", option.Config.StateDir) 1040 if err := os.MkdirAll(option.Config.StateDir, defaults.StateDirRights); err != nil { 1041 scopedLog.WithError(err).Fatal("Could not create state directory") 1042 } 1043 1044 if err := os.MkdirAll(option.Config.LibDir, defaults.RuntimePathRights); err != nil { 1045 scopedLog.WithError(err).Fatal("Could not create library directory") 1046 } 1047 if !option.Config.KeepTemplates { 1048 // We need to remove the old probes here as otherwise stale .t tests could 1049 // still reside from newer Cilium versions which might break downgrade. 1050 if err := os.RemoveAll(filepath.Join(option.Config.BpfDir, "/probes/")); err != nil { 1051 scopedLog.WithError(err).Fatal("Could not delete old probes from library directory") 1052 } 1053 if err := RestoreAssets(option.Config.LibDir, defaults.BpfDir); err != nil { 1054 scopedLog.WithError(err).Fatal("Unable to restore agent assets") 1055 } 1056 // Restore permissions of executable files 1057 if err := RestoreExecPermissions(option.Config.LibDir, `.*\.sh`); err != nil { 1058 scopedLog.WithError(err).Fatal("Unable to restore agent assets") 1059 } 1060 } 1061 if option.Config.MaxControllerInterval < 0 { 1062 scopedLog.Fatalf("Invalid %s value %d", option.MaxCtrlIntervalName, option.Config.MaxControllerInterval) 1063 } 1064 1065 checkMinRequirements() 1066 1067 if err := pidfile.Write(defaults.PidFilePath); err != nil { 1068 log.WithField(logfields.Path, defaults.PidFilePath).WithError(err).Fatal("Failed to create Pidfile") 1069 } 1070 1071 option.Config.AllowLocalhost = strings.ToLower(option.Config.AllowLocalhost) 1072 switch option.Config.AllowLocalhost { 1073 case option.AllowLocalhostAlways, option.AllowLocalhostAuto, option.AllowLocalhostPolicy: 1074 default: 1075 log.Fatalf("Invalid setting for --allow-localhost, must be { %s, %s, %s }", 1076 option.AllowLocalhostAuto, option.AllowLocalhostAlways, option.AllowLocalhostPolicy) 1077 } 1078 1079 option.Config.ModePreFilter = strings.ToLower(option.Config.ModePreFilter) 1080 switch option.Config.ModePreFilter { 1081 case option.ModePreFilterNative: 1082 option.Config.ModePreFilter = "xdpdrv" 1083 case option.ModePreFilterGeneric: 1084 option.Config.ModePreFilter = "xdpgeneric" 1085 default: 1086 log.Fatalf("Invalid setting for --prefilter-mode, must be { %s, %s }", 1087 option.ModePreFilterNative, option.ModePreFilterGeneric) 1088 } 1089 1090 scopedLog = log.WithField(logfields.Path, option.Config.SocketPath) 1091 socketDir := path.Dir(option.Config.SocketPath) 1092 if err := os.MkdirAll(socketDir, defaults.RuntimePathRights); err != nil { 1093 scopedLog.WithError(err).Fatal("Cannot mkdir directory for cilium socket") 1094 } 1095 1096 if err := os.Remove(option.Config.SocketPath); !os.IsNotExist(err) && err != nil { 1097 scopedLog.WithError(err).Fatal("Cannot remove existing Cilium sock") 1098 } 1099 1100 // The standard operation is to mount the BPF filesystem to the 1101 // standard location (/sys/fs/bpf). The user may chose to specify 1102 // the path to an already mounted filesystem instead. This is 1103 // useful if the daemon is being round inside a namespace and the 1104 // BPF filesystem is mapped into the slave namespace. 1105 bpf.CheckOrMountFS(option.Config.BPFRoot) 1106 cgroups.CheckOrMountCgrpFS(option.Config.CGroupRoot) 1107 1108 option.Config.Opts.SetBool(option.Debug, option.Config.Debug) 1109 option.Config.Opts.SetBool(option.DebugLB, option.Config.Debug) 1110 option.Config.Opts.SetBool(option.DropNotify, true) 1111 option.Config.Opts.SetBool(option.TraceNotify, true) 1112 option.Config.Opts.SetBool(option.PolicyTracing, option.Config.EnableTracing) 1113 option.Config.Opts.SetBool(option.Conntrack, !option.Config.DisableConntrack) 1114 option.Config.Opts.SetBool(option.ConntrackAccounting, !option.Config.DisableConntrack) 1115 option.Config.Opts.SetBool(option.ConntrackLocal, false) 1116 1117 monitorAggregationLevel, err := option.ParseMonitorAggregationLevel(option.Config.MonitorAggregation) 1118 if err != nil { 1119 log.WithError(err).Fatalf("Failed to parse %s: %s", 1120 option.MonitorAggregationName, err) 1121 } 1122 option.Config.Opts.SetValidated(option.MonitorAggregation, monitorAggregationLevel) 1123 1124 policy.SetPolicyEnabled(option.Config.EnablePolicy) 1125 1126 if err := cache.AddUserDefinedNumericIdentitySet(option.Config.FixedIdentityMapping); err != nil { 1127 log.Fatalf("Invalid fixed identities provided: %s", err) 1128 } 1129 1130 if !option.Config.EnableIPv4 && !option.Config.EnableIPv6 { 1131 log.Fatal("Either IPv4 or IPv6 addressing must be enabled") 1132 } 1133 if err := labels.ParseLabelPrefixCfg(option.Config.Labels, option.Config.LabelPrefixFile); err != nil { 1134 log.WithError(err).Fatal("Unable to parse Label prefix configuration") 1135 } 1136 1137 _, r, err := net.ParseCIDR(option.Config.NAT46Range) 1138 if err != nil { 1139 log.WithError(err).WithField(logfields.V6Prefix, option.Config.NAT46Range).Fatal("Invalid NAT46 prefix") 1140 } 1141 1142 option.Config.NAT46Prefix = r 1143 1144 switch option.Config.DatapathMode { 1145 case option.DatapathModeVeth: 1146 if name := viper.GetString(option.IpvlanMasterDevice); name != "undefined" { 1147 log.WithField(logfields.IpvlanMasterDevice, name). 1148 Fatal("ipvlan master device cannot be set in the 'veth' datapath mode") 1149 } 1150 if option.Config.Tunnel == "" { 1151 option.Config.Tunnel = option.TunnelVXLAN 1152 } 1153 if option.Config.IsFlannelMasterDeviceSet() { 1154 if option.Config.Tunnel != option.TunnelDisabled { 1155 log.Warnf("Running Cilium in flannel mode requires tunnel mode be '%s'. Changing tunnel mode to: %s", option.TunnelDisabled, option.TunnelDisabled) 1156 option.Config.Tunnel = option.TunnelDisabled 1157 } 1158 if option.Config.EnableIPv6 { 1159 log.Warn("Running Cilium in flannel mode requires IPv6 mode be 'false'. Disabling IPv6 mode") 1160 option.Config.EnableIPv6 = false 1161 } 1162 if option.Config.FlannelManageExistingContainers && !option.Config.WorkloadsEnabled() { 1163 log.Warnf("Managing existing flannel containers with Cilium requires container workloads. Changing %s to %q", option.ContainerRuntime, "auto") 1164 option.Config.Workloads = option.ContainerRuntimeAuto 1165 } 1166 } 1167 case option.DatapathModeIpvlan: 1168 if option.Config.Tunnel != "" && option.Config.Tunnel != option.TunnelDisabled { 1169 log.WithField(logfields.Tunnel, option.Config.Tunnel). 1170 Fatal("tunnel cannot be set in the 'ipvlan' datapath mode") 1171 } 1172 if option.Config.Device != "undefined" { 1173 log.WithField(logfields.Device, option.Config.Device). 1174 Fatal("device cannot be set in the 'ipvlan' datapath mode") 1175 } 1176 if option.Config.EnableIPSec { 1177 log.Fatal("Currently ipsec cannot be used in the 'ipvlan' datapath mode.") 1178 } 1179 1180 option.Config.Tunnel = option.TunnelDisabled 1181 // We disallow earlier command line combination of --device with 1182 // --datapath-mode ipvlan. But given all the remaining logic is 1183 // shared with option.Config.Device, override it here internally 1184 // with the specified ipvlan master device. Reason to have a 1185 // separate, more specific command line parameter here and in 1186 // the swagger API is that in future we might deprecate --device 1187 // parameter with e.g. some auto-detection mechanism, thus for 1188 // ipvlan it is desired to have a separate one, see PR #6608. 1189 option.Config.Device = viper.GetString(option.IpvlanMasterDevice) 1190 if option.Config.Device == "undefined" { 1191 log.WithField(logfields.IpvlanMasterDevice, option.Config.Device). 1192 Fatal("ipvlan master device must be specified in the 'ipvlan' datapath mode") 1193 } 1194 link, err := netlink.LinkByName(option.Config.Device) 1195 if err != nil { 1196 log.WithError(err).WithField(logfields.IpvlanMasterDevice, option.Config.Device). 1197 Fatal("Cannot find device interface") 1198 } 1199 option.Config.Ipvlan.MasterDeviceIndex = link.Attrs().Index 1200 option.Config.Ipvlan.OperationMode = option.OperationModeL3 1201 if option.Config.InstallIptRules { 1202 option.Config.Ipvlan.OperationMode = option.OperationModeL3S 1203 } 1204 default: 1205 log.WithField(logfields.DatapathMode, option.Config.DatapathMode).Fatal("Invalid datapath mode") 1206 } 1207 1208 if option.Config.EnableL7Proxy && !option.Config.InstallIptRules { 1209 log.Fatal("L7 proxy requires iptables rules (--install-iptables-rules=\"true\")") 1210 } 1211 1212 if option.Config.EnableIPSec && option.Config.Tunnel == option.TunnelDisabled && option.Config.EncryptInterface == "" { 1213 link, err := linuxdatapath.NodeDeviceNameWithDefaultRoute() 1214 if err != nil { 1215 log.WithError(err).Fatal("Ipsec default interface lookup failed, consider \"encrypt-interface\" to manually configure interface.", err) 1216 } 1217 option.Config.EncryptInterface = link 1218 } 1219 1220 // BPF masquerade specified, rejecting unsupported options for this mode. 1221 if !option.Config.InstallIptRules && option.Config.Masquerade { 1222 if option.Config.DatapathMode != option.DatapathModeIpvlan { 1223 log.WithField(logfields.DatapathMode, option.Config.DatapathMode). 1224 Fatal("BPF masquerade currently only in ipvlan datapath mode (restriction will be lifted soon)") 1225 } 1226 if option.Config.Tunnel != option.TunnelDisabled { 1227 log.WithField(logfields.Tunnel, option.Config.Tunnel). 1228 Fatal("BPF masquerade only in direct routing mode supported") 1229 } 1230 if option.Config.Device == "undefined" { 1231 log.WithField(logfields.Device, option.Config.Device). 1232 Fatal("BPF masquerade needs external facing device specified") 1233 } 1234 } 1235 1236 if option.Config.EnableNodePort && 1237 !(option.Config.EnableHostReachableServices && 1238 option.Config.EnableHostServicesTCP && option.Config.EnableHostServicesUDP) { 1239 // We enable host reachable services in order to allow 1240 // access to node port services from the host. 1241 log.Info("Auto-enabling host reachable services for UDP and TCP as required by BPF NodePort.") 1242 option.Config.EnableHostReachableServices = true 1243 option.Config.EnableHostServicesTCP = true 1244 option.Config.EnableHostServicesUDP = true 1245 } 1246 1247 if option.Config.EnableNodePort && option.Config.Device == "undefined" { 1248 device, err := linuxdatapath.NodeDeviceNameWithDefaultRoute() 1249 if err != nil { 1250 log.WithError(err).Fatal("BPF NodePort's external facing device could not be determined. Use --device to specify.") 1251 } 1252 log.WithField(logfields.Interface, device). 1253 Info("Using auto-derived device for BPF node port") 1254 option.Config.Device = device 1255 } 1256 1257 if option.Config.EnableHostReachableServices { 1258 if option.Config.EnableHostServicesTCP && 1259 (option.Config.EnableIPv4 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_INET4_CONNECT) != nil || 1260 option.Config.EnableIPv6 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_INET6_CONNECT) != nil) { 1261 log.Fatal("BPF host reachable services for TCP needs kernel 4.17.0 or newer.") 1262 } 1263 // NOTE: as host-lb is a hard dependency for NodePort BPF, the following 1264 // probe will catch if the fib_lookup helper is missing (< 4.18), 1265 // which is another hard dependency for NodePort BPF. 1266 if option.Config.EnableHostServicesUDP && 1267 (option.Config.EnableIPv4 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_UDP4_RECVMSG) != nil || 1268 option.Config.EnableIPv6 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_UDP6_RECVMSG) != nil) { 1269 log.Fatal("BPF host reachable services for UDP needs kernel 4.19.57, 5.1.16, 5.2.0 or newer. If you run an older kernel and only need TCP, then specify: --host-reachable-services-protos=tcp") 1270 } 1271 } 1272 1273 // If device has been specified, use it to derive better default 1274 // allocation prefixes 1275 if option.Config.Device != "undefined" { 1276 node.InitDefaultPrefix(option.Config.Device) 1277 } 1278 1279 if option.Config.IPv6NodeAddr != "auto" { 1280 if ip := net.ParseIP(option.Config.IPv6NodeAddr); ip == nil { 1281 log.WithField(logfields.IPAddr, option.Config.IPv6NodeAddr).Fatal("Invalid IPv6 node address") 1282 } else { 1283 if !ip.IsGlobalUnicast() { 1284 log.WithField(logfields.IPAddr, ip).Fatal("Invalid IPv6 node address: not a global unicast address") 1285 } 1286 1287 node.SetIPv6(ip) 1288 } 1289 } 1290 1291 if option.Config.IPv4NodeAddr != "auto" { 1292 if ip := net.ParseIP(option.Config.IPv4NodeAddr); ip == nil { 1293 log.WithField(logfields.IPAddr, option.Config.IPv4NodeAddr).Fatal("Invalid IPv4 node address") 1294 } else { 1295 node.SetExternalIPv4(ip) 1296 } 1297 } 1298 1299 if option.Config.SidecarHTTPProxy { 1300 log.Warn(`"sidecar-http-proxy" flag is deprecated and has no effect`) 1301 } 1302 1303 k8s.SidecarIstioProxyImageRegexp, err = regexp.Compile(option.Config.SidecarIstioProxyImage) 1304 if err != nil { 1305 log.WithError(err).Fatal("Invalid sidecar-istio-proxy-image regular expression") 1306 return 1307 } 1308 } 1309 1310 // waitForHostDeviceWhenReady waits the given ifaceName to be up and ready. If 1311 // ifaceName is not found, then it will wait forever until the device is 1312 // created. 1313 func waitForHostDeviceWhenReady(ifaceName string) error { 1314 for i := 0; ; i++ { 1315 if i%10 == 0 { 1316 log.WithField(logfields.Interface, ifaceName). 1317 Info("Waiting for the underlying interface to be initialized with containers") 1318 } 1319 _, err := netlink.LinkByName(ifaceName) 1320 if err == nil { 1321 log.WithField(logfields.Interface, ifaceName). 1322 Info("Underlying interface initialized with containers!") 1323 break 1324 } 1325 select { 1326 case <-cleanUPSig: 1327 return errors.New("clean up signal triggered") 1328 default: 1329 time.Sleep(time.Second) 1330 } 1331 } 1332 return nil 1333 } 1334 1335 func endParallelMapMode() { 1336 ipcachemap.IPCache.EndParallelMode() 1337 } 1338 1339 func (d *Daemon) initKVStore() { 1340 goopts := &kvstore.ExtraOptions{ 1341 ClusterSizeDependantInterval: d.nodeDiscovery.Manager.ClusterSizeDependantInterval, 1342 } 1343 1344 controller.NewManager().UpdateController("kvstore-locks-gc", 1345 controller.ControllerParams{ 1346 DoFunc: func(ctx context.Context) error { 1347 kvstore.RunLockGC() 1348 return nil 1349 }, 1350 RunInterval: defaults.KVStoreStaleLockTimeout, 1351 }, 1352 ) 1353 1354 // If K8s is enabled we can do the service translation automagically by 1355 // looking at services from k8s and retrieve the service IP from that. 1356 // This makes cilium to not depend on kube dns to interact with etcd 1357 _, isETCDOperator := kvstore.IsEtcdOperator(option.Config.KVStore, option.Config.KVStoreOpt, option.Config.K8sNamespace) 1358 if k8s.IsEnabled() && isETCDOperator { 1359 // Wait services and endpoints cache are synced with k8s before setting 1360 // up etcd so we can perform the name resolution for etcd-operator 1361 // to the service IP as well perform the service -> backend IPs for 1362 // that service IP. 1363 d.waitForCacheSync(k8sAPIGroupServiceV1Core, k8sAPIGroupEndpointV1Core) 1364 log := log.WithField(logfields.LogSubsys, "etcd") 1365 goopts.DialOption = []grpc.DialOption{ 1366 grpc.WithDialer(k8s.CreateCustomDialer(&d.k8sSvcCache, log)), 1367 } 1368 } 1369 1370 if err := kvstore.Setup(option.Config.KVStore, option.Config.KVStoreOpt, goopts); err != nil { 1371 addrkey := fmt.Sprintf("%s.address", option.Config.KVStore) 1372 addr := option.Config.KVStoreOpt[addrkey] 1373 1374 log.WithError(err).WithFields(logrus.Fields{ 1375 "kvstore": option.Config.KVStore, 1376 "address": addr, 1377 }).Fatal("Unable to setup kvstore") 1378 } 1379 } 1380 1381 func runDaemon() { 1382 datapathConfig := linuxdatapath.DatapathConfiguration{ 1383 HostDevice: option.Config.HostDevice, 1384 EncryptInterface: option.Config.EncryptInterface, 1385 } 1386 1387 log.Info("Initializing daemon") 1388 1389 // Since flannel doesn't create the cni0 interface until the first container 1390 // is initialized we need to wait until it is initialized so we can attach 1391 // the BPF program to it. If Cilium is running as a Kubernetes DaemonSet, 1392 // there is also a script waiting for the interface to be created. 1393 if option.Config.IsFlannelMasterDeviceSet() { 1394 err := waitForHostDeviceWhenReady(option.Config.FlannelMasterDevice) 1395 if err != nil { 1396 log.WithError(err).WithFields(logrus.Fields{ 1397 logfields.Interface: option.Config.FlannelMasterDevice, 1398 }).Error("unable to check for host device") 1399 return 1400 } 1401 } 1402 1403 option.Config.RunMonitorAgent = true 1404 1405 if err := enableIPForwarding(); err != nil { 1406 log.WithError(err).Fatal("Error when enabling sysctl parameters") 1407 } 1408 1409 iptablesManager := &iptables.IptablesManager{} 1410 iptablesManager.Init() 1411 1412 d, restoredEndpoints, err := NewDaemon(linuxdatapath.NewDatapath(datapathConfig, iptablesManager), iptablesManager) 1413 if err != nil { 1414 log.WithError(err).Fatal("Error while creating daemon") 1415 return 1416 } 1417 1418 // This validation needs to be done outside of the agent until 1419 // datapath.NodeAddressing is used consistently across the code base. 1420 log.Info("Validating configured node address ranges") 1421 if err := node.ValidatePostInit(); err != nil { 1422 log.WithError(err).Fatal("postinit failed") 1423 } 1424 1425 if option.Config.IsFlannelMasterDeviceSet() && option.Config.FlannelUninstallOnExit { 1426 cleanup.DeferTerminationCleanupFunction(cleanUPWg, cleanUPSig, func() { 1427 d.compilationMutex.Lock() 1428 loader.DeleteDatapath(context.Background(), option.FlannelMasterDevice, "egress") 1429 d.compilationMutex.Unlock() 1430 }) 1431 } 1432 1433 bootstrapStats.enableConntrack.Start() 1434 log.Info("Starting connection tracking garbage collector") 1435 endpointmanager.EnableConntrackGC(option.Config.EnableIPv4, option.Config.EnableIPv6, 1436 restoredEndpoints.restored) 1437 bootstrapStats.enableConntrack.End(true) 1438 1439 endpointmanager.EndpointSynchronizer = &endpointsynchronizer.EndpointSynchronizer{} 1440 1441 bootstrapStats.k8sInit.Start() 1442 1443 // We need to set up etcd in parallel so we will initialize the k8s 1444 // subsystem as well in parallel so caches will start to be synchronized 1445 // with k8s. 1446 k8sCachesSynced := d.initK8sSubsystem() 1447 if option.Config.KVStore == "" { 1448 log.Info("Skipping kvstore configuration") 1449 } else { 1450 d.initKVStore() 1451 } 1452 1453 // Wait only for certain caches, but not all! 1454 // (Check Daemon.initK8sSubsystem() for more info) 1455 <-k8sCachesSynced 1456 bootstrapStats.k8sInit.End(true) 1457 restoreComplete := d.initRestore(restoredEndpoints) 1458 1459 if option.Config.IsFlannelMasterDeviceSet() { 1460 if option.Config.EnableEndpointHealthChecking { 1461 log.Warn("Running Cilium in flannel mode doesn't support endpoint connectivity health checking. Disabling endpoint connectivity health check.") 1462 option.Config.EnableEndpointHealthChecking = false 1463 } 1464 1465 err := node.SetInternalIPv4From(option.Config.FlannelMasterDevice) 1466 if err != nil { 1467 log.WithError(err).WithField("device", option.Config.FlannelMasterDevice).Fatal("Unable to set internal IPv4") 1468 } 1469 if option.Config.FlannelManageExistingContainers { 1470 log.Info("Searching for existing containers...") 1471 d.attachExistingInfraContainers() 1472 } 1473 } 1474 1475 if !option.Config.DryMode { 1476 go func() { 1477 if restoreComplete != nil { 1478 <-restoreComplete 1479 } 1480 d.dnsNameManager.CompleteBootstrap() 1481 maps.CollectStaleMapGarbage() 1482 maps.RemoveDisabledMaps() 1483 }() 1484 } 1485 1486 // The workload event listener *must* be enabled *after* restored endpoints 1487 // are added into the endpoint manager; otherwise, updates to important 1488 // endpoint metadata, such as Kubernetes pod name and namespace, will not 1489 // be performed on the endpoint. 1490 eventsCh, err := workloads.EnableEventListener() 1491 if err != nil { 1492 log.WithError(err).Fatal("Error while enabling workload event watcher") 1493 } else { 1494 d.workloadsEventsCh = eventsCh 1495 } 1496 1497 bootstrapStats.healthCheck.Start() 1498 if option.Config.EnableHealthChecking { 1499 d.initHealth() 1500 } 1501 bootstrapStats.healthCheck.End(true) 1502 1503 d.startStatusCollector() 1504 1505 metricsErrs := initMetrics() 1506 1507 bootstrapStats.initAPI.Start() 1508 api := d.instantiateAPI() 1509 1510 server := server.NewServer(api) 1511 server.EnabledListeners = []string{"unix"} 1512 server.SocketPath = flags.Filename(option.Config.SocketPath) 1513 server.ReadTimeout = apiTimeout 1514 server.WriteTimeout = apiTimeout 1515 defer server.Shutdown() 1516 1517 server.ConfigureAPI() 1518 bootstrapStats.initAPI.End(true) 1519 1520 repr, err := monitorAPI.TimeRepr(time.Now()) 1521 if err != nil { 1522 log.WithError(err).Warn("Failed to generate agent start monitor message") 1523 } else { 1524 d.SendNotification(monitorAPI.AgentNotifyStart, repr) 1525 } 1526 1527 log.WithField("bootstrapTime", time.Since(bootstrapTimestamp)). 1528 Info("Daemon initialization completed") 1529 1530 if option.Config.WriteCNIConfigurationWhenReady != "" { 1531 input, err := ioutil.ReadFile(option.Config.ReadCNIConfiguration) 1532 if err != nil { 1533 log.WithError(err).Fatal("Unable to read CNI configuration file") 1534 } 1535 1536 if err = ioutil.WriteFile(option.Config.WriteCNIConfigurationWhenReady, input, 0644); err != nil { 1537 log.WithError(err).Fatalf("Unable to write CNI configuration file to %s", option.Config.WriteCNIConfigurationWhenReady) 1538 } else { 1539 log.Infof("Wrote CNI configuration file to %s", option.Config.WriteCNIConfigurationWhenReady) 1540 } 1541 } 1542 1543 errs := make(chan error, 1) 1544 1545 go func() { 1546 errs <- server.Serve() 1547 }() 1548 1549 if k8s.IsEnabled() { 1550 bootstrapStats.k8sInit.Start() 1551 k8s.Client().MarkNodeReady(node.GetName()) 1552 bootstrapStats.k8sInit.End(true) 1553 } 1554 1555 bootstrapStats.overall.End(true) 1556 bootstrapStats.updateMetrics() 1557 1558 select { 1559 case err := <-metricsErrs: 1560 if err != nil { 1561 log.WithError(err).Fatal("Cannot start metrics server") 1562 } 1563 case err := <-errs: 1564 if err != nil { 1565 log.WithError(err).Fatal("Error returned from non-returning Serve() call") 1566 } 1567 } 1568 } 1569 1570 func (d *Daemon) instantiateAPI() *restapi.CiliumAPI { 1571 1572 swaggerSpec, err := loads.Analyzed(server.SwaggerJSON, "") 1573 if err != nil { 1574 log.WithError(err).Fatal("Cannot load swagger spec") 1575 } 1576 1577 log.Info("Initializing Cilium API") 1578 api := restapi.NewCiliumAPI(swaggerSpec) 1579 1580 api.Logger = log.Infof 1581 1582 // /healthz/ 1583 api.DaemonGetHealthzHandler = NewGetHealthzHandler(d) 1584 1585 // /cluster/nodes 1586 api.DaemonGetClusterNodesHandler = NewGetClusterNodesHandler(d) 1587 1588 // /config/ 1589 api.DaemonGetConfigHandler = NewGetConfigHandler(d) 1590 api.DaemonPatchConfigHandler = NewPatchConfigHandler(d) 1591 1592 // /endpoint/ 1593 api.EndpointGetEndpointHandler = NewGetEndpointHandler(d) 1594 1595 // /endpoint/{id} 1596 api.EndpointGetEndpointIDHandler = NewGetEndpointIDHandler(d) 1597 api.EndpointPutEndpointIDHandler = NewPutEndpointIDHandler(d) 1598 api.EndpointPatchEndpointIDHandler = NewPatchEndpointIDHandler(d) 1599 api.EndpointDeleteEndpointIDHandler = NewDeleteEndpointIDHandler(d) 1600 1601 // /endpoint/{id}config/ 1602 api.EndpointGetEndpointIDConfigHandler = NewGetEndpointIDConfigHandler(d) 1603 api.EndpointPatchEndpointIDConfigHandler = NewPatchEndpointIDConfigHandler(d) 1604 1605 // /endpoint/{id}/labels/ 1606 api.EndpointGetEndpointIDLabelsHandler = NewGetEndpointIDLabelsHandler(d) 1607 api.EndpointPatchEndpointIDLabelsHandler = NewPatchEndpointIDLabelsHandler(d) 1608 1609 // /endpoint/{id}/log/ 1610 api.EndpointGetEndpointIDLogHandler = NewGetEndpointIDLogHandler(d) 1611 1612 // /endpoint/{id}/healthz 1613 api.EndpointGetEndpointIDHealthzHandler = NewGetEndpointIDHealthzHandler(d) 1614 1615 // /identity/ 1616 api.PolicyGetIdentityHandler = newGetIdentityHandler(d) 1617 api.PolicyGetIdentityIDHandler = newGetIdentityIDHandler(d) 1618 1619 // /identity/endpoints 1620 api.PolicyGetIdentityEndpointsHandler = newGetIdentityEndpointsIDHandler(d) 1621 1622 // /policy/ 1623 api.PolicyGetPolicyHandler = newGetPolicyHandler(d) 1624 api.PolicyPutPolicyHandler = newPutPolicyHandler(d) 1625 api.PolicyDeletePolicyHandler = newDeletePolicyHandler(d) 1626 api.PolicyGetPolicySelectorsHandler = newGetPolicyCacheHandler(d) 1627 1628 // /policy/resolve/ 1629 api.PolicyGetPolicyResolveHandler = NewGetPolicyResolveHandler(d) 1630 1631 // /service/{id}/ 1632 api.ServiceGetServiceIDHandler = NewGetServiceIDHandler(d) 1633 api.ServiceDeleteServiceIDHandler = NewDeleteServiceIDHandler(d) 1634 api.ServicePutServiceIDHandler = NewPutServiceIDHandler(d) 1635 1636 // /service/ 1637 api.ServiceGetServiceHandler = NewGetServiceHandler(d) 1638 1639 // /prefilter/ 1640 api.PrefilterGetPrefilterHandler = NewGetPrefilterHandler(d) 1641 api.PrefilterDeletePrefilterHandler = NewDeletePrefilterHandler(d) 1642 api.PrefilterPatchPrefilterHandler = NewPatchPrefilterHandler(d) 1643 1644 // /ipam/{ip}/ 1645 api.IPAMPostIPAMHandler = NewPostIPAMHandler(d) 1646 api.IPAMPostIPAMIPHandler = NewPostIPAMIPHandler(d) 1647 api.IPAMDeleteIPAMIPHandler = NewDeleteIPAMIPHandler(d) 1648 1649 // /debuginfo 1650 api.DaemonGetDebuginfoHandler = NewGetDebugInfoHandler(d) 1651 1652 // /map 1653 api.DaemonGetMapHandler = NewGetMapHandler(d) 1654 api.DaemonGetMapNameHandler = NewGetMapNameHandler(d) 1655 1656 // metrics 1657 api.MetricsGetMetricsHandler = NewGetMetricsHandler(d) 1658 1659 // /fqdn/cache 1660 api.PolicyGetFqdnCacheHandler = NewGetFqdnCacheHandler(d) 1661 api.PolicyDeleteFqdnCacheHandler = NewDeleteFqdnCacheHandler(d) 1662 api.PolicyGetFqdnCacheIDHandler = NewGetFqdnCacheIDHandler(d) 1663 1664 return api 1665 }