github.com/fafucoder/cilium@v1.6.11/daemon/daemon_main.go (about) 1 // Copyright 2016-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "context" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "net" 23 "os" 24 "os/exec" 25 "path" 26 "path/filepath" 27 "regexp" 28 "strings" 29 "time" 30 31 "github.com/cilium/cilium/api/v1/server" 32 "github.com/cilium/cilium/api/v1/server/restapi" 33 "github.com/cilium/cilium/common" 34 _ "github.com/cilium/cilium/pkg/alignchecker" 35 "github.com/cilium/cilium/pkg/bpf" 36 "github.com/cilium/cilium/pkg/cgroups" 37 "github.com/cilium/cilium/pkg/cleanup" 38 "github.com/cilium/cilium/pkg/components" 39 "github.com/cilium/cilium/pkg/controller" 40 "github.com/cilium/cilium/pkg/datapath/iptables" 41 linuxdatapath "github.com/cilium/cilium/pkg/datapath/linux" 42 "github.com/cilium/cilium/pkg/datapath/loader" 43 "github.com/cilium/cilium/pkg/datapath/maps" 44 "github.com/cilium/cilium/pkg/defaults" 45 "github.com/cilium/cilium/pkg/endpointmanager" 46 "github.com/cilium/cilium/pkg/envoy" 47 "github.com/cilium/cilium/pkg/flowdebug" 48 "github.com/cilium/cilium/pkg/identity" 49 "github.com/cilium/cilium/pkg/identity/cache" 50 "github.com/cilium/cilium/pkg/k8s" 51 "github.com/cilium/cilium/pkg/k8s/endpointsynchronizer" 52 "github.com/cilium/cilium/pkg/kvstore" 53 "github.com/cilium/cilium/pkg/labels" 54 "github.com/cilium/cilium/pkg/loadinfo" 55 "github.com/cilium/cilium/pkg/logging" 56 "github.com/cilium/cilium/pkg/logging/logfields" 57 ipcachemap "github.com/cilium/cilium/pkg/maps/ipcache" 58 "github.com/cilium/cilium/pkg/metrics" 59 monitorAPI "github.com/cilium/cilium/pkg/monitor/api" 60 "github.com/cilium/cilium/pkg/node" 61 "github.com/cilium/cilium/pkg/option" 62 "github.com/cilium/cilium/pkg/pidfile" 63 "github.com/cilium/cilium/pkg/policy" 64 "github.com/cilium/cilium/pkg/pprof" 65 "github.com/cilium/cilium/pkg/service" 66 "github.com/cilium/cilium/pkg/version" 67 "github.com/cilium/cilium/pkg/versioncheck" 68 "github.com/cilium/cilium/pkg/workloads" 69 70 "github.com/go-openapi/loads" 71 gops "github.com/google/gops/agent" 72 go_version "github.com/hashicorp/go-version" 73 "github.com/jessevdk/go-flags" 74 "github.com/sirupsen/logrus" 75 "github.com/spf13/cobra" 76 "github.com/spf13/viper" 77 "github.com/vishvananda/netlink" 78 "golang.org/x/sys/unix" 79 "google.golang.org/grpc" 80 ) 81 82 var ( 83 minKernelVer = versioncheck.MustCompile(">= 4.8.0") 84 minClangVer = versioncheck.MustCompile(">= 3.8.0") 85 86 recKernelVer = versioncheck.MustCompile(">= 4.9.0") 87 recClangVer = versioncheck.MustCompile(">= 3.9.0") 88 89 // LLVM/clang version which supports `-mattr=dwarfris` 90 dwarfrisClangVer = versioncheck.MustCompile(">= 7.0.0") 91 canDisableDwarfRelocations bool 92 ) 93 94 const ( 95 // list of supported verbose debug groups 96 argDebugVerboseFlow = "flow" 97 argDebugVerboseKvstore = "kvstore" 98 argDebugVerboseEnvoy = "envoy" 99 100 apiTimeout = 60 * time.Second 101 daemonSubsys = "daemon" 102 103 // fatalSleep is the duration Cilium should sleep before existing in case 104 // of a log.Fatal is issued or a CLI flag is specified but does not exist. 105 fatalSleep = 2 * time.Second 106 ) 107 108 var ( 109 log = logging.DefaultLogger.WithField(logfields.LogSubsys, daemonSubsys) 110 111 bootstrapTimestamp = time.Now() 112 113 // RootCmd represents the base command when called without any subcommands 114 RootCmd = &cobra.Command{ 115 Use: "cilium-agent", 116 Short: "Run the cilium agent", 117 Run: func(cmd *cobra.Command, args []string) { 118 bootstrapStats.earlyInit.Start() 119 initEnv(cmd) 120 bootstrapStats.earlyInit.End(true) 121 runDaemon() 122 }, 123 } 124 125 bootstrapStats = bootstrapStatistics{} 126 ) 127 128 func init() { 129 RootCmd.SetFlagErrorFunc(func(_ *cobra.Command, e error) error { 130 time.Sleep(fatalSleep) 131 return e 132 }) 133 logrus.RegisterExitHandler(func() { 134 time.Sleep(fatalSleep) 135 }, 136 ) 137 } 138 139 func daemonMain() { 140 bootstrapStats.overall.Start() 141 142 // Open socket for using gops to get stacktraces of the agent. 143 if err := gops.Listen(gops.Options{}); err != nil { 144 errorString := fmt.Sprintf("unable to start gops: %s", err) 145 fmt.Println(errorString) 146 os.Exit(-1) 147 } 148 interruptCh := registerSigHandler() 149 if err := RootCmd.Execute(); err != nil { 150 fmt.Println(err) 151 os.Exit(-1) 152 } 153 <-interruptCh 154 os.Exit(0) 155 } 156 157 func parseKernelVersion(ver string) (*go_version.Version, error) { 158 verStrs := strings.Split(ver, ".") 159 switch { 160 case len(verStrs) < 2: 161 return nil, fmt.Errorf("unable to get kernel version from %q", ver) 162 case len(verStrs) < 3: 163 verStrs = append(verStrs, "0") 164 } 165 // We are assuming the kernel version will be something as: 166 // 4.9.17-040917-generic 167 168 // If verStrs is []string{ "4", "9", "17-040917-generic" } 169 // then we need to retrieve patch number. 170 patch := regexp.MustCompilePOSIX(`^[0-9]+`).FindString(verStrs[2]) 171 if patch == "" { 172 verStrs[2] = "0" 173 } else { 174 verStrs[2] = patch 175 } 176 return go_version.NewVersion(strings.Join(verStrs[:3], ".")) 177 } 178 179 func getKernelVersion() (*go_version.Version, error) { 180 var unameBuf unix.Utsname 181 if err := unix.Uname(&unameBuf); err != nil { 182 log.WithError(err).Fatal("kernel version: NOT OK") 183 } 184 return parseKernelVersion(string(unameBuf.Release[:])) 185 } 186 187 func getClangVersion(filePath string) (*go_version.Version, error) { 188 verOut, err := exec.Command(filePath, "--version").CombinedOutput() 189 if err != nil { 190 log.WithError(err).Fatal("clang version: NOT OK") 191 } 192 res := regexp.MustCompile(`(clang version )([^ ]*)`).FindStringSubmatch(string(verOut)) 193 if len(res) != 3 { 194 log.Fatalf("clang version: NOT OK: unable to get clang's version "+ 195 "from: %q", string(verOut)) 196 } 197 // at this point res is []string{"clang", "version", "maj.min.patch"} 198 verStrs := strings.Split(res[2], ".") 199 if len(verStrs) < 3 { 200 return nil, fmt.Errorf("unable to get clang version from %q", string(verOut)) 201 } 202 v := strings.Join(verStrs[:3], ".") 203 // Handle Ubuntu versioning by removing the dash and everything after. 204 // F. ex. `4.0.0-1ubuntu1~16 -> 4.0.0` and `3.8.0-2ubuntu4 -> 3.8.0`. 205 v = strings.Split(v, "-")[0] 206 return go_version.NewVersion(v) 207 } 208 209 func checkBPFLogs(logType string, fatal bool) { 210 bpfLogFile := logType + ".log" 211 bpfLogPath := filepath.Join(option.Config.StateDir, bpfLogFile) 212 213 if _, err := os.Stat(bpfLogPath); os.IsNotExist(err) { 214 log.Infof("%s check: OK!", logType) 215 } else if err == nil { 216 bpfFeaturesLog, err := ioutil.ReadFile(bpfLogPath) 217 if err != nil { 218 log.WithError(err).WithField(logfields.Path, bpfLogPath).Fatalf("%s check: NOT OK. Unable to read", logType) 219 } 220 printer := log.Debugf 221 if fatal { 222 printer = log.Errorf 223 printer("%s check: NOT OK", logType) 224 } else { 225 printer("%s check: Some features may be limited:", logType) 226 } 227 lines := strings.Trim(string(bpfFeaturesLog), "\n") 228 for _, line := range strings.Split(lines, "\n") { 229 printer(line) 230 } 231 if fatal { 232 log.Fatalf("%s check failed.", logType) 233 } 234 } else { 235 log.WithError(err).WithField(logfields.Path, bpfLogPath).Fatalf("%s check: NOT OK. Unable to read", logType) 236 } 237 } 238 239 func checkMinRequirements() { 240 kernelVersion, err := getKernelVersion() 241 if err != nil { 242 log.WithError(err).Fatal("kernel version: NOT OK") 243 } 244 if !minKernelVer.Check(kernelVersion) { 245 log.Fatalf("kernel version: NOT OK: minimal supported kernel "+ 246 "version is %s; kernel version that is running is: %s", minKernelVer, kernelVersion) 247 } 248 249 if option.Config.EnableIPv6 { 250 if _, err := os.Stat("/proc/net/if_inet6"); os.IsNotExist(err) { 251 log.Fatalf("kernel: ipv6 is enabled in agent but ipv6 is either disabled or not compiled in the kernel") 252 } 253 } 254 255 if filePath, err := exec.LookPath("clang"); err != nil { 256 log.WithError(err).Fatal("clang: NOT OK") 257 } else { 258 clangVersion, err := getClangVersion(filePath) 259 if err != nil { 260 log.WithError(err).Fatal("clang: NOT OK") 261 } 262 if !minClangVer.Check(clangVersion) { 263 log.Fatalf("clang version: NOT OK: minimal supported clang "+ 264 "version is %s; clang version that is running is: %s", minClangVer, clangVersion) 265 } 266 //clang >= 3.9 / kernel < 4.9 - does not work 267 if recClangVer.Check(clangVersion) && !recKernelVer.Check(kernelVersion) { 268 log.Fatalf("clang (%s) and kernel (%s) version: NOT OK: please upgrade "+ 269 "your kernel version to at least %s", 270 clangVersion, kernelVersion, recKernelVer) 271 } 272 canDisableDwarfRelocations = dwarfrisClangVer.Check(clangVersion) 273 log.Infof("clang (%s) and kernel (%s) versions: OK!", clangVersion, kernelVersion) 274 } 275 276 if filePath, err := exec.LookPath("llc"); err != nil { 277 log.WithError(err).Fatal("llc: NOT OK") 278 } else { 279 lccVersion, err := exec.Command(filePath, "--version").CombinedOutput() 280 if err == nil { 281 if strings.Contains(strings.ToLower(string(lccVersion)), "debug") { 282 log.Warn("llc version was compiled in debug mode, expect higher latency!") 283 } 284 } 285 // /usr/include/gnu/stubs-32.h is installed by 'glibc-devel.i686' in fedora 286 // /usr/include/sys/cdefs.h is installed by 'libc6-dev-i386' in ubuntu 287 // both files exist on both systems but cdefs.h already exists in fedora 288 // without 'glibc-devel.i686' so we check for 'stubs-32.h first. 289 if _, err := os.Stat("/usr/include/gnu/stubs-32.h"); os.IsNotExist(err) { 290 log.Fatal("linking environment: NOT OK, please make sure you have 'glibc-devel.i686' if you use fedora system or 'libc6-dev-i386' if you use ubuntu system") 291 } 292 if _, err := os.Stat("/usr/include/sys/cdefs.h"); os.IsNotExist(err) { 293 log.Fatal("linking environment: NOT OK, please make sure you have 'libc6-dev-i386' in your ubuntu system") 294 } 295 log.Info("linking environment: OK!") 296 } 297 298 globalsDir := option.Config.GetGlobalsDir() 299 if err := os.MkdirAll(globalsDir, defaults.StateDirRights); err != nil { 300 log.WithError(err).WithField(logfields.Path, globalsDir).Fatal("Could not create runtime directory") 301 } 302 if err := os.Chdir(option.Config.LibDir); err != nil { 303 log.WithError(err).WithField(logfields.Path, option.Config.LibDir).Fatal("Could not change to runtime directory") 304 } 305 probeScript := filepath.Join(option.Config.BpfDir, "run_probes.sh") 306 if err := exec.Command(probeScript, option.Config.BpfDir, option.Config.StateDir).Run(); err != nil { 307 log.WithError(err).Fatal("BPF Verifier: NOT OK. Unable to run checker for bpf_features") 308 } 309 featuresFilePath := filepath.Join(globalsDir, "bpf_features.h") 310 if _, err := os.Stat(featuresFilePath); os.IsNotExist(err) { 311 log.WithError(err).WithField(logfields.Path, globalsDir).Fatal("BPF Verifier: NOT OK. Unable to read bpf_features.h") 312 } 313 314 checkBPFLogs("bpf_requirements", true) 315 checkBPFLogs("bpf_features", false) 316 bpf.ReadFeatureProbes(featuresFilePath) 317 } 318 319 func skipInit(basePath string) bool { 320 switch basePath { 321 case components.CiliumAgentName, components.CiliumDaemonTestName: 322 return false 323 default: 324 return true 325 } 326 } 327 328 func init() { 329 if skipInit(path.Base(os.Args[0])) { 330 log.Debug("Skipping preparation of cilium-agent environment") 331 return 332 } 333 334 cobra.OnInitialize(initConfig) 335 336 // Reset the help function to also exit, as we block elsewhere in interrupts 337 // and would not exit when called with -h. 338 oldHelpFunc := RootCmd.HelpFunc() 339 RootCmd.SetHelpFunc(func(c *cobra.Command, a []string) { 340 oldHelpFunc(c, a) 341 os.Exit(0) 342 }) 343 344 flags := RootCmd.Flags() 345 346 // Validators 347 option.Config.FixedIdentityMappingValidator = option.Validator(func(val string) (string, error) { 348 vals := strings.Split(val, "=") 349 if len(vals) != 2 { 350 return "", fmt.Errorf(`invalid fixed identity: expecting "<numeric-identity>=<identity-name>" got %q`, val) 351 } 352 ni, err := identity.ParseNumericIdentity(vals[0]) 353 if err != nil { 354 return "", fmt.Errorf(`invalid numeric identity %q: %s`, val, err) 355 } 356 if !identity.IsUserReservedIdentity(ni) { 357 return "", fmt.Errorf(`invalid numeric identity %q: valid numeric identity is between %d and %d`, 358 val, identity.UserReservedNumericIdentity.Uint32(), identity.MinimalNumericIdentity.Uint32()) 359 } 360 lblStr := vals[1] 361 lbl := labels.ParseLabel(lblStr) 362 if lbl.IsReservedSource() { 363 return "", fmt.Errorf(`invalid source %q for label: %s`, labels.LabelSourceReserved, lblStr) 364 } 365 return val, nil 366 }) 367 368 // Env bindings 369 flags.String(option.AccessLog, "", "Path to access log of supported L7 requests observed") 370 option.BindEnv(option.AccessLog) 371 372 flags.StringSlice(option.AgentLabels, []string{}, "Additional labels to identify this agent") 373 option.BindEnv(option.AgentLabels) 374 375 flags.String(option.AllowLocalhost, option.AllowLocalhostAuto, "Policy when to allow local stack to reach local endpoints { auto | always | policy }") 376 option.BindEnv(option.AllowLocalhost) 377 378 flags.Bool(option.AnnotateK8sNode, defaults.AnnotateK8sNode, "Annotate Kubernetes node") 379 option.BindEnv(option.AnnotateK8sNode) 380 381 flags.Bool(option.BlacklistConflictingRoutes, defaults.BlacklistConflictingRoutes, "Don't blacklist IP allocations conflicting with local non-cilium routes") 382 option.BindEnv(option.BlacklistConflictingRoutes) 383 384 flags.Bool(option.AutoCreateCiliumNodeResource, defaults.AutoCreateCiliumNodeResource, "Automatically create CiliumNode resource for own node on startup") 385 option.BindEnv(option.AutoCreateCiliumNodeResource) 386 387 flags.String(option.BPFRoot, "", "Path to BPF filesystem") 388 option.BindEnv(option.BPFRoot) 389 390 flags.String(option.CGroupRoot, "", "Path to Cgroup2 filesystem") 391 option.BindEnv(option.CGroupRoot) 392 393 flags.Bool(option.BPFCompileDebugName, false, "Enable debugging of the BPF compilation process") 394 option.BindEnv(option.BPFCompileDebugName) 395 396 flags.Bool(option.SockopsEnableName, defaults.SockopsEnable, "Enable sockops when kernel supported") 397 option.BindEnv(option.SockopsEnableName) 398 399 flags.Int(option.ClusterIDName, 0, "Unique identifier of the cluster") 400 option.BindEnv(option.ClusterIDName) 401 402 flags.String(option.ClusterName, defaults.ClusterName, "Name of the cluster") 403 option.BindEnv(option.ClusterName) 404 405 flags.String(option.ClusterMeshConfigName, "", "Path to the ClusterMesh configuration directory") 406 option.BindEnv(option.ClusterMeshConfigName) 407 408 flags.String(option.ConfigFile, "", `Configuration file (default "$HOME/ciliumd.yaml")`) 409 option.BindEnv(option.ConfigFile) 410 411 flags.String(option.ConfigDir, "", `Configuration directory that contains a file for each option`) 412 option.BindEnv(option.ConfigDir) 413 414 flags.Uint(option.ConntrackGarbageCollectorIntervalDeprecated, 0, "Garbage collection interval for the connection tracking table (in seconds)") 415 flags.MarkDeprecated(option.ConntrackGarbageCollectorIntervalDeprecated, fmt.Sprintf("please use --%s", option.ConntrackGCInterval)) 416 option.BindEnv(option.ConntrackGarbageCollectorIntervalDeprecated) 417 418 flags.Duration(option.ConntrackGCInterval, time.Duration(0), "Overwrite the connection-tracking garbage collection interval") 419 option.BindEnv(option.ConntrackGCInterval) 420 421 flags.StringSlice(option.ContainerRuntime, option.ContainerRuntimeAuto, `Sets the container runtime(s) used by Cilium { containerd | crio | docker | none | auto } ( "auto" uses the container runtime found in the order: "docker", "containerd", "crio" )`) 422 option.BindEnv(option.ContainerRuntime) 423 424 flags.Var(option.NewNamedMapOptions(option.ContainerRuntimeEndpoint, &option.Config.ContainerRuntimeEndpoint, nil), 425 option.ContainerRuntimeEndpoint, `Container runtime(s) endpoint(s). (default: `+workloads.GetDefaultEPOptsStringWithPrefix("--container-runtime-endpoint=")+`)`) 426 option.BindEnv(option.ContainerRuntimeEndpoint) 427 428 flags.BoolP(option.DebugArg, "D", false, "Enable debugging mode") 429 option.BindEnv(option.DebugArg) 430 431 flags.StringSlice(option.DebugVerbose, []string{}, "List of enabled verbose debug groups") 432 option.BindEnv(option.DebugVerbose) 433 434 flags.StringP(option.Device, "d", "undefined", "Device facing cluster/external network for direct L3 (non-overlay mode)") 435 option.BindEnv(option.Device) 436 437 flags.String(option.DatapathMode, defaults.DatapathMode, "Datapath mode name") 438 option.BindEnv(option.DatapathMode) 439 440 flags.StringP(option.IpvlanMasterDevice, "", "undefined", "Device facing external network acting as ipvlan master") 441 option.BindEnv(option.IpvlanMasterDevice) 442 443 flags.Bool(option.DisableConntrack, false, "Disable connection tracking") 444 option.BindEnv(option.DisableConntrack) 445 446 flags.Bool(option.LegacyDisableIPv4Name, false, "Disable IPv4 mode") 447 flags.MarkHidden(option.LegacyDisableIPv4Name) 448 option.BindEnv(option.LegacyDisableIPv4Name) 449 450 flags.Bool(option.EnableEndpointRoutes, defaults.EnableEndpointRoutes, "Use per endpoint routes instead of routing via cilium_host") 451 option.BindEnv(option.EnableEndpointRoutes) 452 453 flags.Bool(option.EnableHealthChecking, defaults.EnableHealthChecking, "Enable connectivity health checking") 454 option.BindEnv(option.EnableHealthChecking) 455 456 flags.Bool(option.EnableEndpointHealthChecking, defaults.EnableEndpointHealthChecking, "Enable connectivity health checking between virtual endpoints") 457 option.BindEnv(option.EnableEndpointHealthChecking) 458 459 flags.Bool(option.EnableIPv4Name, defaults.EnableIPv4, "Enable IPv4 support") 460 option.BindEnv(option.EnableIPv4Name) 461 462 flags.Bool(option.EnableIPv6Name, defaults.EnableIPv6, "Enable IPv6 support") 463 option.BindEnv(option.EnableIPv6Name) 464 465 flags.String(option.EncryptInterface, "", "Transparent encryption interface") 466 option.BindEnv(option.EncryptInterface) 467 468 flags.Bool(option.EncryptNode, defaults.EncryptNode, "Enables encrypting traffic from non-Cilium pods and host networking") 469 option.BindEnv(option.EncryptNode) 470 471 flags.StringSlice(option.IPv4PodSubnets, []string{}, "List of IPv4 pod subnets to preconfigure for encryption") 472 option.BindEnv(option.IPv4PodSubnets) 473 474 flags.StringSlice(option.IPv6PodSubnets, []string{}, "List of IPv6 pod subnets to preconfigure for encryption") 475 option.BindEnv(option.IPv6PodSubnets) 476 477 flags.String(option.EndpointInterfaceNamePrefix, defaults.EndpointInterfaceNamePrefix, "Prefix of interface name shared by all endpoints") 478 option.BindEnv(option.EndpointInterfaceNamePrefix) 479 480 flags.StringSlice(option.ExcludeLocalAddress, []string{}, "Exclude CIDR from being recognized as local address") 481 option.BindEnv(option.ExcludeLocalAddress) 482 483 flags.Bool(option.DisableCiliumEndpointCRDName, false, "Disable use of CiliumEndpoint CRD") 484 option.BindEnv(option.DisableCiliumEndpointCRDName) 485 486 flags.Bool(option.DisableK8sServices, false, "Disable east-west K8s load balancing by cilium") 487 option.BindEnv(option.DisableK8sServices) 488 489 flags.String(option.EgressMasqueradeInterfaces, "", "Limit egress masquerading to interface selector") 490 option.BindEnv(option.EgressMasqueradeInterfaces) 491 492 flags.Bool(option.EnableHostReachableServices, false, "Enable reachability of services for host applications (beta)") 493 option.BindEnv(option.EnableHostReachableServices) 494 495 flags.StringSlice(option.HostReachableServicesProtos, []string{option.HostServicesTCP, option.HostServicesUDP}, "Only enable reachability of services for host applications for specific protocols") 496 option.BindEnv(option.HostReachableServicesProtos) 497 498 flags.Bool(option.DeprecatedEnableLegacyServices, false, "Enable legacy (prior-v1.5) services") 499 flags.MarkDeprecated(option.DeprecatedEnableLegacyServices, "this option is deprecated as of v1.6") 500 option.BindEnv(option.DeprecatedEnableLegacyServices) 501 502 flags.StringP(option.Docker, "e", workloads.GetRuntimeDefaultOpt(workloads.Docker, "endpoint"), "Path to docker runtime socket (DEPRECATED: use container-runtime-endpoint instead)") 503 option.BindEnv(option.Docker) 504 505 flags.Bool(option.EnableAutoDirectRoutingName, defaults.EnableAutoDirectRouting, "Enable automatic L2 routing between nodes") 506 option.BindEnv(option.EnableAutoDirectRoutingName) 507 508 flags.Bool(option.EnableXTSocketFallbackName, defaults.EnableXTSocketFallback, "Enable fallback for missing xt_socket module") 509 option.BindEnv(option.EnableXTSocketFallbackName) 510 511 flags.String(option.EnablePolicy, option.DefaultEnforcement, "Enable policy enforcement") 512 option.BindEnv(option.EnablePolicy) 513 514 flags.Bool(option.EnableL7Proxy, defaults.EnableL7Proxy, "Enable L7 proxy for L7 policy enforcement") 515 option.BindEnv(option.EnableL7Proxy) 516 517 flags.Bool(option.EnableTracing, false, "Enable tracing while determining policy (debugging)") 518 option.BindEnv(option.EnableTracing) 519 520 flags.String(option.EnvoyLog, "", "Path to a separate Envoy log file, if any") 521 option.BindEnv(option.EnvoyLog) 522 523 flags.Bool(option.EnableIPSecName, defaults.EnableIPSec, "Enable IPSec support") 524 option.BindEnv(option.EnableIPSecName) 525 526 flags.String(option.IPSecKeyFileName, "", "Path to IPSec key file") 527 option.BindEnv(option.IPSecKeyFileName) 528 529 flags.Bool(option.ForceLocalPolicyEvalAtSource, defaults.ForceLocalPolicyEvalAtSource, "Force policy evaluation of all local communication at the source endpoint") 530 option.BindEnv(option.ForceLocalPolicyEvalAtSource) 531 532 flags.String(option.HTTP403Message, "", "Message returned in proxy L7 403 body") 533 flags.MarkHidden(option.HTTP403Message) 534 option.BindEnv(option.HTTP403Message) 535 536 flags.Uint(option.HTTPRequestTimeout, 60*60, "Time after which a forwarded HTTP request is considered failed unless completed (in seconds); Use 0 for unlimited") 537 option.BindEnv(option.HTTPRequestTimeout) 538 539 flags.Uint(option.HTTPIdleTimeout, 0, "Time after which a non-gRPC HTTP stream is considered failed unless traffic in the stream has been processed (in seconds); defaults to 0 (unlimited)") 540 option.BindEnv(option.HTTPIdleTimeout) 541 542 flags.Uint(option.HTTPMaxGRPCTimeout, 0, "Time after which a forwarded gRPC request is considered failed unless completed (in seconds). A \"grpc-timeout\" header may override this with a shorter value; defaults to 0 (unlimited)") 543 option.BindEnv(option.HTTPMaxGRPCTimeout) 544 545 flags.Uint(option.HTTPRetryCount, 3, "Number of retries performed after a forwarded request attempt fails") 546 option.BindEnv(option.HTTPRetryCount) 547 548 flags.Uint(option.HTTPRetryTimeout, 0, "Time after which a forwarded but uncompleted request is retried (connection failures are retried immediately); defaults to 0 (never)") 549 option.BindEnv(option.HTTPRetryTimeout) 550 551 flags.Uint(option.ProxyConnectTimeout, 1, "Time after which a TCP connect attempt is considered failed unless completed (in seconds)") 552 option.BindEnv(option.ProxyConnectTimeout) 553 554 flags.Bool(option.DisableEnvoyVersionCheck, false, "Do not perform Envoy binary version check on startup") 555 flags.MarkHidden(option.DisableEnvoyVersionCheck) 556 // Disable version check if Envoy build is disabled 557 option.BindEnvWithLegacyEnvFallback(option.DisableEnvoyVersionCheck, "CILIUM_DISABLE_ENVOY_BUILD") 558 559 flags.Var(option.NewNamedMapOptions(option.FixedIdentityMapping, &option.Config.FixedIdentityMapping, option.Config.FixedIdentityMappingValidator), 560 option.FixedIdentityMapping, "Key-value for the fixed identity mapping which allows to use reserved label for fixed identities") 561 option.BindEnv(option.FixedIdentityMapping) 562 563 flags.Duration(option.IdentityChangeGracePeriod, defaults.IdentityChangeGracePeriod, "Time to wait before using new identity on endpoint identity change") 564 option.BindEnv(option.IdentityChangeGracePeriod) 565 566 flags.String(option.IdentityAllocationMode, option.IdentityAllocationModeKVstore, "Method to use for identity allocation") 567 option.BindEnv(option.IdentityAllocationMode) 568 569 flags.String(option.IPAM, "", "Backend to use for IPAM") 570 option.BindEnv(option.IPAM) 571 572 flags.Int(option.IPv4ClusterCIDRMaskSize, 8, "Mask size for the cluster wide CIDR") 573 option.BindEnv(option.IPv4ClusterCIDRMaskSize) 574 575 flags.String(option.IPv4Range, AutoCIDR, "Per-node IPv4 endpoint prefix, e.g. 10.16.0.0/16") 576 option.BindEnv(option.IPv4Range) 577 578 flags.String(option.IPv6Range, AutoCIDR, "Per-node IPv6 endpoint prefix, must be /96, e.g. fd02:1:1::/96") 579 option.BindEnv(option.IPv6Range) 580 581 flags.String(option.IPv6ClusterAllocCIDRName, defaults.IPv6ClusterAllocCIDR, "IPv6 /64 CIDR used to allocate per node endpoint /96 CIDR") 582 option.BindEnv(option.IPv6ClusterAllocCIDRName) 583 584 flags.String(option.IPv4ServiceRange, AutoCIDR, "Kubernetes IPv4 services CIDR if not inside cluster prefix") 585 option.BindEnv(option.IPv4ServiceRange) 586 587 flags.String(option.IPv6ServiceRange, AutoCIDR, "Kubernetes IPv6 services CIDR if not inside cluster prefix") 588 option.BindEnv(option.IPv6ServiceRange) 589 590 flags.Bool(option.K8sEventHandover, defaults.K8sEventHandover, "Enable k8s event handover to kvstore for improved scalability") 591 option.BindEnv(option.K8sEventHandover) 592 593 flags.String(option.K8sAPIServer, "", "Kubernetes api address server (for https use --k8s-kubeconfig-path instead)") 594 option.BindEnv(option.K8sAPIServer) 595 596 flags.String(option.K8sKubeConfigPath, "", "Absolute path of the kubernetes kubeconfig file") 597 option.BindEnv(option.K8sKubeConfigPath) 598 599 flags.String(option.K8sNamespaceName, "", "Name of the Kubernetes namespace in which Cilium is deployed in") 600 flags.MarkHidden(option.K8sNamespaceName) 601 option.BindEnv(option.K8sNamespaceName) 602 603 flags.Bool(option.K8sRequireIPv4PodCIDRName, false, "Require IPv4 PodCIDR to be specified in node resource") 604 option.BindEnv(option.K8sRequireIPv4PodCIDRName) 605 606 flags.Bool(option.K8sRequireIPv6PodCIDRName, false, "Require IPv6 PodCIDR to be specified in node resource") 607 option.BindEnv(option.K8sRequireIPv6PodCIDRName) 608 609 flags.Uint(option.K8sServiceCacheSize, defaults.K8sServiceCacheSize, "Cilium service cache size for kubernetes") 610 option.BindEnv(option.K8sServiceCacheSize) 611 flags.MarkHidden(option.K8sServiceCacheSize) 612 613 flags.Bool(option.K8sForceJSONPatch, false, "When set uses JSON Patch to update CNP and CEP status in kube-apiserver") 614 option.BindEnv(option.K8sForceJSONPatch) 615 flags.MarkHidden(option.K8sForceJSONPatch) 616 617 flags.String(option.K8sWatcherEndpointSelector, defaults.K8sWatcherEndpointSelector, "K8s endpoint watcher will watch for these k8s endpoints") 618 option.BindEnv(option.K8sWatcherEndpointSelector) 619 620 flags.Bool(option.KeepConfig, false, "When restoring state, keeps containers' configuration in place") 621 option.BindEnv(option.KeepConfig) 622 623 flags.Bool(option.KeepBPFTemplates, false, "Do not restore BPF template files from binary") 624 option.BindEnv(option.KeepBPFTemplates) 625 626 flags.String(option.KVStore, "", "Key-value store type") 627 option.BindEnv(option.KVStore) 628 629 flags.Duration(option.KVstoreLeaseTTL, defaults.KVstoreLeaseTTL, "Time-to-live for the KVstore lease.") 630 flags.MarkHidden(option.KVstoreLeaseTTL) 631 option.BindEnv(option.KVstoreLeaseTTL) 632 633 flags.Duration(option.KVstorePeriodicSync, defaults.KVstorePeriodicSync, "Periodic KVstore synchronization interval") 634 option.BindEnv(option.KVstorePeriodicSync) 635 636 flags.Duration(option.KVstoreConnectivityTimeout, defaults.KVstoreConnectivityTimeout, "Time after which an incomplete kvstore operation is considered failed") 637 option.BindEnv(option.KVstoreConnectivityTimeout) 638 639 flags.Duration(option.IPAllocationTimeout, defaults.IPAllocationTimeout, "Time after which an incomplete CIDR allocation is considered failed") 640 option.BindEnv(option.IPAllocationTimeout) 641 642 flags.Var(option.NewNamedMapOptions(option.KVStoreOpt, &option.Config.KVStoreOpt, nil), 643 option.KVStoreOpt, "Key-value store options") 644 option.BindEnv(option.KVStoreOpt) 645 646 flags.Uint(option.K8sWatcherQueueSize, 1024, "Queue size used to serialize each k8s event type") 647 option.BindEnv(option.K8sWatcherQueueSize) 648 649 flags.String(option.LabelPrefixFile, "", "Valid label prefixes file path") 650 option.BindEnv(option.LabelPrefixFile) 651 652 flags.StringSlice(option.Labels, []string{}, "List of label prefixes used to determine identity of an endpoint") 653 option.BindEnv(option.Labels) 654 655 flags.String(option.LB, "", "Enables load balancer mode where load balancer bpf program is attached to the given interface") 656 option.BindEnv(option.LB) 657 658 flags.Bool(option.EnableNodePort, false, "Enable NodePort type services by Cilium (beta)") 659 option.BindEnv(option.EnableNodePort) 660 661 flags.StringSlice(option.NodePortRange, []string{fmt.Sprintf("%d", option.NodePortMinDefault), fmt.Sprintf("%d", option.NodePortMaxDefault)}, fmt.Sprintf("Set the min/max NodePort port range")) 662 option.BindEnv(option.NodePortRange) 663 664 flags.String(option.LibDir, defaults.LibraryPath, "Directory path to store runtime build environment") 665 option.BindEnv(option.LibDir) 666 667 flags.StringSlice(option.LogDriver, []string{}, "Logging endpoints to use for example syslog") 668 option.BindEnv(option.LogDriver) 669 670 flags.Var(option.NewNamedMapOptions(option.LogOpt, &option.Config.LogOpt, nil), 671 option.LogOpt, "Log driver options for cilium") 672 option.BindEnv(option.LogOpt) 673 674 flags.Bool(option.LogSystemLoadConfigName, false, "Enable periodic logging of system load") 675 option.BindEnv(option.LogSystemLoadConfigName) 676 677 flags.String(option.LoopbackIPv4, defaults.LoopbackIPv4, "IPv4 address for service loopback SNAT") 678 option.BindEnv(option.LoopbackIPv4) 679 680 flags.String(option.NAT46Range, defaults.DefaultNAT46Prefix, "IPv6 prefix to map IPv4 addresses to") 681 option.BindEnv(option.NAT46Range) 682 683 flags.Bool(option.Masquerade, true, "Masquerade packets from endpoints leaving the host") 684 option.BindEnv(option.Masquerade) 685 686 flags.Bool(option.InstallIptRules, true, "Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading)") 687 option.BindEnv(option.InstallIptRules) 688 689 flags.Duration(option.IPTablesLockTimeout, 5*time.Second, "Time to pass to each iptables invocation to wait for xtables lock acquisition") 690 option.BindEnv(option.IPTablesLockTimeout) 691 692 flags.Int(option.MaxCtrlIntervalName, 0, "Maximum interval (in seconds) between controller runs. Zero is no limit.") 693 flags.MarkHidden(option.MaxCtrlIntervalName) 694 option.BindEnv(option.MaxCtrlIntervalName) 695 696 flags.StringSlice(option.Metrics, []string{}, "Metrics that should be enabled or disabled from the default metric list. (+metric_foo to enable metric_foo , -metric_bar to disable metric_bar)") 697 option.BindEnv(option.Metrics) 698 699 flags.String(option.MonitorAggregationName, "None", 700 "Level of monitor aggregation for traces from the datapath") 701 option.BindEnvWithLegacyEnvFallback(option.MonitorAggregationName, "CILIUM_MONITOR_AGGREGATION_LEVEL") 702 703 flags.Int(option.MonitorQueueSizeName, 0, "Size of the event queue when reading monitor events") 704 option.BindEnv(option.MonitorQueueSizeName) 705 706 flags.Int(option.MTUName, 0, "Overwrite auto-detected MTU of underlying network") 707 option.BindEnv(option.MTUName) 708 709 flags.Bool(option.PrependIptablesChainsName, true, "Prepend custom iptables chains instead of appending") 710 option.BindEnvWithLegacyEnvFallback(option.PrependIptablesChainsName, "CILIUM_PREPEND_IPTABLES_CHAIN") 711 712 flags.String(option.IPv6NodeAddr, "auto", "IPv6 address of node") 713 option.BindEnv(option.IPv6NodeAddr) 714 715 flags.String(option.IPv4NodeAddr, "auto", "IPv4 address of node") 716 option.BindEnv(option.IPv4NodeAddr) 717 718 flags.String(option.ReadCNIConfiguration, "", "Read to the CNI configuration at specified path to extract per node configuration") 719 option.BindEnv(option.ReadCNIConfiguration) 720 721 flags.Bool(option.Restore, true, "Restores state, if possible, from previous daemon") 722 option.BindEnv(option.Restore) 723 724 flags.Bool(option.SidecarHTTPProxy, false, "Disable host HTTP proxy, assuming proxies in sidecar containers") 725 flags.MarkHidden(option.SidecarHTTPProxy) 726 option.BindEnv(option.SidecarHTTPProxy) 727 728 flags.String(option.SidecarIstioProxyImage, k8s.DefaultSidecarIstioProxyImageRegexp, 729 "Regular expression matching compatible Istio sidecar istio-proxy container image names") 730 option.BindEnv(option.SidecarIstioProxyImage) 731 732 flags.Bool(option.SingleClusterRouteName, false, 733 "Use a single cluster route instead of per node routes") 734 option.BindEnv(option.SingleClusterRouteName) 735 736 flags.String(option.SocketPath, defaults.SockPath, "Sets daemon's socket path to listen for connections") 737 option.BindEnv(option.SocketPath) 738 739 flags.String(option.StateDir, defaults.RuntimePath, "Directory path to store runtime state") 740 option.BindEnv(option.StateDir) 741 742 flags.StringP(option.TunnelName, "t", "", fmt.Sprintf("Tunnel mode {%s} (default \"vxlan\" for the \"veth\" datapath mode)", option.GetTunnelModes())) 743 option.BindEnv(option.TunnelName) 744 745 flags.Int(option.TracePayloadlen, 128, "Length of payload to capture when tracing") 746 option.BindEnv(option.TracePayloadlen) 747 748 flags.Bool(option.Version, false, "Print version information") 749 option.BindEnv(option.Version) 750 751 flags.String(option.FlannelMasterDevice, "", 752 "Installs a BPF program to allow for policy enforcement in the given network interface. "+ 753 "Allows to run Cilium on top of other CNI plugins that provide networking, "+ 754 "e.g. flannel, where for flannel, this value should be set with 'cni0'. [EXPERIMENTAL]") 755 option.BindEnv(option.FlannelMasterDevice) 756 757 flags.Bool(option.FlannelUninstallOnExit, false, fmt.Sprintf("When used along the %s "+ 758 "flag, it cleans up all BPF programs installed when Cilium agent is terminated.", option.FlannelMasterDevice)) 759 option.BindEnv(option.FlannelUninstallOnExit) 760 761 flags.Bool(option.FlannelManageExistingContainers, false, 762 fmt.Sprintf("Installs a BPF program to allow for policy enforcement in already running containers managed by Flannel."+ 763 " Require Cilium to be running in the hostPID.")) 764 option.BindEnv(option.FlannelManageExistingContainers) 765 766 flags.Bool(option.PProf, false, "Enable serving the pprof debugging API") 767 option.BindEnv(option.PProf) 768 769 flags.String(option.PrefilterDevice, "undefined", "Device facing external network for XDP prefiltering") 770 option.BindEnv(option.PrefilterDevice) 771 772 flags.String(option.PrefilterMode, option.ModePreFilterNative, "Prefilter mode { "+option.ModePreFilterNative+" | "+option.ModePreFilterGeneric+" } (default: "+option.ModePreFilterNative+")") 773 option.BindEnv(option.PrefilterMode) 774 775 flags.Bool(option.PreAllocateMapsName, defaults.PreAllocateMaps, "Enable BPF map pre-allocation") 776 option.BindEnv(option.PreAllocateMapsName) 777 778 // We expect only one of the possible variables to be filled. The evaluation order is: 779 // --prometheus-serve-addr, CILIUM_PROMETHEUS_SERVE_ADDR, then PROMETHEUS_SERVE_ADDR 780 // The second environment variable (without the CILIUM_ prefix) is here to 781 // handle the case where someone uses a new image with an older spec, and the 782 // older spec used the older variable name. 783 flags.String(option.PrometheusServeAddr, "", "IP:Port on which to serve prometheus metrics (pass \":Port\" to bind on all interfaces, \"\" is off)") 784 option.BindEnvWithLegacyEnvFallback(option.PrometheusServeAddr, "PROMETHEUS_SERVE_ADDR") 785 786 flags.Int(option.CTMapEntriesGlobalTCPName, option.CTMapEntriesGlobalTCPDefault, "Maximum number of entries in TCP CT table") 787 option.BindEnvWithLegacyEnvFallback(option.CTMapEntriesGlobalTCPName, "CILIUM_GLOBAL_CT_MAX_TCP") 788 789 flags.Int(option.CTMapEntriesGlobalAnyName, option.CTMapEntriesGlobalAnyDefault, "Maximum number of entries in non-TCP CT table") 790 option.BindEnvWithLegacyEnvFallback(option.CTMapEntriesGlobalAnyName, "CILIUM_GLOBAL_CT_MAX_ANY") 791 792 flags.Duration(option.CTMapEntriesTimeoutTCPName, 21600*time.Second, "Timeout for established entries in TCP CT table") 793 option.BindEnv(option.CTMapEntriesTimeoutTCPName) 794 795 flags.Duration(option.CTMapEntriesTimeoutAnyName, 60*time.Second, "Timeout for entries in non-TCP CT table") 796 option.BindEnv(option.CTMapEntriesTimeoutAnyName) 797 798 flags.Duration(option.CTMapEntriesTimeoutSVCTCPName, 21600*time.Second, "Timeout for established service entries in TCP CT table") 799 option.BindEnv(option.CTMapEntriesTimeoutSVCTCPName) 800 801 flags.Duration(option.CTMapEntriesTimeoutSVCAnyName, 60*time.Second, "Timeout for service entries in non-TCP CT table") 802 option.BindEnv(option.CTMapEntriesTimeoutSVCAnyName) 803 804 flags.Duration(option.CTMapEntriesTimeoutSYNName, 60*time.Second, "Establishment timeout for entries in TCP CT table") 805 option.BindEnv(option.CTMapEntriesTimeoutSYNName) 806 807 flags.Duration(option.CTMapEntriesTimeoutFINName, 10*time.Second, "Teardown timeout for entries in TCP CT table") 808 option.BindEnv(option.CTMapEntriesTimeoutFINName) 809 810 flags.Int(option.NATMapEntriesGlobalName, option.NATMapEntriesGlobalDefault, "Maximum number of entries for the global BPF NAT table") 811 option.BindEnv(option.NATMapEntriesGlobalName) 812 813 flags.Int(option.PolicyMapEntriesName, defaults.PolicyMapEntries, "Maximum number of entries in endpoint policy map (per endpoint)") 814 option.BindEnv(option.PolicyMapEntriesName) 815 816 flags.String(option.CMDRef, "", "Path to cmdref output directory") 817 flags.MarkHidden(option.CMDRef) 818 option.BindEnv(option.CMDRef) 819 820 flags.Int(option.ToFQDNsMinTTL, 0, fmt.Sprintf("The minimum time, in seconds, to use DNS data for toFQDNs policies. (default %d when --tofqdns-enable-poller, %d otherwise)", defaults.ToFQDNsMinTTLPoller, defaults.ToFQDNsMinTTL)) 821 option.BindEnv(option.ToFQDNsMinTTL) 822 823 flags.Int(option.ToFQDNsProxyPort, 0, "Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port.") 824 option.BindEnv(option.ToFQDNsProxyPort) 825 826 flags.Bool(option.ToFQDNsEnablePoller, false, "Enable proactive polling of DNS names in toFQDNs.matchName rules.") 827 option.BindEnv(option.ToFQDNsEnablePoller) 828 829 flags.Bool(option.ToFQDNsEnablePollerEvents, true, "Emit DNS responses seen by the DNS poller as Monitor events, if the poller is enabled.") 830 option.BindEnv(option.ToFQDNsEnablePollerEvents) 831 832 flags.StringVar(&option.Config.FQDNRejectResponse, option.FQDNRejectResponseCode, option.FQDNProxyDenyWithRefused, fmt.Sprintf("DNS response code for rejecting DNS requests, available options are '%v'", option.FQDNRejectOptions)) 833 option.BindEnv(option.FQDNRejectResponseCode) 834 835 flags.Int(option.ToFQDNsMaxIPsPerHost, defaults.ToFQDNsMaxIPsPerHost, "Maximum number of IPs to maintain per FQDN name for each endpoint") 836 option.BindEnv(option.ToFQDNsMaxIPsPerHost) 837 838 flags.DurationVar(&option.Config.FQDNProxyResponseMaxDelay, option.FQDNProxyResponseMaxDelay, 50*time.Millisecond, "The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information.") 839 option.BindEnv(option.FQDNProxyResponseMaxDelay) 840 841 flags.String(option.ToFQDNsPreCache, defaults.ToFQDNsPreCache, "DNS cache data at this path is preloaded on agent startup") 842 option.BindEnv(option.ToFQDNsPreCache) 843 844 flags.Int(option.PolicyQueueSize, defaults.PolicyQueueSize, "size of queues for policy-related events") 845 option.BindEnv(option.PolicyQueueSize) 846 847 flags.Int(option.EndpointQueueSize, defaults.EndpointQueueSize, "size of EventQueue per-endpoint") 848 option.BindEnv(option.EndpointQueueSize) 849 850 flags.Bool(option.SelectiveRegeneration, true, "only regenerate endpoints which need to be regenerated upon policy changes") 851 flags.MarkHidden(option.SelectiveRegeneration) 852 option.BindEnv(option.SelectiveRegeneration) 853 854 flags.Bool(option.SkipCRDCreation, false, "Skip Kubernetes Custom Resource Definitions creations") 855 option.BindEnv(option.SkipCRDCreation) 856 857 flags.String(option.WriteCNIConfigurationWhenReady, "", fmt.Sprintf("Write the CNI configuration as specified via --%s to path when agent is ready", option.ReadCNIConfiguration)) 858 option.BindEnv(option.WriteCNIConfigurationWhenReady) 859 860 flags.Duration(option.PolicyTriggerInterval, defaults.PolicyTriggerInterval, "Time between triggers of policy updates (regenerations for all endpoints)") 861 flags.MarkHidden(option.PolicyTriggerInterval) 862 option.BindEnv(option.PolicyTriggerInterval) 863 864 flags.Bool(option.DisableCNPStatusUpdates, false, "Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with `cnp-node-status-gc=false` in cilium-operator)") 865 option.BindEnv(option.DisableCNPStatusUpdates) 866 867 viper.BindPFlags(flags) 868 } 869 870 // RestoreExecPermissions restores file permissions to 0740 of all files inside 871 // `searchDir` with the given regex `patterns`. 872 func RestoreExecPermissions(searchDir string, patterns ...string) error { 873 fileList := []string{} 874 err := filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error { 875 for _, pattern := range patterns { 876 if regexp.MustCompile(pattern).MatchString(f.Name()) { 877 fileList = append(fileList, path) 878 break 879 } 880 } 881 return nil 882 }) 883 for _, fileToChange := range fileList { 884 // Changing files permissions to -rwx:r--:---, we are only 885 // adding executable permission to the owner and keeping the 886 // same permissions stored by go-bindata. 887 if err := os.Chmod(fileToChange, os.FileMode(0740)); err != nil { 888 return err 889 } 890 } 891 return err 892 } 893 894 // initConfig reads in config file and ENV variables if set. 895 func initConfig() { 896 if viper.GetBool("version") { 897 fmt.Printf("Cilium %s\n", version.Version) 898 os.Exit(0) 899 } 900 901 if option.Config.CMDRefDir != "" { 902 return 903 } 904 905 option.Config.ConfigFile = viper.GetString(option.ConfigFile) // enable ability to specify config file via flag 906 option.Config.ConfigDir = viper.GetString(option.ConfigDir) 907 viper.SetEnvPrefix("cilium") 908 909 if option.Config.ConfigDir != "" { 910 if _, err := os.Stat(option.Config.ConfigDir); os.IsNotExist(err) { 911 log.Fatalf("Non-existent configuration directory %s", option.Config.ConfigDir) 912 } 913 914 if m, err := option.ReadDirConfig(option.Config.ConfigDir); err != nil { 915 log.Fatalf("Unable to read configuration directory %s: %s", option.Config.ConfigDir, err) 916 } else { 917 // replace deprecated fields with new fields 918 option.ReplaceDeprecatedFields(m) 919 err := option.MergeConfig(m) 920 if err != nil { 921 log.Fatalf("Unable to merge configuration: %s", err) 922 } 923 } 924 } 925 926 if option.Config.ConfigFile != "" { 927 viper.SetConfigFile(option.Config.ConfigFile) 928 } else { 929 viper.SetConfigName("ciliumd") // name of config file (without extension) 930 viper.AddConfigPath("$HOME") // adding home directory as first search path 931 } 932 933 // If a config file is found, read it in. 934 if err := viper.ReadInConfig(); err == nil { 935 log.WithField(logfields.Path, viper.ConfigFileUsed()). 936 Info("Using config from file") 937 } else if option.Config.ConfigFile != "" { 938 log.WithField(logfields.Path, option.Config.ConfigFile). 939 Fatal("Error reading config file") 940 } else { 941 log.WithField(logfields.Reason, err).Info("Skipped reading configuration file") 942 } 943 } 944 945 func initEnv(cmd *cobra.Command) { 946 // Prepopulate option.Config with options from CLI. 947 option.Config.Populate() 948 949 // add hooks after setting up metrics in the option.Confog 950 logging.DefaultLogger.Hooks.Add(metrics.NewLoggingHook(components.CiliumAgentName)) 951 952 // Logging should always be bootstrapped first. Do not add any code above this! 953 logging.SetupLogging(option.Config.LogDriver, option.Config.LogOpt, "cilium-agent", option.Config.Debug) 954 955 if option.Config.CMDRefDir != "" { 956 genMarkdown(cmd) 957 } 958 959 option.LogRegisteredOptions(log) 960 961 for _, grp := range option.Config.DebugVerbose { 962 switch grp { 963 case argDebugVerboseFlow: 964 log.Debugf("Enabling flow debug") 965 flowdebug.Enable() 966 case argDebugVerboseKvstore: 967 kvstore.EnableTracing() 968 case argDebugVerboseEnvoy: 969 log.Debugf("Enabling Envoy tracing") 970 envoy.EnableTracing() 971 default: 972 log.Warningf("Unknown verbose debug group: %s", grp) 973 } 974 } 975 976 common.RequireRootPrivilege("cilium-agent") 977 978 log.Info(" _ _ _") 979 log.Info(" ___|_| |_|_ _ _____") 980 log.Info("| _| | | | | | |") 981 log.Info("|___|_|_|_|___|_|_|_|") 982 log.Infof("Cilium %s", version.Version) 983 984 if option.Config.LogSystemLoadConfig { 985 loadinfo.StartBackgroundLogger() 986 } 987 988 if option.Config.DisableEnvoyVersionCheck { 989 log.Info("Envoy version check disabled") 990 } else { 991 envoyVersion := envoy.GetEnvoyVersion() 992 log.Infof("%s", envoyVersion) 993 994 envoyVersionArray := strings.Fields(envoyVersion) 995 if len(envoyVersionArray) < 3 { 996 log.Fatal("Truncated Envoy version string, cannot verify version match.") 997 } 998 // Make sure Envoy version matches ours 999 if !strings.HasPrefix(envoyVersionArray[2], envoy.RequiredEnvoyVersionSHA) { 1000 log.Fatalf("Envoy version %s does not match with required version %s ,aborting.", 1001 envoyVersionArray[2], envoy.RequiredEnvoyVersionSHA) 1002 } 1003 } 1004 1005 // This check is here instead of in DaemonConfig.Populate (invoked at the 1006 // start of this function as option.Config.Populate) to avoid an import loop. 1007 if option.Config.IdentityAllocationMode == option.IdentityAllocationModeCRD && !k8s.IsEnabled() { 1008 log.Fatal("CRD Identity allocation mode requires k8s to be configured.") 1009 } 1010 1011 if option.Config.PProf { 1012 pprof.Enable() 1013 } 1014 1015 if option.Config.PreAllocateMaps { 1016 bpf.EnableMapPreAllocation() 1017 } 1018 1019 scopedLog := log.WithFields(logrus.Fields{ 1020 logfields.Path + ".RunDir": option.Config.RunDir, 1021 logfields.Path + ".LibDir": option.Config.LibDir, 1022 }) 1023 1024 if option.Config.LBInterface != "" { 1025 service.EnableGlobalServiceID(true) 1026 } 1027 1028 option.Config.BpfDir = filepath.Join(option.Config.LibDir, defaults.BpfDir) 1029 scopedLog = scopedLog.WithField(logfields.Path+".BPFDir", defaults.BpfDir) 1030 if err := os.MkdirAll(option.Config.RunDir, defaults.RuntimePathRights); err != nil { 1031 scopedLog.WithError(err).Fatal("Could not create runtime directory") 1032 } 1033 1034 option.Config.StateDir = filepath.Join(option.Config.RunDir, defaults.StateDir) 1035 scopedLog = scopedLog.WithField(logfields.Path+".StateDir", option.Config.StateDir) 1036 if err := os.MkdirAll(option.Config.StateDir, defaults.StateDirRights); err != nil { 1037 scopedLog.WithError(err).Fatal("Could not create state directory") 1038 } 1039 1040 if err := os.MkdirAll(option.Config.LibDir, defaults.RuntimePathRights); err != nil { 1041 scopedLog.WithError(err).Fatal("Could not create library directory") 1042 } 1043 if !option.Config.KeepTemplates { 1044 // We need to remove the old probes here as otherwise stale .t tests could 1045 // still reside from newer Cilium versions which might break downgrade. 1046 if err := os.RemoveAll(filepath.Join(option.Config.BpfDir, "/probes/")); err != nil { 1047 scopedLog.WithError(err).Fatal("Could not delete old probes from library directory") 1048 } 1049 if err := RestoreAssets(option.Config.LibDir, defaults.BpfDir); err != nil { 1050 scopedLog.WithError(err).Fatal("Unable to restore agent assets") 1051 } 1052 // Restore permissions of executable files 1053 if err := RestoreExecPermissions(option.Config.LibDir, `.*\.sh`); err != nil { 1054 scopedLog.WithError(err).Fatal("Unable to restore agent assets") 1055 } 1056 } 1057 if option.Config.MaxControllerInterval < 0 { 1058 scopedLog.Fatalf("Invalid %s value %d", option.MaxCtrlIntervalName, option.Config.MaxControllerInterval) 1059 } 1060 1061 checkMinRequirements() 1062 1063 if err := pidfile.Write(defaults.PidFilePath); err != nil { 1064 log.WithField(logfields.Path, defaults.PidFilePath).WithError(err).Fatal("Failed to create Pidfile") 1065 } 1066 1067 option.Config.AllowLocalhost = strings.ToLower(option.Config.AllowLocalhost) 1068 switch option.Config.AllowLocalhost { 1069 case option.AllowLocalhostAlways, option.AllowLocalhostAuto, option.AllowLocalhostPolicy: 1070 default: 1071 log.Fatalf("Invalid setting for --allow-localhost, must be { %s, %s, %s }", 1072 option.AllowLocalhostAuto, option.AllowLocalhostAlways, option.AllowLocalhostPolicy) 1073 } 1074 1075 option.Config.ModePreFilter = strings.ToLower(option.Config.ModePreFilter) 1076 switch option.Config.ModePreFilter { 1077 case option.ModePreFilterNative: 1078 option.Config.ModePreFilter = "xdpdrv" 1079 case option.ModePreFilterGeneric: 1080 option.Config.ModePreFilter = "xdpgeneric" 1081 default: 1082 log.Fatalf("Invalid setting for --prefilter-mode, must be { %s, %s }", 1083 option.ModePreFilterNative, option.ModePreFilterGeneric) 1084 } 1085 1086 scopedLog = log.WithField(logfields.Path, option.Config.SocketPath) 1087 socketDir := path.Dir(option.Config.SocketPath) 1088 if err := os.MkdirAll(socketDir, defaults.RuntimePathRights); err != nil { 1089 scopedLog.WithError(err).Fatal("Cannot mkdir directory for cilium socket") 1090 } 1091 1092 if err := os.Remove(option.Config.SocketPath); !os.IsNotExist(err) && err != nil { 1093 scopedLog.WithError(err).Fatal("Cannot remove existing Cilium sock") 1094 } 1095 1096 // The standard operation is to mount the BPF filesystem to the 1097 // standard location (/sys/fs/bpf). The user may chose to specify 1098 // the path to an already mounted filesystem instead. This is 1099 // useful if the daemon is being round inside a namespace and the 1100 // BPF filesystem is mapped into the slave namespace. 1101 bpf.CheckOrMountFS(option.Config.BPFRoot) 1102 cgroups.CheckOrMountCgrpFS(option.Config.CGroupRoot) 1103 1104 option.Config.Opts.SetBool(option.Debug, option.Config.Debug) 1105 option.Config.Opts.SetBool(option.DebugLB, option.Config.Debug) 1106 option.Config.Opts.SetBool(option.DropNotify, true) 1107 option.Config.Opts.SetBool(option.TraceNotify, true) 1108 option.Config.Opts.SetBool(option.PolicyTracing, option.Config.EnableTracing) 1109 option.Config.Opts.SetBool(option.Conntrack, !option.Config.DisableConntrack) 1110 option.Config.Opts.SetBool(option.ConntrackAccounting, !option.Config.DisableConntrack) 1111 option.Config.Opts.SetBool(option.ConntrackLocal, false) 1112 1113 monitorAggregationLevel, err := option.ParseMonitorAggregationLevel(option.Config.MonitorAggregation) 1114 if err != nil { 1115 log.WithError(err).Fatalf("Failed to parse %s: %s", 1116 option.MonitorAggregationName, err) 1117 } 1118 option.Config.Opts.SetValidated(option.MonitorAggregation, monitorAggregationLevel) 1119 1120 policy.SetPolicyEnabled(option.Config.EnablePolicy) 1121 1122 if err := cache.AddUserDefinedNumericIdentitySet(option.Config.FixedIdentityMapping); err != nil { 1123 log.Fatalf("Invalid fixed identities provided: %s", err) 1124 } 1125 1126 if !option.Config.EnableIPv4 && !option.Config.EnableIPv6 { 1127 log.Fatal("Either IPv4 or IPv6 addressing must be enabled") 1128 } 1129 if err := labels.ParseLabelPrefixCfg(option.Config.Labels, option.Config.LabelPrefixFile); err != nil { 1130 log.WithError(err).Fatal("Unable to parse Label prefix configuration") 1131 } 1132 1133 _, r, err := net.ParseCIDR(option.Config.NAT46Range) 1134 if err != nil { 1135 log.WithError(err).WithField(logfields.V6Prefix, option.Config.NAT46Range).Fatal("Invalid NAT46 prefix") 1136 } 1137 1138 option.Config.NAT46Prefix = r 1139 1140 switch option.Config.DatapathMode { 1141 case option.DatapathModeVeth: 1142 if name := viper.GetString(option.IpvlanMasterDevice); name != "undefined" { 1143 log.WithField(logfields.IpvlanMasterDevice, name). 1144 Fatal("ipvlan master device cannot be set in the 'veth' datapath mode") 1145 } 1146 if option.Config.Tunnel == "" { 1147 option.Config.Tunnel = option.TunnelVXLAN 1148 } 1149 if option.Config.IsFlannelMasterDeviceSet() { 1150 if option.Config.Tunnel != option.TunnelDisabled { 1151 log.Warnf("Running Cilium in flannel mode requires tunnel mode be '%s'. Changing tunnel mode to: %s", option.TunnelDisabled, option.TunnelDisabled) 1152 option.Config.Tunnel = option.TunnelDisabled 1153 } 1154 if option.Config.EnableIPv6 { 1155 log.Warn("Running Cilium in flannel mode requires IPv6 mode be 'false'. Disabling IPv6 mode") 1156 option.Config.EnableIPv6 = false 1157 } 1158 if option.Config.FlannelManageExistingContainers && !option.Config.WorkloadsEnabled() { 1159 log.Warnf("Managing existing flannel containers with Cilium requires container workloads. Changing %s to %q", option.ContainerRuntime, "auto") 1160 option.Config.Workloads = option.ContainerRuntimeAuto 1161 } 1162 } 1163 case option.DatapathModeIpvlan: 1164 if option.Config.Tunnel != "" && option.Config.Tunnel != option.TunnelDisabled { 1165 log.WithField(logfields.Tunnel, option.Config.Tunnel). 1166 Fatal("tunnel cannot be set in the 'ipvlan' datapath mode") 1167 } 1168 if option.Config.Device != "undefined" { 1169 log.WithField(logfields.Device, option.Config.Device). 1170 Fatal("device cannot be set in the 'ipvlan' datapath mode") 1171 } 1172 if option.Config.EnableIPSec { 1173 log.Fatal("Currently ipsec cannot be used in the 'ipvlan' datapath mode.") 1174 } 1175 1176 option.Config.Tunnel = option.TunnelDisabled 1177 // We disallow earlier command line combination of --device with 1178 // --datapath-mode ipvlan. But given all the remaining logic is 1179 // shared with option.Config.Device, override it here internally 1180 // with the specified ipvlan master device. Reason to have a 1181 // separate, more specific command line parameter here and in 1182 // the swagger API is that in future we might deprecate --device 1183 // parameter with e.g. some auto-detection mechanism, thus for 1184 // ipvlan it is desired to have a separate one, see PR #6608. 1185 option.Config.Device = viper.GetString(option.IpvlanMasterDevice) 1186 if option.Config.Device == "undefined" { 1187 log.WithField(logfields.IpvlanMasterDevice, option.Config.Device). 1188 Fatal("ipvlan master device must be specified in the 'ipvlan' datapath mode") 1189 } 1190 link, err := netlink.LinkByName(option.Config.Device) 1191 if err != nil { 1192 log.WithError(err).WithField(logfields.IpvlanMasterDevice, option.Config.Device). 1193 Fatal("Cannot find device interface") 1194 } 1195 option.Config.Ipvlan.MasterDeviceIndex = link.Attrs().Index 1196 option.Config.Ipvlan.OperationMode = option.OperationModeL3 1197 if option.Config.InstallIptRules { 1198 option.Config.Ipvlan.OperationMode = option.OperationModeL3S 1199 } 1200 default: 1201 log.WithField(logfields.DatapathMode, option.Config.DatapathMode).Fatal("Invalid datapath mode") 1202 } 1203 1204 if option.Config.EnableL7Proxy && !option.Config.InstallIptRules { 1205 log.Fatal("L7 proxy requires iptables rules (--install-iptables-rules=\"true\")") 1206 } 1207 1208 if option.Config.EnableIPSec && option.Config.Tunnel == option.TunnelDisabled && option.Config.EncryptInterface == "" { 1209 link, err := linuxdatapath.NodeDeviceNameWithDefaultRoute() 1210 if err != nil { 1211 log.WithError(err).Fatal("Ipsec default interface lookup failed, consider \"encrypt-interface\" to manually configure interface.", err) 1212 } 1213 option.Config.EncryptInterface = link 1214 } 1215 1216 // BPF masquerade specified, rejecting unsupported options for this mode. 1217 if !option.Config.InstallIptRules && option.Config.Masquerade { 1218 if option.Config.DatapathMode != option.DatapathModeIpvlan { 1219 log.WithField(logfields.DatapathMode, option.Config.DatapathMode). 1220 Fatal("BPF masquerade currently only in ipvlan datapath mode (restriction will be lifted soon)") 1221 } 1222 if option.Config.Tunnel != option.TunnelDisabled { 1223 log.WithField(logfields.Tunnel, option.Config.Tunnel). 1224 Fatal("BPF masquerade only in direct routing mode supported") 1225 } 1226 if option.Config.Device == "undefined" { 1227 log.WithField(logfields.Device, option.Config.Device). 1228 Fatal("BPF masquerade needs external facing device specified") 1229 } 1230 } 1231 1232 if option.Config.EnableNodePort && 1233 !(option.Config.EnableHostReachableServices && 1234 option.Config.EnableHostServicesTCP && option.Config.EnableHostServicesUDP) { 1235 // We enable host reachable services in order to allow 1236 // access to node port services from the host. 1237 log.Info("Auto-enabling host reachable services for UDP and TCP as required by BPF NodePort.") 1238 option.Config.EnableHostReachableServices = true 1239 option.Config.EnableHostServicesTCP = true 1240 option.Config.EnableHostServicesUDP = true 1241 } 1242 1243 if option.Config.EnableNodePort && option.Config.Device == "undefined" { 1244 device, err := linuxdatapath.NodeDeviceNameWithDefaultRoute() 1245 if err != nil { 1246 log.WithError(err).Fatal("BPF NodePort's external facing device could not be determined. Use --device to specify.") 1247 } 1248 log.WithField(logfields.Interface, device). 1249 Info("Using auto-derived device for BPF node port") 1250 option.Config.Device = device 1251 } 1252 1253 if option.Config.EnableHostReachableServices { 1254 if option.Config.EnableHostServicesTCP && 1255 (option.Config.EnableIPv4 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_INET4_CONNECT) != nil || 1256 option.Config.EnableIPv6 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_INET6_CONNECT) != nil) { 1257 log.Fatal("BPF host reachable services for TCP needs kernel 4.17.0 or newer.") 1258 } 1259 // NOTE: as host-lb is a hard dependency for NodePort BPF, the following 1260 // probe will catch if the fib_lookup helper is missing (< 4.18), 1261 // which is another hard dependency for NodePort BPF. 1262 if option.Config.EnableHostServicesUDP && 1263 (option.Config.EnableIPv4 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_UDP4_RECVMSG) != nil || 1264 option.Config.EnableIPv6 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_UDP6_RECVMSG) != nil) { 1265 log.Fatal("BPF host reachable services for UDP needs kernel 4.19.57, 5.1.16, 5.2.0 or newer. If you run an older kernel and only need TCP, then specify: --host-reachable-services-protos=tcp") 1266 } 1267 } 1268 1269 // If device has been specified, use it to derive better default 1270 // allocation prefixes 1271 if option.Config.Device != "undefined" { 1272 node.InitDefaultPrefix(option.Config.Device) 1273 } 1274 1275 if option.Config.IPv6NodeAddr != "auto" { 1276 if ip := net.ParseIP(option.Config.IPv6NodeAddr); ip == nil { 1277 log.WithField(logfields.IPAddr, option.Config.IPv6NodeAddr).Fatal("Invalid IPv6 node address") 1278 } else { 1279 if !ip.IsGlobalUnicast() { 1280 log.WithField(logfields.IPAddr, ip).Fatal("Invalid IPv6 node address: not a global unicast address") 1281 } 1282 1283 node.SetIPv6(ip) 1284 } 1285 } 1286 1287 if option.Config.IPv4NodeAddr != "auto" { 1288 if ip := net.ParseIP(option.Config.IPv4NodeAddr); ip == nil { 1289 log.WithField(logfields.IPAddr, option.Config.IPv4NodeAddr).Fatal("Invalid IPv4 node address") 1290 } else { 1291 node.SetExternalIPv4(ip) 1292 } 1293 } 1294 1295 if option.Config.SidecarHTTPProxy { 1296 log.Warn(`"sidecar-http-proxy" flag is deprecated and has no effect`) 1297 } 1298 1299 k8s.SidecarIstioProxyImageRegexp, err = regexp.Compile(option.Config.SidecarIstioProxyImage) 1300 if err != nil { 1301 log.WithError(err).Fatal("Invalid sidecar-istio-proxy-image regular expression") 1302 return 1303 } 1304 } 1305 1306 // waitForHostDeviceWhenReady waits the given ifaceName to be up and ready. If 1307 // ifaceName is not found, then it will wait forever until the device is 1308 // created. 1309 func waitForHostDeviceWhenReady(ifaceName string) error { 1310 for i := 0; ; i++ { 1311 if i%10 == 0 { 1312 log.WithField(logfields.Interface, ifaceName). 1313 Info("Waiting for the underlying interface to be initialized with containers") 1314 } 1315 _, err := netlink.LinkByName(ifaceName) 1316 if err == nil { 1317 log.WithField(logfields.Interface, ifaceName). 1318 Info("Underlying interface initialized with containers!") 1319 break 1320 } 1321 select { 1322 case <-cleanUPSig: 1323 return errors.New("clean up signal triggered") 1324 default: 1325 time.Sleep(time.Second) 1326 } 1327 } 1328 return nil 1329 } 1330 1331 func endParallelMapMode() { 1332 ipcachemap.IPCache.EndParallelMode() 1333 } 1334 1335 func (d *Daemon) initKVStore() { 1336 goopts := &kvstore.ExtraOptions{ 1337 ClusterSizeDependantInterval: d.nodeDiscovery.Manager.ClusterSizeDependantInterval, 1338 } 1339 1340 controller.NewManager().UpdateController("kvstore-locks-gc", 1341 controller.ControllerParams{ 1342 DoFunc: func(ctx context.Context) error { 1343 kvstore.RunLockGC() 1344 return nil 1345 }, 1346 RunInterval: defaults.KVStoreStaleLockTimeout, 1347 }, 1348 ) 1349 1350 // If K8s is enabled we can do the service translation automagically by 1351 // looking at services from k8s and retrieve the service IP from that. 1352 // This makes cilium to not depend on kube dns to interact with etcd 1353 _, isETCDOperator := kvstore.IsEtcdOperator(option.Config.KVStore, option.Config.KVStoreOpt, option.Config.K8sNamespace) 1354 if k8s.IsEnabled() && isETCDOperator { 1355 // Wait services and endpoints cache are synced with k8s before setting 1356 // up etcd so we can perform the name resolution for etcd-operator 1357 // to the service IP as well perform the service -> backend IPs for 1358 // that service IP. 1359 d.waitForCacheSync(k8sAPIGroupServiceV1Core, k8sAPIGroupEndpointV1Core) 1360 log := log.WithField(logfields.LogSubsys, "etcd") 1361 goopts.DialOption = []grpc.DialOption{ 1362 grpc.WithDialer(k8s.CreateCustomDialer(&d.k8sSvcCache, log)), 1363 } 1364 } 1365 1366 if err := kvstore.Setup(option.Config.KVStore, option.Config.KVStoreOpt, goopts); err != nil { 1367 addrkey := fmt.Sprintf("%s.address", option.Config.KVStore) 1368 addr := option.Config.KVStoreOpt[addrkey] 1369 1370 log.WithError(err).WithFields(logrus.Fields{ 1371 "kvstore": option.Config.KVStore, 1372 "address": addr, 1373 }).Fatal("Unable to setup kvstore") 1374 } 1375 } 1376 1377 func runDaemon() { 1378 datapathConfig := linuxdatapath.DatapathConfiguration{ 1379 HostDevice: option.Config.HostDevice, 1380 EncryptInterface: option.Config.EncryptInterface, 1381 } 1382 1383 log.Info("Initializing daemon") 1384 1385 // Since flannel doesn't create the cni0 interface until the first container 1386 // is initialized we need to wait until it is initialized so we can attach 1387 // the BPF program to it. If Cilium is running as a Kubernetes DaemonSet, 1388 // there is also a script waiting for the interface to be created. 1389 if option.Config.IsFlannelMasterDeviceSet() { 1390 err := waitForHostDeviceWhenReady(option.Config.FlannelMasterDevice) 1391 if err != nil { 1392 log.WithError(err).WithFields(logrus.Fields{ 1393 logfields.Interface: option.Config.FlannelMasterDevice, 1394 }).Error("unable to check for host device") 1395 return 1396 } 1397 } 1398 1399 option.Config.RunMonitorAgent = true 1400 1401 if err := enableIPForwarding(); err != nil { 1402 log.WithError(err).Fatal("Error when enabling sysctl parameters") 1403 } 1404 1405 iptablesManager := &iptables.IptablesManager{} 1406 iptablesManager.Init() 1407 1408 d, restoredEndpoints, err := NewDaemon(linuxdatapath.NewDatapath(datapathConfig, iptablesManager), iptablesManager) 1409 if err != nil { 1410 log.WithError(err).Fatal("Error while creating daemon") 1411 return 1412 } 1413 1414 // This validation needs to be done outside of the agent until 1415 // datapath.NodeAddressing is used consistently across the code base. 1416 log.Info("Validating configured node address ranges") 1417 if err := node.ValidatePostInit(); err != nil { 1418 log.WithError(err).Fatal("postinit failed") 1419 } 1420 1421 if option.Config.IsFlannelMasterDeviceSet() && option.Config.FlannelUninstallOnExit { 1422 cleanup.DeferTerminationCleanupFunction(cleanUPWg, cleanUPSig, func() { 1423 d.compilationMutex.Lock() 1424 loader.DeleteDatapath(context.Background(), option.FlannelMasterDevice, "egress") 1425 d.compilationMutex.Unlock() 1426 }) 1427 } 1428 1429 bootstrapStats.enableConntrack.Start() 1430 log.Info("Starting connection tracking garbage collector") 1431 endpointmanager.EnableConntrackGC(option.Config.EnableIPv4, option.Config.EnableIPv6, 1432 restoredEndpoints.restored) 1433 bootstrapStats.enableConntrack.End(true) 1434 1435 endpointmanager.EndpointSynchronizer = &endpointsynchronizer.EndpointSynchronizer{} 1436 1437 bootstrapStats.k8sInit.Start() 1438 1439 // We need to set up etcd in parallel so we will initialize the k8s 1440 // subsystem as well in parallel so caches will start to be synchronized 1441 // with k8s. 1442 k8sCachesSynced := d.initK8sSubsystem() 1443 if option.Config.KVStore == "" { 1444 log.Info("Skipping kvstore configuration") 1445 } else { 1446 d.initKVStore() 1447 } 1448 1449 // Wait only for certain caches, but not all! 1450 // (Check Daemon.initK8sSubsystem() for more info) 1451 <-k8sCachesSynced 1452 bootstrapStats.k8sInit.End(true) 1453 restoreComplete := d.initRestore(restoredEndpoints) 1454 1455 if option.Config.IsFlannelMasterDeviceSet() { 1456 if option.Config.EnableEndpointHealthChecking { 1457 log.Warn("Running Cilium in flannel mode doesn't support endpoint connectivity health checking. Disabling endpoint connectivity health check.") 1458 option.Config.EnableEndpointHealthChecking = false 1459 } 1460 1461 err := node.SetInternalIPv4From(option.Config.FlannelMasterDevice) 1462 if err != nil { 1463 log.WithError(err).WithField("device", option.Config.FlannelMasterDevice).Fatal("Unable to set internal IPv4") 1464 } 1465 if option.Config.FlannelManageExistingContainers { 1466 log.Info("Searching for existing containers...") 1467 d.attachExistingInfraContainers() 1468 } 1469 } 1470 1471 if !option.Config.DryMode { 1472 go func() { 1473 if restoreComplete != nil { 1474 <-restoreComplete 1475 } 1476 d.dnsNameManager.CompleteBootstrap() 1477 maps.CollectStaleMapGarbage() 1478 maps.RemoveDisabledMaps() 1479 }() 1480 } 1481 1482 // The workload event listener *must* be enabled *after* restored endpoints 1483 // are added into the endpoint manager; otherwise, updates to important 1484 // endpoint metadata, such as Kubernetes pod name and namespace, will not 1485 // be performed on the endpoint. 1486 eventsCh, err := workloads.EnableEventListener() 1487 if err != nil { 1488 log.WithError(err).Fatal("Error while enabling workload event watcher") 1489 } else { 1490 d.workloadsEventsCh = eventsCh 1491 } 1492 1493 bootstrapStats.healthCheck.Start() 1494 if option.Config.EnableHealthChecking { 1495 d.initHealth() 1496 } 1497 bootstrapStats.healthCheck.End(true) 1498 1499 d.startStatusCollector() 1500 1501 metricsErrs := initMetrics() 1502 1503 bootstrapStats.initAPI.Start() 1504 api := d.instantiateAPI() 1505 1506 server := server.NewServer(api) 1507 server.EnabledListeners = []string{"unix"} 1508 server.SocketPath = flags.Filename(option.Config.SocketPath) 1509 server.ReadTimeout = apiTimeout 1510 server.WriteTimeout = apiTimeout 1511 defer server.Shutdown() 1512 1513 server.ConfigureAPI() 1514 bootstrapStats.initAPI.End(true) 1515 1516 repr, err := monitorAPI.TimeRepr(time.Now()) 1517 if err != nil { 1518 log.WithError(err).Warn("Failed to generate agent start monitor message") 1519 } else { 1520 d.SendNotification(monitorAPI.AgentNotifyStart, repr) 1521 } 1522 1523 log.WithField("bootstrapTime", time.Since(bootstrapTimestamp)). 1524 Info("Daemon initialization completed") 1525 1526 if option.Config.WriteCNIConfigurationWhenReady != "" { 1527 input, err := ioutil.ReadFile(option.Config.ReadCNIConfiguration) 1528 if err != nil { 1529 log.WithError(err).Fatal("Unable to read CNI configuration file") 1530 } 1531 1532 if err = ioutil.WriteFile(option.Config.WriteCNIConfigurationWhenReady, input, 0644); err != nil { 1533 log.WithError(err).Fatalf("Unable to write CNI configuration file to %s", option.Config.WriteCNIConfigurationWhenReady) 1534 } else { 1535 log.Infof("Wrote CNI configuration file to %s", option.Config.WriteCNIConfigurationWhenReady) 1536 } 1537 } 1538 1539 errs := make(chan error, 1) 1540 1541 go func() { 1542 errs <- server.Serve() 1543 }() 1544 1545 if k8s.IsEnabled() { 1546 bootstrapStats.k8sInit.Start() 1547 k8s.Client().MarkNodeReady(node.GetName()) 1548 bootstrapStats.k8sInit.End(true) 1549 } 1550 1551 bootstrapStats.overall.End(true) 1552 bootstrapStats.updateMetrics() 1553 1554 select { 1555 case err := <-metricsErrs: 1556 if err != nil { 1557 log.WithError(err).Fatal("Cannot start metrics server") 1558 } 1559 case err := <-errs: 1560 if err != nil { 1561 log.WithError(err).Fatal("Error returned from non-returning Serve() call") 1562 } 1563 } 1564 } 1565 1566 func (d *Daemon) instantiateAPI() *restapi.CiliumAPI { 1567 1568 swaggerSpec, err := loads.Analyzed(server.SwaggerJSON, "") 1569 if err != nil { 1570 log.WithError(err).Fatal("Cannot load swagger spec") 1571 } 1572 1573 log.Info("Initializing Cilium API") 1574 api := restapi.NewCiliumAPI(swaggerSpec) 1575 1576 api.Logger = log.Infof 1577 1578 // /healthz/ 1579 api.DaemonGetHealthzHandler = NewGetHealthzHandler(d) 1580 1581 // /cluster/nodes 1582 api.DaemonGetClusterNodesHandler = NewGetClusterNodesHandler(d) 1583 1584 // /config/ 1585 api.DaemonGetConfigHandler = NewGetConfigHandler(d) 1586 api.DaemonPatchConfigHandler = NewPatchConfigHandler(d) 1587 1588 // /endpoint/ 1589 api.EndpointGetEndpointHandler = NewGetEndpointHandler(d) 1590 1591 // /endpoint/{id} 1592 api.EndpointGetEndpointIDHandler = NewGetEndpointIDHandler(d) 1593 api.EndpointPutEndpointIDHandler = NewPutEndpointIDHandler(d) 1594 api.EndpointPatchEndpointIDHandler = NewPatchEndpointIDHandler(d) 1595 api.EndpointDeleteEndpointIDHandler = NewDeleteEndpointIDHandler(d) 1596 1597 // /endpoint/{id}config/ 1598 api.EndpointGetEndpointIDConfigHandler = NewGetEndpointIDConfigHandler(d) 1599 api.EndpointPatchEndpointIDConfigHandler = NewPatchEndpointIDConfigHandler(d) 1600 1601 // /endpoint/{id}/labels/ 1602 api.EndpointGetEndpointIDLabelsHandler = NewGetEndpointIDLabelsHandler(d) 1603 api.EndpointPatchEndpointIDLabelsHandler = NewPatchEndpointIDLabelsHandler(d) 1604 1605 // /endpoint/{id}/log/ 1606 api.EndpointGetEndpointIDLogHandler = NewGetEndpointIDLogHandler(d) 1607 1608 // /endpoint/{id}/healthz 1609 api.EndpointGetEndpointIDHealthzHandler = NewGetEndpointIDHealthzHandler(d) 1610 1611 // /identity/ 1612 api.PolicyGetIdentityHandler = newGetIdentityHandler(d) 1613 api.PolicyGetIdentityIDHandler = newGetIdentityIDHandler(d) 1614 1615 // /identity/endpoints 1616 api.PolicyGetIdentityEndpointsHandler = newGetIdentityEndpointsIDHandler(d) 1617 1618 // /policy/ 1619 api.PolicyGetPolicyHandler = newGetPolicyHandler(d) 1620 api.PolicyPutPolicyHandler = newPutPolicyHandler(d) 1621 api.PolicyDeletePolicyHandler = newDeletePolicyHandler(d) 1622 api.PolicyGetPolicySelectorsHandler = newGetPolicyCacheHandler(d) 1623 1624 // /policy/resolve/ 1625 api.PolicyGetPolicyResolveHandler = NewGetPolicyResolveHandler(d) 1626 1627 // /service/{id}/ 1628 api.ServiceGetServiceIDHandler = NewGetServiceIDHandler(d) 1629 api.ServiceDeleteServiceIDHandler = NewDeleteServiceIDHandler(d) 1630 api.ServicePutServiceIDHandler = NewPutServiceIDHandler(d) 1631 1632 // /service/ 1633 api.ServiceGetServiceHandler = NewGetServiceHandler(d) 1634 1635 // /prefilter/ 1636 api.PrefilterGetPrefilterHandler = NewGetPrefilterHandler(d) 1637 api.PrefilterDeletePrefilterHandler = NewDeletePrefilterHandler(d) 1638 api.PrefilterPatchPrefilterHandler = NewPatchPrefilterHandler(d) 1639 1640 // /ipam/{ip}/ 1641 api.IPAMPostIPAMHandler = NewPostIPAMHandler(d) 1642 api.IPAMPostIPAMIPHandler = NewPostIPAMIPHandler(d) 1643 api.IPAMDeleteIPAMIPHandler = NewDeleteIPAMIPHandler(d) 1644 1645 // /debuginfo 1646 api.DaemonGetDebuginfoHandler = NewGetDebugInfoHandler(d) 1647 1648 // /map 1649 api.DaemonGetMapHandler = NewGetMapHandler(d) 1650 api.DaemonGetMapNameHandler = NewGetMapNameHandler(d) 1651 1652 // metrics 1653 api.MetricsGetMetricsHandler = NewGetMetricsHandler(d) 1654 1655 // /fqdn/cache 1656 api.PolicyGetFqdnCacheHandler = NewGetFqdnCacheHandler(d) 1657 api.PolicyDeleteFqdnCacheHandler = NewDeleteFqdnCacheHandler(d) 1658 api.PolicyGetFqdnCacheIDHandler = NewGetFqdnCacheIDHandler(d) 1659 1660 return api 1661 }