github.com/looshlee/beatles@v0.0.0-20220727174639-742810ab631c/daemon/daemon_main.go (about) 1 // Copyright 2016-2019 Authors of Cilium 2 // 3 // Licensed under the Apache License, Version 2.0 (the "License"); 4 // you may not use this file except in compliance with the License. 5 // You may obtain a copy of the License at 6 // 7 // http://www.apache.org/licenses/LICENSE-2.0 8 // 9 // Unless required by applicable law or agreed to in writing, software 10 // distributed under the License is distributed on an "AS IS" BASIS, 11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 // See the License for the specific language governing permissions and 13 // limitations under the License. 14 15 package main 16 17 import ( 18 "context" 19 "errors" 20 "fmt" 21 "io/ioutil" 22 "net" 23 "os" 24 "os/exec" 25 "path" 26 "path/filepath" 27 "regexp" 28 "strings" 29 "time" 30 31 "github.com/cilium/cilium/api/v1/server" 32 "github.com/cilium/cilium/api/v1/server/restapi" 33 "github.com/cilium/cilium/common" 34 _ "github.com/cilium/cilium/pkg/alignchecker" 35 "github.com/cilium/cilium/pkg/bpf" 36 "github.com/cilium/cilium/pkg/cgroups" 37 "github.com/cilium/cilium/pkg/cleanup" 38 "github.com/cilium/cilium/pkg/components" 39 "github.com/cilium/cilium/pkg/controller" 40 "github.com/cilium/cilium/pkg/datapath/iptables" 41 linuxdatapath "github.com/cilium/cilium/pkg/datapath/linux" 42 "github.com/cilium/cilium/pkg/datapath/loader" 43 "github.com/cilium/cilium/pkg/datapath/maps" 44 "github.com/cilium/cilium/pkg/defaults" 45 "github.com/cilium/cilium/pkg/endpointmanager" 46 "github.com/cilium/cilium/pkg/envoy" 47 "github.com/cilium/cilium/pkg/flowdebug" 48 "github.com/cilium/cilium/pkg/identity" 49 "github.com/cilium/cilium/pkg/identity/cache" 50 "github.com/cilium/cilium/pkg/k8s" 51 "github.com/cilium/cilium/pkg/k8s/endpointsynchronizer" 52 "github.com/cilium/cilium/pkg/kvstore" 53 "github.com/cilium/cilium/pkg/labels" 54 "github.com/cilium/cilium/pkg/loadinfo" 55 "github.com/cilium/cilium/pkg/logging" 56 "github.com/cilium/cilium/pkg/logging/logfields" 57 ipcachemap "github.com/cilium/cilium/pkg/maps/ipcache" 58 "github.com/cilium/cilium/pkg/metrics" 59 monitorAPI "github.com/cilium/cilium/pkg/monitor/api" 60 "github.com/cilium/cilium/pkg/node" 61 "github.com/cilium/cilium/pkg/option" 62 "github.com/cilium/cilium/pkg/pidfile" 63 "github.com/cilium/cilium/pkg/policy" 64 "github.com/cilium/cilium/pkg/pprof" 65 "github.com/cilium/cilium/pkg/service" 66 "github.com/cilium/cilium/pkg/version" 67 "github.com/cilium/cilium/pkg/versioncheck" 68 "github.com/cilium/cilium/pkg/workloads" 69 70 "github.com/go-openapi/loads" 71 gops "github.com/google/gops/agent" 72 go_version "github.com/hashicorp/go-version" 73 "github.com/jessevdk/go-flags" 74 "github.com/sirupsen/logrus" 75 "github.com/spf13/cobra" 76 "github.com/spf13/viper" 77 "github.com/vishvananda/netlink" 78 "golang.org/x/sys/unix" 79 "google.golang.org/grpc" 80 ) 81 82 var ( 83 minKernelVer = versioncheck.MustCompile(">= 4.8.0") 84 minClangVer = versioncheck.MustCompile(">= 3.8.0") 85 86 recKernelVer = versioncheck.MustCompile(">= 4.9.0") 87 recClangVer = versioncheck.MustCompile(">= 3.9.0") 88 89 // LLVM/clang version which supports `-mattr=dwarfris` 90 dwarfrisClangVer = versioncheck.MustCompile(">= 7.0.0") 91 canDisableDwarfRelocations bool 92 ) 93 94 const ( 95 // list of supported verbose debug groups 96 argDebugVerboseFlow = "flow" 97 argDebugVerboseKvstore = "kvstore" 98 argDebugVerboseEnvoy = "envoy" 99 100 apiTimeout = 60 * time.Second 101 daemonSubsys = "daemon" 102 103 // fatalSleep is the duration Cilium should sleep before existing in case 104 // of a log.Fatal is issued or a CLI flag is specified but does not exist. 105 fatalSleep = 2 * time.Second 106 ) 107 108 var ( 109 log = logging.DefaultLogger.WithField(logfields.LogSubsys, daemonSubsys) 110 111 bootstrapTimestamp = time.Now() 112 113 // RootCmd represents the base command when called without any subcommands 114 RootCmd = &cobra.Command{ 115 Use: "cilium-agent", 116 Short: "Run the cilium agent", 117 Run: func(cmd *cobra.Command, args []string) { 118 bootstrapStats.earlyInit.Start() 119 initEnv(cmd) 120 bootstrapStats.earlyInit.End(true) 121 runDaemon() 122 }, 123 } 124 125 bootstrapStats = bootstrapStatistics{} 126 ) 127 128 func init() { 129 RootCmd.SetFlagErrorFunc(func(_ *cobra.Command, e error) error { 130 time.Sleep(fatalSleep) 131 return e 132 }) 133 logrus.RegisterExitHandler(func() { 134 time.Sleep(fatalSleep) 135 }, 136 ) 137 } 138 139 func daemonMain() { 140 bootstrapStats.overall.Start() 141 142 // Open socket for using gops to get stacktraces of the agent. 143 if err := gops.Listen(gops.Options{}); err != nil { 144 errorString := fmt.Sprintf("unable to start gops: %s", err) 145 fmt.Println(errorString) 146 os.Exit(-1) 147 } 148 interruptCh := registerSigHandler() 149 if err := RootCmd.Execute(); err != nil { 150 fmt.Println(err) 151 os.Exit(-1) 152 } 153 <-interruptCh 154 os.Exit(0) 155 } 156 157 func parseKernelVersion(ver string) (*go_version.Version, error) { 158 verStrs := strings.Split(ver, ".") 159 switch { 160 case len(verStrs) < 2: 161 return nil, fmt.Errorf("unable to get kernel version from %q", ver) 162 case len(verStrs) < 3: 163 verStrs = append(verStrs, "0") 164 } 165 // We are assuming the kernel version will be something as: 166 // 4.9.17-040917-generic 167 168 // If verStrs is []string{ "4", "9", "17-040917-generic" } 169 // then we need to retrieve patch number. 170 patch := regexp.MustCompilePOSIX(`^[0-9]+`).FindString(verStrs[2]) 171 if patch == "" { 172 verStrs[2] = "0" 173 } else { 174 verStrs[2] = patch 175 } 176 return go_version.NewVersion(strings.Join(verStrs[:3], ".")) 177 } 178 179 func getKernelVersion() (*go_version.Version, error) { 180 var unameBuf unix.Utsname 181 if err := unix.Uname(&unameBuf); err != nil { 182 log.WithError(err).Fatal("kernel version: NOT OK") 183 } 184 return parseKernelVersion(string(unameBuf.Release[:])) 185 } 186 187 func getClangVersion(filePath string) (*go_version.Version, error) { 188 verOut, err := exec.Command(filePath, "--version").CombinedOutput() 189 if err != nil { 190 log.WithError(err).Fatal("clang version: NOT OK") 191 } 192 res := regexp.MustCompile(`(clang version )([^ ]*)`).FindStringSubmatch(string(verOut)) 193 if len(res) != 3 { 194 log.Fatalf("clang version: NOT OK: unable to get clang's version "+ 195 "from: %q", string(verOut)) 196 } 197 // at this point res is []string{"clang", "version", "maj.min.patch"} 198 verStrs := strings.Split(res[2], ".") 199 if len(verStrs) < 3 { 200 return nil, fmt.Errorf("unable to get clang version from %q", string(verOut)) 201 } 202 v := strings.Join(verStrs[:3], ".") 203 // Handle Ubuntu versioning by removing the dash and everything after. 204 // F. ex. `4.0.0-1ubuntu1~16 -> 4.0.0` and `3.8.0-2ubuntu4 -> 3.8.0`. 205 v = strings.Split(v, "-")[0] 206 return go_version.NewVersion(v) 207 } 208 209 func checkBPFLogs(logType string, fatal bool) { 210 bpfLogFile := logType + ".log" 211 bpfLogPath := filepath.Join(option.Config.StateDir, bpfLogFile) 212 213 if _, err := os.Stat(bpfLogPath); os.IsNotExist(err) { 214 log.Infof("%s check: OK!", logType) 215 } else if err == nil { 216 bpfFeaturesLog, err := ioutil.ReadFile(bpfLogPath) 217 if err != nil { 218 log.WithError(err).WithField(logfields.Path, bpfLogPath).Fatalf("%s check: NOT OK. Unable to read", logType) 219 } 220 printer := log.Debugf 221 if fatal { 222 printer = log.Errorf 223 printer("%s check: NOT OK", logType) 224 } else { 225 printer("%s check: Some features may be limited:", logType) 226 } 227 lines := strings.Trim(string(bpfFeaturesLog), "\n") 228 for _, line := range strings.Split(lines, "\n") { 229 printer(line) 230 } 231 if fatal { 232 log.Fatalf("%s check failed.", logType) 233 } 234 } else { 235 log.WithError(err).WithField(logfields.Path, bpfLogPath).Fatalf("%s check: NOT OK. Unable to read", logType) 236 } 237 } 238 239 func checkMinRequirements() { 240 kernelVersion, err := getKernelVersion() 241 if err != nil { 242 log.WithError(err).Fatal("kernel version: NOT OK") 243 } 244 if !minKernelVer.Check(kernelVersion) { 245 log.Fatalf("kernel version: NOT OK: minimal supported kernel "+ 246 "version is %s; kernel version that is running is: %s", minKernelVer, kernelVersion) 247 } 248 249 if option.Config.EnableIPv6 { 250 if _, err := os.Stat("/proc/net/if_inet6"); os.IsNotExist(err) { 251 log.Fatalf("kernel: ipv6 is enabled in agent but ipv6 is either disabled or not compiled in the kernel") 252 } 253 } 254 255 if filePath, err := exec.LookPath("clang"); err != nil { 256 log.WithError(err).Fatal("clang: NOT OK") 257 } else { 258 clangVersion, err := getClangVersion(filePath) 259 if err != nil { 260 log.WithError(err).Fatal("clang: NOT OK") 261 } 262 if !minClangVer.Check(clangVersion) { 263 log.Fatalf("clang version: NOT OK: minimal supported clang "+ 264 "version is %s; clang version that is running is: %s", minClangVer, clangVersion) 265 } 266 //clang >= 3.9 / kernel < 4.9 - does not work 267 if recClangVer.Check(clangVersion) && !recKernelVer.Check(kernelVersion) { 268 log.Fatalf("clang (%s) and kernel (%s) version: NOT OK: please upgrade "+ 269 "your kernel version to at least %s", 270 clangVersion, kernelVersion, recKernelVer) 271 } 272 canDisableDwarfRelocations = dwarfrisClangVer.Check(clangVersion) 273 log.Infof("clang (%s) and kernel (%s) versions: OK!", clangVersion, kernelVersion) 274 } 275 276 if filePath, err := exec.LookPath("llc"); err != nil { 277 log.WithError(err).Fatal("llc: NOT OK") 278 } else { 279 lccVersion, err := exec.Command(filePath, "--version").CombinedOutput() 280 if err == nil { 281 if strings.Contains(strings.ToLower(string(lccVersion)), "debug") { 282 log.Warn("llc version was compiled in debug mode, expect higher latency!") 283 } 284 } 285 // /usr/include/gnu/stubs-32.h is installed by 'glibc-devel.i686' in fedora 286 // /usr/include/sys/cdefs.h is installed by 'libc6-dev-i386' in ubuntu 287 // both files exist on both systems but cdefs.h already exists in fedora 288 // without 'glibc-devel.i686' so we check for 'stubs-32.h first. 289 if _, err := os.Stat("/usr/include/gnu/stubs-32.h"); os.IsNotExist(err) { 290 log.Fatal("linking environment: NOT OK, please make sure you have 'glibc-devel.i686' if you use fedora system or 'libc6-dev-i386' if you use ubuntu system") 291 } 292 if _, err := os.Stat("/usr/include/sys/cdefs.h"); os.IsNotExist(err) { 293 log.Fatal("linking environment: NOT OK, please make sure you have 'libc6-dev-i386' in your ubuntu system") 294 } 295 log.Info("linking environment: OK!") 296 } 297 298 globalsDir := option.Config.GetGlobalsDir() 299 if err := os.MkdirAll(globalsDir, defaults.StateDirRights); err != nil { 300 log.WithError(err).WithField(logfields.Path, globalsDir).Fatal("Could not create runtime directory") 301 } 302 if err := os.Chdir(option.Config.LibDir); err != nil { 303 log.WithError(err).WithField(logfields.Path, option.Config.LibDir).Fatal("Could not change to runtime directory") 304 } 305 probeScript := filepath.Join(option.Config.BpfDir, "run_probes.sh") 306 if err := exec.Command(probeScript, option.Config.BpfDir, option.Config.StateDir).Run(); err != nil { 307 log.WithError(err).Fatal("BPF Verifier: NOT OK. Unable to run checker for bpf_features") 308 } 309 featuresFilePath := filepath.Join(globalsDir, "bpf_features.h") 310 if _, err := os.Stat(featuresFilePath); os.IsNotExist(err) { 311 log.WithError(err).WithField(logfields.Path, globalsDir).Fatal("BPF Verifier: NOT OK. Unable to read bpf_features.h") 312 } 313 314 checkBPFLogs("bpf_requirements", true) 315 checkBPFLogs("bpf_features", false) 316 bpf.ReadFeatureProbes(featuresFilePath) 317 } 318 319 func skipInit(basePath string) bool { 320 switch basePath { 321 case components.CiliumAgentName, components.CiliumDaemonTestName: 322 return false 323 default: 324 return true 325 } 326 } 327 328 func init() { 329 if skipInit(path.Base(os.Args[0])) { 330 log.Debug("Skipping preparation of cilium-agent environment") 331 return 332 } 333 334 cobra.OnInitialize(initConfig) 335 336 // Reset the help function to also exit, as we block elsewhere in interrupts 337 // and would not exit when called with -h. 338 oldHelpFunc := RootCmd.HelpFunc() 339 RootCmd.SetHelpFunc(func(c *cobra.Command, a []string) { 340 oldHelpFunc(c, a) 341 os.Exit(0) 342 }) 343 344 flags := RootCmd.Flags() 345 346 // Validators 347 option.Config.FixedIdentityMappingValidator = option.Validator(func(val string) (string, error) { 348 vals := strings.Split(val, "=") 349 if len(vals) != 2 { 350 return "", fmt.Errorf(`invalid fixed identity: expecting "<numeric-identity>=<identity-name>" got %q`, val) 351 } 352 ni, err := identity.ParseNumericIdentity(vals[0]) 353 if err != nil { 354 return "", fmt.Errorf(`invalid numeric identity %q: %s`, val, err) 355 } 356 if !identity.IsUserReservedIdentity(ni) { 357 return "", fmt.Errorf(`invalid numeric identity %q: valid numeric identity is between %d and %d`, 358 val, identity.UserReservedNumericIdentity.Uint32(), identity.MinimalNumericIdentity.Uint32()) 359 } 360 lblStr := vals[1] 361 lbl := labels.ParseLabel(lblStr) 362 if lbl.IsReservedSource() { 363 return "", fmt.Errorf(`invalid source %q for label: %s`, labels.LabelSourceReserved, lblStr) 364 } 365 return val, nil 366 }) 367 368 // Env bindings 369 flags.String(option.AccessLog, "", "Path to access log of supported L7 requests observed") 370 option.BindEnv(option.AccessLog) 371 372 flags.StringSlice(option.AgentLabels, []string{}, "Additional labels to identify this agent") 373 option.BindEnv(option.AgentLabels) 374 375 flags.String(option.AllowLocalhost, option.AllowLocalhostAuto, "Policy when to allow local stack to reach local endpoints { auto | always | policy }") 376 option.BindEnv(option.AllowLocalhost) 377 378 flags.Bool(option.AnnotateK8sNode, defaults.AnnotateK8sNode, "Annotate Kubernetes node") 379 option.BindEnv(option.AnnotateK8sNode) 380 381 flags.Bool(option.BlacklistConflictingRoutes, defaults.BlacklistConflictingRoutes, "Don't blacklist IP allocations conflicting with local non-cilium routes") 382 option.BindEnv(option.BlacklistConflictingRoutes) 383 384 flags.Bool(option.AutoCreateCiliumNodeResource, defaults.AutoCreateCiliumNodeResource, "Automatically create CiliumNode resource for own node on startup") 385 option.BindEnv(option.AutoCreateCiliumNodeResource) 386 387 flags.String(option.BPFRoot, "", "Path to BPF filesystem") 388 option.BindEnv(option.BPFRoot) 389 390 flags.String(option.CGroupRoot, "", "Path to Cgroup2 filesystem") 391 option.BindEnv(option.CGroupRoot) 392 393 flags.Bool(option.BPFCompileDebugName, false, "Enable debugging of the BPF compilation process") 394 option.BindEnv(option.BPFCompileDebugName) 395 396 flags.Bool(option.SockopsEnableName, defaults.SockopsEnable, "Enable sockops when kernel supported") 397 option.BindEnv(option.SockopsEnableName) 398 399 flags.Int(option.ClusterIDName, 0, "Unique identifier of the cluster") 400 option.BindEnv(option.ClusterIDName) 401 402 flags.String(option.ClusterName, defaults.ClusterName, "Name of the cluster") 403 option.BindEnv(option.ClusterName) 404 405 flags.String(option.ClusterMeshConfigName, "", "Path to the ClusterMesh configuration directory") 406 option.BindEnv(option.ClusterMeshConfigName) 407 408 flags.String(option.ConfigFile, "", `Configuration file (default "$HOME/ciliumd.yaml")`) 409 option.BindEnv(option.ConfigFile) 410 411 flags.String(option.ConfigDir, "", `Configuration directory that contains a file for each option`) 412 option.BindEnv(option.ConfigDir) 413 414 flags.Uint(option.ConntrackGarbageCollectorIntervalDeprecated, 0, "Garbage collection interval for the connection tracking table (in seconds)") 415 flags.MarkDeprecated(option.ConntrackGarbageCollectorIntervalDeprecated, fmt.Sprintf("please use --%s", option.ConntrackGCInterval)) 416 option.BindEnv(option.ConntrackGarbageCollectorIntervalDeprecated) 417 418 flags.Duration(option.ConntrackGCInterval, time.Duration(0), "Overwrite the connection-tracking garbage collection interval") 419 option.BindEnv(option.ConntrackGCInterval) 420 421 flags.StringSlice(option.ContainerRuntime, option.ContainerRuntimeAuto, `Sets the container runtime(s) used by Cilium { containerd | crio | docker | none | auto } ( "auto" uses the container runtime found in the order: "docker", "containerd", "crio" )`) 422 option.BindEnv(option.ContainerRuntime) 423 424 flags.Var(option.NewNamedMapOptions(option.ContainerRuntimeEndpoint, &option.Config.ContainerRuntimeEndpoint, nil), 425 option.ContainerRuntimeEndpoint, `Container runtime(s) endpoint(s). (default: `+workloads.GetDefaultEPOptsStringWithPrefix("--container-runtime-endpoint=")+`)`) 426 option.BindEnv(option.ContainerRuntimeEndpoint) 427 428 flags.BoolP(option.DebugArg, "D", false, "Enable debugging mode") 429 option.BindEnv(option.DebugArg) 430 431 flags.StringSlice(option.DebugVerbose, []string{}, "List of enabled verbose debug groups") 432 option.BindEnv(option.DebugVerbose) 433 434 flags.StringP(option.Device, "d", "undefined", "Device facing cluster/external network for direct L3 (non-overlay mode)") 435 option.BindEnv(option.Device) 436 437 flags.String(option.DatapathMode, defaults.DatapathMode, "Datapath mode name") 438 option.BindEnv(option.DatapathMode) 439 440 flags.StringP(option.IpvlanMasterDevice, "", "undefined", "Device facing external network acting as ipvlan master") 441 option.BindEnv(option.IpvlanMasterDevice) 442 443 flags.Bool(option.DisableConntrack, false, "Disable connection tracking") 444 option.BindEnv(option.DisableConntrack) 445 446 flags.Bool(option.LegacyDisableIPv4Name, false, "Disable IPv4 mode") 447 flags.MarkHidden(option.LegacyDisableIPv4Name) 448 option.BindEnv(option.LegacyDisableIPv4Name) 449 450 flags.Bool(option.EnableEndpointRoutes, defaults.EnableEndpointRoutes, "Use per endpoint routes instead of routing via cilium_host") 451 option.BindEnv(option.EnableEndpointRoutes) 452 453 flags.Bool(option.EnableHealthChecking, defaults.EnableHealthChecking, "Enable connectivity health checking") 454 option.BindEnv(option.EnableHealthChecking) 455 456 flags.Bool(option.EnableEndpointHealthChecking, defaults.EnableEndpointHealthChecking, "Enable connectivity health checking between virtual endpoints") 457 option.BindEnv(option.EnableEndpointHealthChecking) 458 459 flags.Bool(option.EnableIPv4Name, defaults.EnableIPv4, "Enable IPv4 support") 460 option.BindEnv(option.EnableIPv4Name) 461 462 flags.Bool(option.EnableIPv6Name, defaults.EnableIPv6, "Enable IPv6 support") 463 option.BindEnv(option.EnableIPv6Name) 464 465 flags.String(option.EncryptInterface, "", "Transparent encryption interface") 466 option.BindEnv(option.EncryptInterface) 467 468 flags.Bool(option.EncryptNode, defaults.EncryptNode, "Enables encrypting traffic from non-Cilium pods and host networking") 469 option.BindEnv(option.EncryptNode) 470 471 flags.StringSlice(option.IPv4PodSubnets, []string{}, "List of IPv4 pod subnets to preconfigure for encryption") 472 option.BindEnv(option.IPv4PodSubnets) 473 474 flags.StringSlice(option.IPv6PodSubnets, []string{}, "List of IPv6 pod subnets to preconfigure for encryption") 475 option.BindEnv(option.IPv6PodSubnets) 476 477 flags.String(option.EndpointInterfaceNamePrefix, defaults.EndpointInterfaceNamePrefix, "Prefix of interface name shared by all endpoints") 478 option.BindEnv(option.EndpointInterfaceNamePrefix) 479 480 flags.StringSlice(option.ExcludeLocalAddress, []string{}, "Exclude CIDR from being recognized as local address") 481 option.BindEnv(option.ExcludeLocalAddress) 482 483 flags.Bool(option.DisableCiliumEndpointCRDName, false, "Disable use of CiliumEndpoint CRD") 484 option.BindEnv(option.DisableCiliumEndpointCRDName) 485 486 flags.Bool(option.DisableK8sServices, false, "Disable east-west K8s load balancing by cilium") 487 option.BindEnv(option.DisableK8sServices) 488 489 flags.String(option.EgressMasqueradeInterfaces, "", "Limit egress masquerading to interface selector") 490 option.BindEnv(option.EgressMasqueradeInterfaces) 491 492 flags.Bool(option.EnableHostReachableServices, false, "Enable reachability of services for host applications (beta)") 493 option.BindEnv(option.EnableHostReachableServices) 494 495 flags.StringSlice(option.HostReachableServicesProtos, []string{option.HostServicesTCP, option.HostServicesUDP}, "Only enable reachability of services for host applications for specific protocols") 496 option.BindEnv(option.HostReachableServicesProtos) 497 498 flags.Bool(option.DeprecatedEnableLegacyServices, false, "Enable legacy (prior-v1.5) services") 499 flags.MarkDeprecated(option.DeprecatedEnableLegacyServices, "this option is deprecated as of v1.6") 500 option.BindEnv(option.DeprecatedEnableLegacyServices) 501 502 flags.StringP(option.Docker, "e", workloads.GetRuntimeDefaultOpt(workloads.Docker, "endpoint"), "Path to docker runtime socket (DEPRECATED: use container-runtime-endpoint instead)") 503 option.BindEnv(option.Docker) 504 505 flags.Bool(option.EnableAutoDirectRoutingName, defaults.EnableAutoDirectRouting, "Enable automatic L2 routing between nodes") 506 option.BindEnv(option.EnableAutoDirectRoutingName) 507 508 flags.Bool(option.EnableXTSocketFallbackName, defaults.EnableXTSocketFallback, "Enable fallback for missing xt_socket module") 509 option.BindEnv(option.EnableXTSocketFallbackName) 510 511 flags.String(option.EnablePolicy, option.DefaultEnforcement, "Enable policy enforcement") 512 option.BindEnv(option.EnablePolicy) 513 514 flags.Bool(option.EnableL7Proxy, defaults.EnableL7Proxy, "Enable L7 proxy for L7 policy enforcement") 515 option.BindEnv(option.EnableL7Proxy) 516 517 flags.Bool(option.EnableTracing, false, "Enable tracing while determining policy (debugging)") 518 option.BindEnv(option.EnableTracing) 519 520 flags.String(option.EnvoyLog, "", "Path to a separate Envoy log file, if any") 521 option.BindEnv(option.EnvoyLog) 522 523 flags.Bool(option.EnableIPSecName, defaults.EnableIPSec, "Enable IPSec support") 524 option.BindEnv(option.EnableIPSecName) 525 526 flags.String(option.IPSecKeyFileName, "", "Path to IPSec key file") 527 option.BindEnv(option.IPSecKeyFileName) 528 529 flags.Bool(option.ForceLocalPolicyEvalAtSource, defaults.ForceLocalPolicyEvalAtSource, "Force policy evaluation of all local communication at the source endpoint") 530 option.BindEnv(option.ForceLocalPolicyEvalAtSource) 531 532 flags.String(option.HTTP403Message, "", "Message returned in proxy L7 403 body") 533 flags.MarkHidden(option.HTTP403Message) 534 option.BindEnv(option.HTTP403Message) 535 536 flags.Uint(option.HTTPRequestTimeout, 60*60, "Time after which a forwarded HTTP request is considered failed unless completed (in seconds); Use 0 for unlimited") 537 option.BindEnv(option.HTTPRequestTimeout) 538 539 flags.Uint(option.HTTPIdleTimeout, 0, "Time after which a non-gRPC HTTP stream is considered failed unless traffic in the stream has been processed (in seconds); defaults to 0 (unlimited)") 540 option.BindEnv(option.HTTPIdleTimeout) 541 542 flags.Uint(option.HTTPMaxGRPCTimeout, 0, "Time after which a forwarded gRPC request is considered failed unless completed (in seconds). A \"grpc-timeout\" header may override this with a shorter value; defaults to 0 (unlimited)") 543 option.BindEnv(option.HTTPMaxGRPCTimeout) 544 545 flags.Uint(option.HTTPRetryCount, 3, "Number of retries performed after a forwarded request attempt fails") 546 option.BindEnv(option.HTTPRetryCount) 547 548 flags.Uint(option.HTTPRetryTimeout, 0, "Time after which a forwarded but uncompleted request is retried (connection failures are retried immediately); defaults to 0 (never)") 549 option.BindEnv(option.HTTPRetryTimeout) 550 551 flags.Uint(option.ProxyConnectTimeout, 1, "Time after which a TCP connect attempt is considered failed unless completed (in seconds)") 552 option.BindEnv(option.ProxyConnectTimeout) 553 554 flags.Bool(option.DisableEnvoyVersionCheck, false, "Do not perform Envoy binary version check on startup") 555 flags.MarkHidden(option.DisableEnvoyVersionCheck) 556 // Disable version check if Envoy build is disabled 557 option.BindEnvWithLegacyEnvFallback(option.DisableEnvoyVersionCheck, "CILIUM_DISABLE_ENVOY_BUILD") 558 559 flags.Var(option.NewNamedMapOptions(option.FixedIdentityMapping, &option.Config.FixedIdentityMapping, option.Config.FixedIdentityMappingValidator), 560 option.FixedIdentityMapping, "Key-value for the fixed identity mapping which allows to use reserved label for fixed identities") 561 option.BindEnv(option.FixedIdentityMapping) 562 563 flags.Duration(option.IdentityChangeGracePeriod, defaults.IdentityChangeGracePeriod, "Time to wait before using new identity on endpoint identity change") 564 option.BindEnv(option.IdentityChangeGracePeriod) 565 566 flags.String(option.IdentityAllocationMode, option.IdentityAllocationModeKVstore, "Method to use for identity allocation") 567 option.BindEnv(option.IdentityAllocationMode) 568 569 flags.String(option.IPAM, "", "Backend to use for IPAM") 570 option.BindEnv(option.IPAM) 571 572 flags.Int(option.IPv4ClusterCIDRMaskSize, 8, "Mask size for the cluster wide CIDR") 573 option.BindEnv(option.IPv4ClusterCIDRMaskSize) 574 575 flags.String(option.IPv4Range, AutoCIDR, "Per-node IPv4 endpoint prefix, e.g. 10.16.0.0/16") 576 option.BindEnv(option.IPv4Range) 577 578 flags.String(option.IPv6Range, AutoCIDR, "Per-node IPv6 endpoint prefix, must be /96, e.g. fd02:1:1::/96") 579 option.BindEnv(option.IPv6Range) 580 581 flags.String(option.IPv6ClusterAllocCIDRName, defaults.IPv6ClusterAllocCIDR, "IPv6 /64 CIDR used to allocate per node endpoint /96 CIDR") 582 option.BindEnv(option.IPv6ClusterAllocCIDRName) 583 584 flags.String(option.IPv4ServiceRange, AutoCIDR, "Kubernetes IPv4 services CIDR if not inside cluster prefix") 585 option.BindEnv(option.IPv4ServiceRange) 586 587 flags.String(option.IPv6ServiceRange, AutoCIDR, "Kubernetes IPv6 services CIDR if not inside cluster prefix") 588 option.BindEnv(option.IPv6ServiceRange) 589 590 flags.Bool(option.K8sEventHandover, defaults.K8sEventHandover, "Enable k8s event handover to kvstore for improved scalability") 591 option.BindEnv(option.K8sEventHandover) 592 593 flags.String(option.K8sAPIServer, "", "Kubernetes api address server (for https use --k8s-kubeconfig-path instead)") 594 option.BindEnv(option.K8sAPIServer) 595 596 flags.String(option.K8sKubeConfigPath, "", "Absolute path of the kubernetes kubeconfig file") 597 option.BindEnv(option.K8sKubeConfigPath) 598 599 flags.String(option.K8sNamespaceName, "", "Name of the Kubernetes namespace in which Cilium is deployed in") 600 flags.MarkHidden(option.K8sNamespaceName) 601 option.BindEnv(option.K8sNamespaceName) 602 603 flags.Bool(option.K8sRequireIPv4PodCIDRName, false, "Require IPv4 PodCIDR to be specified in node resource") 604 option.BindEnv(option.K8sRequireIPv4PodCIDRName) 605 606 flags.Bool(option.K8sRequireIPv6PodCIDRName, false, "Require IPv6 PodCIDR to be specified in node resource") 607 option.BindEnv(option.K8sRequireIPv6PodCIDRName) 608 609 flags.Uint(option.K8sServiceCacheSize, defaults.K8sServiceCacheSize, "Cilium service cache size for kubernetes") 610 option.BindEnv(option.K8sServiceCacheSize) 611 flags.MarkHidden(option.K8sServiceCacheSize) 612 613 flags.Bool(option.K8sForceJSONPatch, false, "When set uses JSON Patch to update CNP and CEP status in kube-apiserver") 614 option.BindEnv(option.K8sForceJSONPatch) 615 flags.MarkHidden(option.K8sForceJSONPatch) 616 617 flags.String(option.K8sWatcherEndpointSelector, defaults.K8sWatcherEndpointSelector, "K8s endpoint watcher will watch for these k8s endpoints") 618 option.BindEnv(option.K8sWatcherEndpointSelector) 619 620 flags.Bool(option.KeepConfig, false, "When restoring state, keeps containers' configuration in place") 621 option.BindEnv(option.KeepConfig) 622 623 flags.Bool(option.KeepBPFTemplates, false, "Do not restore BPF template files from binary") 624 option.BindEnv(option.KeepBPFTemplates) 625 626 flags.String(option.KVStore, "", "Key-value store type") 627 option.BindEnv(option.KVStore) 628 629 flags.Duration(option.KVstoreLeaseTTL, defaults.KVstoreLeaseTTL, "Time-to-live for the KVstore lease.") 630 flags.MarkHidden(option.KVstoreLeaseTTL) 631 option.BindEnv(option.KVstoreLeaseTTL) 632 633 flags.Duration(option.KVstorePeriodicSync, defaults.KVstorePeriodicSync, "Periodic KVstore synchronization interval") 634 option.BindEnv(option.KVstorePeriodicSync) 635 636 flags.Duration(option.KVstoreConnectivityTimeout, defaults.KVstoreConnectivityTimeout, "Time after which an incomplete kvstore operation is considered failed") 637 option.BindEnv(option.KVstoreConnectivityTimeout) 638 639 flags.Duration(option.IPAllocationTimeout, defaults.IPAllocationTimeout, "Time after which an incomplete CIDR allocation is considered failed") 640 option.BindEnv(option.IPAllocationTimeout) 641 642 flags.Var(option.NewNamedMapOptions(option.KVStoreOpt, &option.Config.KVStoreOpt, nil), 643 option.KVStoreOpt, "Key-value store options") 644 option.BindEnv(option.KVStoreOpt) 645 646 flags.Uint(option.K8sWatcherQueueSize, 1024, "Queue size used to serialize each k8s event type") 647 option.BindEnv(option.K8sWatcherQueueSize) 648 649 flags.String(option.LabelPrefixFile, "", "Valid label prefixes file path") 650 option.BindEnv(option.LabelPrefixFile) 651 652 flags.StringSlice(option.Labels, []string{}, "List of label prefixes used to determine identity of an endpoint") 653 option.BindEnv(option.Labels) 654 655 flags.String(option.LB, "", "Enables load balancer mode where load balancer bpf program is attached to the given interface") 656 option.BindEnv(option.LB) 657 658 flags.Bool(option.EnableNodePort, false, "Enable NodePort type services by Cilium (beta)") 659 option.BindEnv(option.EnableNodePort) 660 661 flags.StringSlice(option.NodePortRange, []string{fmt.Sprintf("%d", option.NodePortMinDefault), fmt.Sprintf("%d", option.NodePortMaxDefault)}, fmt.Sprintf("Set the min/max NodePort port range")) 662 option.BindEnv(option.NodePortRange) 663 664 flags.String(option.LibDir, defaults.LibraryPath, "Directory path to store runtime build environment") 665 option.BindEnv(option.LibDir) 666 667 flags.StringSlice(option.LogDriver, []string{}, "Logging endpoints to use for example syslog") 668 option.BindEnv(option.LogDriver) 669 670 flags.Var(option.NewNamedMapOptions(option.LogOpt, &option.Config.LogOpt, nil), 671 option.LogOpt, "Log driver options for cilium") 672 option.BindEnv(option.LogOpt) 673 674 flags.Bool(option.LogSystemLoadConfigName, false, "Enable periodic logging of system load") 675 option.BindEnv(option.LogSystemLoadConfigName) 676 677 flags.String(option.LoopbackIPv4, defaults.LoopbackIPv4, "IPv4 address for service loopback SNAT") 678 option.BindEnv(option.LoopbackIPv4) 679 680 flags.String(option.NAT46Range, defaults.DefaultNAT46Prefix, "IPv6 prefix to map IPv4 addresses to") 681 option.BindEnv(option.NAT46Range) 682 683 flags.Bool(option.Masquerade, true, "Masquerade packets from endpoints leaving the host") 684 option.BindEnv(option.Masquerade) 685 686 flags.Bool(option.InstallIptRules, true, "Install base iptables rules for cilium to mainly interact with kube-proxy (and masquerading)") 687 option.BindEnv(option.InstallIptRules) 688 689 flags.Duration(option.IPTablesLockTimeout, 5*time.Second, "Time to pass to each iptables invocation to wait for xtables lock acquisition") 690 option.BindEnv(option.IPTablesLockTimeout) 691 692 flags.String(option.IPTablesLockWaitInterval, "", "Interval to wait per each iptables lock acquisition (in microseconds)") 693 option.BindEnv(option.IPTablesLockWaitInterval) 694 695 flags.Int(option.MaxCtrlIntervalName, 0, "Maximum interval (in seconds) between controller runs. Zero is no limit.") 696 flags.MarkHidden(option.MaxCtrlIntervalName) 697 option.BindEnv(option.MaxCtrlIntervalName) 698 699 flags.StringSlice(option.Metrics, []string{}, "Metrics that should be enabled or disabled from the default metric list. (+metric_foo to enable metric_foo , -metric_bar to disable metric_bar)") 700 option.BindEnv(option.Metrics) 701 702 flags.String(option.MonitorAggregationName, "None", 703 "Level of monitor aggregation for traces from the datapath") 704 option.BindEnvWithLegacyEnvFallback(option.MonitorAggregationName, "CILIUM_MONITOR_AGGREGATION_LEVEL") 705 706 flags.Int(option.MonitorQueueSizeName, 0, "Size of the event queue when reading monitor events") 707 option.BindEnv(option.MonitorQueueSizeName) 708 709 flags.Int(option.MTUName, 0, "Overwrite auto-detected MTU of underlying network") 710 option.BindEnv(option.MTUName) 711 712 flags.Bool(option.PrependIptablesChainsName, true, "Prepend custom iptables chains instead of appending") 713 option.BindEnvWithLegacyEnvFallback(option.PrependIptablesChainsName, "CILIUM_PREPEND_IPTABLES_CHAIN") 714 715 flags.String(option.IPv6NodeAddr, "auto", "IPv6 address of node") 716 option.BindEnv(option.IPv6NodeAddr) 717 718 flags.String(option.IPv4NodeAddr, "auto", "IPv4 address of node") 719 option.BindEnv(option.IPv4NodeAddr) 720 721 flags.String(option.ReadCNIConfiguration, "", "Read to the CNI configuration at specified path to extract per node configuration") 722 option.BindEnv(option.ReadCNIConfiguration) 723 724 flags.Bool(option.Restore, true, "Restores state, if possible, from previous daemon") 725 option.BindEnv(option.Restore) 726 727 flags.Bool(option.SidecarHTTPProxy, false, "Disable host HTTP proxy, assuming proxies in sidecar containers") 728 flags.MarkHidden(option.SidecarHTTPProxy) 729 option.BindEnv(option.SidecarHTTPProxy) 730 731 flags.String(option.SidecarIstioProxyImage, k8s.DefaultSidecarIstioProxyImageRegexp, 732 "Regular expression matching compatible Istio sidecar istio-proxy container image names") 733 option.BindEnv(option.SidecarIstioProxyImage) 734 735 flags.Bool(option.SingleClusterRouteName, false, 736 "Use a single cluster route instead of per node routes") 737 option.BindEnv(option.SingleClusterRouteName) 738 739 flags.String(option.SocketPath, defaults.SockPath, "Sets daemon's socket path to listen for connections") 740 option.BindEnv(option.SocketPath) 741 742 flags.String(option.StateDir, defaults.RuntimePath, "Directory path to store runtime state") 743 option.BindEnv(option.StateDir) 744 745 flags.StringP(option.TunnelName, "t", "", fmt.Sprintf("Tunnel mode {%s} (default \"vxlan\" for the \"veth\" datapath mode)", option.GetTunnelModes())) 746 option.BindEnv(option.TunnelName) 747 748 flags.Int(option.TracePayloadlen, 128, "Length of payload to capture when tracing") 749 option.BindEnv(option.TracePayloadlen) 750 751 flags.Bool(option.Version, false, "Print version information") 752 option.BindEnv(option.Version) 753 754 flags.String(option.FlannelMasterDevice, "", 755 "Installs a BPF program to allow for policy enforcement in the given network interface. "+ 756 "Allows to run Cilium on top of other CNI plugins that provide networking, "+ 757 "e.g. flannel, where for flannel, this value should be set with 'cni0'. [EXPERIMENTAL]") 758 option.BindEnv(option.FlannelMasterDevice) 759 760 flags.Bool(option.FlannelUninstallOnExit, false, fmt.Sprintf("When used along the %s "+ 761 "flag, it cleans up all BPF programs installed when Cilium agent is terminated.", option.FlannelMasterDevice)) 762 option.BindEnv(option.FlannelUninstallOnExit) 763 764 flags.Bool(option.FlannelManageExistingContainers, false, 765 fmt.Sprintf("Installs a BPF program to allow for policy enforcement in already running containers managed by Flannel."+ 766 " Require Cilium to be running in the hostPID.")) 767 option.BindEnv(option.FlannelManageExistingContainers) 768 769 flags.Bool(option.PProf, false, "Enable serving the pprof debugging API") 770 option.BindEnv(option.PProf) 771 772 flags.String(option.PrefilterDevice, "undefined", "Device facing external network for XDP prefiltering") 773 option.BindEnv(option.PrefilterDevice) 774 775 flags.String(option.PrefilterMode, option.ModePreFilterNative, "Prefilter mode { "+option.ModePreFilterNative+" | "+option.ModePreFilterGeneric+" } (default: "+option.ModePreFilterNative+")") 776 option.BindEnv(option.PrefilterMode) 777 778 flags.Bool(option.PreAllocateMapsName, defaults.PreAllocateMaps, "Enable BPF map pre-allocation") 779 option.BindEnv(option.PreAllocateMapsName) 780 781 // We expect only one of the possible variables to be filled. The evaluation order is: 782 // --prometheus-serve-addr, CILIUM_PROMETHEUS_SERVE_ADDR, then PROMETHEUS_SERVE_ADDR 783 // The second environment variable (without the CILIUM_ prefix) is here to 784 // handle the case where someone uses a new image with an older spec, and the 785 // older spec used the older variable name. 786 flags.String(option.PrometheusServeAddr, "", "IP:Port on which to serve prometheus metrics (pass \":Port\" to bind on all interfaces, \"\" is off)") 787 option.BindEnvWithLegacyEnvFallback(option.PrometheusServeAddr, "PROMETHEUS_SERVE_ADDR") 788 789 flags.Int(option.CTMapEntriesGlobalTCPName, option.CTMapEntriesGlobalTCPDefault, "Maximum number of entries in TCP CT table") 790 option.BindEnvWithLegacyEnvFallback(option.CTMapEntriesGlobalTCPName, "CILIUM_GLOBAL_CT_MAX_TCP") 791 792 flags.Int(option.CTMapEntriesGlobalAnyName, option.CTMapEntriesGlobalAnyDefault, "Maximum number of entries in non-TCP CT table") 793 option.BindEnvWithLegacyEnvFallback(option.CTMapEntriesGlobalAnyName, "CILIUM_GLOBAL_CT_MAX_ANY") 794 795 flags.Duration(option.CTMapEntriesTimeoutTCPName, 21600*time.Second, "Timeout for established entries in TCP CT table") 796 option.BindEnv(option.CTMapEntriesTimeoutTCPName) 797 798 flags.Duration(option.CTMapEntriesTimeoutAnyName, 60*time.Second, "Timeout for entries in non-TCP CT table") 799 option.BindEnv(option.CTMapEntriesTimeoutAnyName) 800 801 flags.Duration(option.CTMapEntriesTimeoutSVCTCPName, 21600*time.Second, "Timeout for established service entries in TCP CT table") 802 option.BindEnv(option.CTMapEntriesTimeoutSVCTCPName) 803 804 flags.Duration(option.CTMapEntriesTimeoutSVCAnyName, 60*time.Second, "Timeout for service entries in non-TCP CT table") 805 option.BindEnv(option.CTMapEntriesTimeoutSVCAnyName) 806 807 flags.Duration(option.CTMapEntriesTimeoutSYNName, 60*time.Second, "Establishment timeout for entries in TCP CT table") 808 option.BindEnv(option.CTMapEntriesTimeoutSYNName) 809 810 flags.Duration(option.CTMapEntriesTimeoutFINName, 10*time.Second, "Teardown timeout for entries in TCP CT table") 811 option.BindEnv(option.CTMapEntriesTimeoutFINName) 812 813 flags.Int(option.NATMapEntriesGlobalName, option.NATMapEntriesGlobalDefault, "Maximum number of entries for the global BPF NAT table") 814 option.BindEnv(option.NATMapEntriesGlobalName) 815 816 flags.Int(option.PolicyMapEntriesName, defaults.PolicyMapEntries, "Maximum number of entries in endpoint policy map (per endpoint)") 817 option.BindEnv(option.PolicyMapEntriesName) 818 819 flags.String(option.CMDRef, "", "Path to cmdref output directory") 820 flags.MarkHidden(option.CMDRef) 821 option.BindEnv(option.CMDRef) 822 823 flags.Int(option.ToFQDNsMinTTL, 0, fmt.Sprintf("The minimum time, in seconds, to use DNS data for toFQDNs policies. (default %d when --tofqdns-enable-poller, %d otherwise)", defaults.ToFQDNsMinTTLPoller, defaults.ToFQDNsMinTTL)) 824 option.BindEnv(option.ToFQDNsMinTTL) 825 826 flags.Int(option.ToFQDNsProxyPort, 0, "Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port.") 827 option.BindEnv(option.ToFQDNsProxyPort) 828 829 flags.Bool(option.ToFQDNsEnablePoller, false, "Enable proactive polling of DNS names in toFQDNs.matchName rules.") 830 option.BindEnv(option.ToFQDNsEnablePoller) 831 832 flags.Bool(option.ToFQDNsEnablePollerEvents, true, "Emit DNS responses seen by the DNS poller as Monitor events, if the poller is enabled.") 833 option.BindEnv(option.ToFQDNsEnablePollerEvents) 834 835 flags.StringVar(&option.Config.FQDNRejectResponse, option.FQDNRejectResponseCode, option.FQDNProxyDenyWithRefused, fmt.Sprintf("DNS response code for rejecting DNS requests, available options are '%v'", option.FQDNRejectOptions)) 836 option.BindEnv(option.FQDNRejectResponseCode) 837 838 flags.Int(option.ToFQDNsMaxIPsPerHost, defaults.ToFQDNsMaxIPsPerHost, "Maximum number of IPs to maintain per FQDN name for each endpoint") 839 option.BindEnv(option.ToFQDNsMaxIPsPerHost) 840 841 flags.DurationVar(&option.Config.FQDNProxyResponseMaxDelay, option.FQDNProxyResponseMaxDelay, 50*time.Millisecond, "The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information.") 842 option.BindEnv(option.FQDNProxyResponseMaxDelay) 843 844 flags.String(option.ToFQDNsPreCache, defaults.ToFQDNsPreCache, "DNS cache data at this path is preloaded on agent startup") 845 option.BindEnv(option.ToFQDNsPreCache) 846 847 flags.Int(option.PolicyQueueSize, defaults.PolicyQueueSize, "size of queues for policy-related events") 848 option.BindEnv(option.PolicyQueueSize) 849 850 flags.Int(option.EndpointQueueSize, defaults.EndpointQueueSize, "size of EventQueue per-endpoint") 851 option.BindEnv(option.EndpointQueueSize) 852 853 flags.Bool(option.SelectiveRegeneration, true, "only regenerate endpoints which need to be regenerated upon policy changes") 854 flags.MarkHidden(option.SelectiveRegeneration) 855 option.BindEnv(option.SelectiveRegeneration) 856 857 flags.Bool(option.SkipCRDCreation, false, "Skip Kubernetes Custom Resource Definitions creations") 858 option.BindEnv(option.SkipCRDCreation) 859 860 flags.String(option.WriteCNIConfigurationWhenReady, "", fmt.Sprintf("Write the CNI configuration as specified via --%s to path when agent is ready", option.ReadCNIConfiguration)) 861 option.BindEnv(option.WriteCNIConfigurationWhenReady) 862 863 flags.Duration(option.PolicyTriggerInterval, defaults.PolicyTriggerInterval, "Time between triggers of policy updates (regenerations for all endpoints)") 864 flags.MarkHidden(option.PolicyTriggerInterval) 865 option.BindEnv(option.PolicyTriggerInterval) 866 867 flags.Bool(option.DisableCNPStatusUpdates, false, "Do not send CNP NodeStatus updates to the Kubernetes api-server (recommended to run with `cnp-node-status-gc=false` in cilium-operator)") 868 option.BindEnv(option.DisableCNPStatusUpdates) 869 870 viper.BindPFlags(flags) 871 } 872 873 // RestoreExecPermissions restores file permissions to 0740 of all files inside 874 // `searchDir` with the given regex `patterns`. 875 func RestoreExecPermissions(searchDir string, patterns ...string) error { 876 fileList := []string{} 877 err := filepath.Walk(searchDir, func(path string, f os.FileInfo, err error) error { 878 for _, pattern := range patterns { 879 if regexp.MustCompile(pattern).MatchString(f.Name()) { 880 fileList = append(fileList, path) 881 break 882 } 883 } 884 return nil 885 }) 886 for _, fileToChange := range fileList { 887 // Changing files permissions to -rwx:r--:---, we are only 888 // adding executable permission to the owner and keeping the 889 // same permissions stored by go-bindata. 890 if err := os.Chmod(fileToChange, os.FileMode(0740)); err != nil { 891 return err 892 } 893 } 894 return err 895 } 896 897 // initConfig reads in config file and ENV variables if set. 898 func initConfig() { 899 if viper.GetBool("version") { 900 fmt.Printf("Cilium %s\n", version.Version) 901 os.Exit(0) 902 } 903 904 if option.Config.CMDRefDir != "" { 905 return 906 } 907 908 option.Config.ConfigFile = viper.GetString(option.ConfigFile) // enable ability to specify config file via flag 909 option.Config.ConfigDir = viper.GetString(option.ConfigDir) 910 viper.SetEnvPrefix("cilium") 911 912 if option.Config.ConfigDir != "" { 913 if _, err := os.Stat(option.Config.ConfigDir); os.IsNotExist(err) { 914 log.Fatalf("Non-existent configuration directory %s", option.Config.ConfigDir) 915 } 916 917 if m, err := option.ReadDirConfig(option.Config.ConfigDir); err != nil { 918 log.Fatalf("Unable to read configuration directory %s: %s", option.Config.ConfigDir, err) 919 } else { 920 // replace deprecated fields with new fields 921 option.ReplaceDeprecatedFields(m) 922 err := option.MergeConfig(m) 923 if err != nil { 924 log.Fatalf("Unable to merge configuration: %s", err) 925 } 926 } 927 } 928 929 if option.Config.ConfigFile != "" { 930 viper.SetConfigFile(option.Config.ConfigFile) 931 } else { 932 viper.SetConfigName("ciliumd") // name of config file (without extension) 933 viper.AddConfigPath("$HOME") // adding home directory as first search path 934 } 935 936 // If a config file is found, read it in. 937 if err := viper.ReadInConfig(); err == nil { 938 log.WithField(logfields.Path, viper.ConfigFileUsed()). 939 Info("Using config from file") 940 } else if option.Config.ConfigFile != "" { 941 log.WithField(logfields.Path, option.Config.ConfigFile). 942 Fatal("Error reading config file") 943 } else { 944 log.WithField(logfields.Reason, err).Info("Skipped reading configuration file") 945 } 946 } 947 948 func initEnv(cmd *cobra.Command) { 949 // Prepopulate option.Config with options from CLI. 950 option.Config.Populate() 951 952 // add hooks after setting up metrics in the option.Confog 953 logging.DefaultLogger.Hooks.Add(metrics.NewLoggingHook(components.CiliumAgentName)) 954 955 // Logging should always be bootstrapped first. Do not add any code above this! 956 logging.SetupLogging(option.Config.LogDriver, option.Config.LogOpt, "cilium-agent", option.Config.Debug) 957 958 if option.Config.CMDRefDir != "" { 959 genMarkdown(cmd) 960 } 961 962 option.LogRegisteredOptions(log) 963 964 for _, grp := range option.Config.DebugVerbose { 965 switch grp { 966 case argDebugVerboseFlow: 967 log.Debugf("Enabling flow debug") 968 flowdebug.Enable() 969 case argDebugVerboseKvstore: 970 kvstore.EnableTracing() 971 case argDebugVerboseEnvoy: 972 log.Debugf("Enabling Envoy tracing") 973 envoy.EnableTracing() 974 default: 975 log.Warningf("Unknown verbose debug group: %s", grp) 976 } 977 } 978 979 common.RequireRootPrivilege("cilium-agent") 980 981 log.Info(" _ _ _") 982 log.Info(" ___|_| |_|_ _ _____") 983 log.Info("| _| | | | | | |") 984 log.Info("|___|_|_|_|___|_|_|_|") 985 log.Infof("Cilium %s", version.Version) 986 987 if option.Config.LogSystemLoadConfig { 988 loadinfo.StartBackgroundLogger() 989 } 990 991 if option.Config.DisableEnvoyVersionCheck { 992 log.Info("Envoy version check disabled") 993 } else { 994 envoyVersion := envoy.GetEnvoyVersion() 995 log.Infof("%s", envoyVersion) 996 997 envoyVersionArray := strings.Fields(envoyVersion) 998 if len(envoyVersionArray) < 3 { 999 log.Fatal("Truncated Envoy version string, cannot verify version match.") 1000 } 1001 // Make sure Envoy version matches ours 1002 if !strings.HasPrefix(envoyVersionArray[2], envoy.RequiredEnvoyVersionSHA) { 1003 log.Fatalf("Envoy version %s does not match with required version %s ,aborting.", 1004 envoyVersionArray[2], envoy.RequiredEnvoyVersionSHA) 1005 } 1006 } 1007 1008 // This check is here instead of in DaemonConfig.Populate (invoked at the 1009 // start of this function as option.Config.Populate) to avoid an import loop. 1010 if option.Config.IdentityAllocationMode == option.IdentityAllocationModeCRD && !k8s.IsEnabled() { 1011 log.Fatal("CRD Identity allocation mode requires k8s to be configured.") 1012 } 1013 1014 if option.Config.PProf { 1015 pprof.Enable() 1016 } 1017 1018 if option.Config.PreAllocateMaps { 1019 bpf.EnableMapPreAllocation() 1020 } 1021 1022 scopedLog := log.WithFields(logrus.Fields{ 1023 logfields.Path + ".RunDir": option.Config.RunDir, 1024 logfields.Path + ".LibDir": option.Config.LibDir, 1025 }) 1026 1027 if option.Config.LBInterface != "" { 1028 service.EnableGlobalServiceID(true) 1029 } 1030 1031 option.Config.BpfDir = filepath.Join(option.Config.LibDir, defaults.BpfDir) 1032 scopedLog = scopedLog.WithField(logfields.Path+".BPFDir", defaults.BpfDir) 1033 if err := os.MkdirAll(option.Config.RunDir, defaults.RuntimePathRights); err != nil { 1034 scopedLog.WithError(err).Fatal("Could not create runtime directory") 1035 } 1036 1037 option.Config.StateDir = filepath.Join(option.Config.RunDir, defaults.StateDir) 1038 scopedLog = scopedLog.WithField(logfields.Path+".StateDir", option.Config.StateDir) 1039 if err := os.MkdirAll(option.Config.StateDir, defaults.StateDirRights); err != nil { 1040 scopedLog.WithError(err).Fatal("Could not create state directory") 1041 } 1042 1043 if err := os.MkdirAll(option.Config.LibDir, defaults.RuntimePathRights); err != nil { 1044 scopedLog.WithError(err).Fatal("Could not create library directory") 1045 } 1046 if !option.Config.KeepTemplates { 1047 // We need to remove the old probes here as otherwise stale .t tests could 1048 // still reside from newer Cilium versions which might break downgrade. 1049 if err := os.RemoveAll(filepath.Join(option.Config.BpfDir, "/probes/")); err != nil { 1050 scopedLog.WithError(err).Fatal("Could not delete old probes from library directory") 1051 } 1052 if err := RestoreAssets(option.Config.LibDir, defaults.BpfDir); err != nil { 1053 scopedLog.WithError(err).Fatal("Unable to restore agent assets") 1054 } 1055 // Restore permissions of executable files 1056 if err := RestoreExecPermissions(option.Config.LibDir, `.*\.sh`); err != nil { 1057 scopedLog.WithError(err).Fatal("Unable to restore agent assets") 1058 } 1059 } 1060 if option.Config.MaxControllerInterval < 0 { 1061 scopedLog.Fatalf("Invalid %s value %d", option.MaxCtrlIntervalName, option.Config.MaxControllerInterval) 1062 } 1063 1064 checkMinRequirements() 1065 1066 if err := pidfile.Write(defaults.PidFilePath); err != nil { 1067 log.WithField(logfields.Path, defaults.PidFilePath).WithError(err).Fatal("Failed to create Pidfile") 1068 } 1069 1070 option.Config.AllowLocalhost = strings.ToLower(option.Config.AllowLocalhost) 1071 switch option.Config.AllowLocalhost { 1072 case option.AllowLocalhostAlways, option.AllowLocalhostAuto, option.AllowLocalhostPolicy: 1073 default: 1074 log.Fatalf("Invalid setting for --allow-localhost, must be { %s, %s, %s }", 1075 option.AllowLocalhostAuto, option.AllowLocalhostAlways, option.AllowLocalhostPolicy) 1076 } 1077 1078 option.Config.ModePreFilter = strings.ToLower(option.Config.ModePreFilter) 1079 switch option.Config.ModePreFilter { 1080 case option.ModePreFilterNative: 1081 option.Config.ModePreFilter = "xdpdrv" 1082 case option.ModePreFilterGeneric: 1083 option.Config.ModePreFilter = "xdpgeneric" 1084 default: 1085 log.Fatalf("Invalid setting for --prefilter-mode, must be { %s, %s }", 1086 option.ModePreFilterNative, option.ModePreFilterGeneric) 1087 } 1088 1089 scopedLog = log.WithField(logfields.Path, option.Config.SocketPath) 1090 socketDir := path.Dir(option.Config.SocketPath) 1091 if err := os.MkdirAll(socketDir, defaults.RuntimePathRights); err != nil { 1092 scopedLog.WithError(err).Fatal("Cannot mkdir directory for cilium socket") 1093 } 1094 1095 if err := os.Remove(option.Config.SocketPath); !os.IsNotExist(err) && err != nil { 1096 scopedLog.WithError(err).Fatal("Cannot remove existing Cilium sock") 1097 } 1098 1099 // The standard operation is to mount the BPF filesystem to the 1100 // standard location (/sys/fs/bpf). The user may chose to specify 1101 // the path to an already mounted filesystem instead. This is 1102 // useful if the daemon is being round inside a namespace and the 1103 // BPF filesystem is mapped into the slave namespace. 1104 bpf.CheckOrMountFS(option.Config.BPFRoot) 1105 cgroups.CheckOrMountCgrpFS(option.Config.CGroupRoot) 1106 1107 option.Config.Opts.SetBool(option.Debug, option.Config.Debug) 1108 option.Config.Opts.SetBool(option.DebugLB, option.Config.Debug) 1109 option.Config.Opts.SetBool(option.DropNotify, true) 1110 option.Config.Opts.SetBool(option.TraceNotify, true) 1111 option.Config.Opts.SetBool(option.PolicyTracing, option.Config.EnableTracing) 1112 option.Config.Opts.SetBool(option.Conntrack, !option.Config.DisableConntrack) 1113 option.Config.Opts.SetBool(option.ConntrackAccounting, !option.Config.DisableConntrack) 1114 option.Config.Opts.SetBool(option.ConntrackLocal, false) 1115 1116 monitorAggregationLevel, err := option.ParseMonitorAggregationLevel(option.Config.MonitorAggregation) 1117 if err != nil { 1118 log.WithError(err).Fatalf("Failed to parse %s: %s", 1119 option.MonitorAggregationName, err) 1120 } 1121 option.Config.Opts.SetValidated(option.MonitorAggregation, monitorAggregationLevel) 1122 1123 policy.SetPolicyEnabled(option.Config.EnablePolicy) 1124 1125 if err := cache.AddUserDefinedNumericIdentitySet(option.Config.FixedIdentityMapping); err != nil { 1126 log.Fatalf("Invalid fixed identities provided: %s", err) 1127 } 1128 1129 if !option.Config.EnableIPv4 && !option.Config.EnableIPv6 { 1130 log.Fatal("Either IPv4 or IPv6 addressing must be enabled") 1131 } 1132 if err := labels.ParseLabelPrefixCfg(option.Config.Labels, option.Config.LabelPrefixFile); err != nil { 1133 log.WithError(err).Fatal("Unable to parse Label prefix configuration") 1134 } 1135 1136 _, r, err := net.ParseCIDR(option.Config.NAT46Range) 1137 if err != nil { 1138 log.WithError(err).WithField(logfields.V6Prefix, option.Config.NAT46Range).Fatal("Invalid NAT46 prefix") 1139 } 1140 1141 option.Config.NAT46Prefix = r 1142 1143 switch option.Config.DatapathMode { 1144 case option.DatapathModeVeth: 1145 if name := viper.GetString(option.IpvlanMasterDevice); name != "undefined" { 1146 log.WithField(logfields.IpvlanMasterDevice, name). 1147 Fatal("ipvlan master device cannot be set in the 'veth' datapath mode") 1148 } 1149 if option.Config.Tunnel == "" { 1150 option.Config.Tunnel = option.TunnelVXLAN 1151 } 1152 if option.Config.IsFlannelMasterDeviceSet() { 1153 if option.Config.Tunnel != option.TunnelDisabled { 1154 log.Warnf("Running Cilium in flannel mode requires tunnel mode be '%s'. Changing tunnel mode to: %s", option.TunnelDisabled, option.TunnelDisabled) 1155 option.Config.Tunnel = option.TunnelDisabled 1156 } 1157 if option.Config.EnableIPv6 { 1158 log.Warn("Running Cilium in flannel mode requires IPv6 mode be 'false'. Disabling IPv6 mode") 1159 option.Config.EnableIPv6 = false 1160 } 1161 if option.Config.FlannelManageExistingContainers && !option.Config.WorkloadsEnabled() { 1162 log.Warnf("Managing existing flannel containers with Cilium requires container workloads. Changing %s to %q", option.ContainerRuntime, "auto") 1163 option.Config.Workloads = option.ContainerRuntimeAuto 1164 } 1165 } 1166 case option.DatapathModeIpvlan: 1167 if option.Config.Tunnel != "" && option.Config.Tunnel != option.TunnelDisabled { 1168 log.WithField(logfields.Tunnel, option.Config.Tunnel). 1169 Fatal("tunnel cannot be set in the 'ipvlan' datapath mode") 1170 } 1171 if option.Config.Device != "undefined" { 1172 log.WithField(logfields.Device, option.Config.Device). 1173 Fatal("device cannot be set in the 'ipvlan' datapath mode") 1174 } 1175 if option.Config.EnableIPSec { 1176 log.Fatal("Currently ipsec cannot be used in the 'ipvlan' datapath mode.") 1177 } 1178 1179 option.Config.Tunnel = option.TunnelDisabled 1180 // We disallow earlier command line combination of --device with 1181 // --datapath-mode ipvlan. But given all the remaining logic is 1182 // shared with option.Config.Device, override it here internally 1183 // with the specified ipvlan master device. Reason to have a 1184 // separate, more specific command line parameter here and in 1185 // the swagger API is that in future we might deprecate --device 1186 // parameter with e.g. some auto-detection mechanism, thus for 1187 // ipvlan it is desired to have a separate one, see PR #6608. 1188 option.Config.Device = viper.GetString(option.IpvlanMasterDevice) 1189 if option.Config.Device == "undefined" { 1190 log.WithField(logfields.IpvlanMasterDevice, option.Config.Device). 1191 Fatal("ipvlan master device must be specified in the 'ipvlan' datapath mode") 1192 } 1193 link, err := netlink.LinkByName(option.Config.Device) 1194 if err != nil { 1195 log.WithError(err).WithField(logfields.IpvlanMasterDevice, option.Config.Device). 1196 Fatal("Cannot find device interface") 1197 } 1198 option.Config.Ipvlan.MasterDeviceIndex = link.Attrs().Index 1199 option.Config.Ipvlan.OperationMode = option.OperationModeL3 1200 if option.Config.InstallIptRules { 1201 option.Config.Ipvlan.OperationMode = option.OperationModeL3S 1202 } 1203 default: 1204 log.WithField(logfields.DatapathMode, option.Config.DatapathMode).Fatal("Invalid datapath mode") 1205 } 1206 1207 if option.Config.EnableL7Proxy && !option.Config.InstallIptRules { 1208 log.Fatal("L7 proxy requires iptables rules (--install-iptables-rules=\"true\")") 1209 } 1210 1211 if option.Config.EnableIPSec && option.Config.Tunnel == option.TunnelDisabled && option.Config.EncryptInterface == "" { 1212 link, err := linuxdatapath.NodeDeviceNameWithDefaultRoute() 1213 if err != nil { 1214 log.WithError(err).Fatal("Ipsec default interface lookup failed, consider \"encrypt-interface\" to manually configure interface.", err) 1215 } 1216 option.Config.EncryptInterface = link 1217 } 1218 1219 // BPF masquerade specified, rejecting unsupported options for this mode. 1220 if !option.Config.InstallIptRules && option.Config.Masquerade { 1221 if option.Config.DatapathMode != option.DatapathModeIpvlan { 1222 log.WithField(logfields.DatapathMode, option.Config.DatapathMode). 1223 Fatal("BPF masquerade currently only in ipvlan datapath mode (restriction will be lifted soon)") 1224 } 1225 if option.Config.Tunnel != option.TunnelDisabled { 1226 log.WithField(logfields.Tunnel, option.Config.Tunnel). 1227 Fatal("BPF masquerade only in direct routing mode supported") 1228 } 1229 if option.Config.Device == "undefined" { 1230 log.WithField(logfields.Device, option.Config.Device). 1231 Fatal("BPF masquerade needs external facing device specified") 1232 } 1233 } 1234 1235 if option.Config.EnableNodePort && 1236 !(option.Config.EnableHostReachableServices && 1237 option.Config.EnableHostServicesTCP && option.Config.EnableHostServicesUDP) { 1238 // We enable host reachable services in order to allow 1239 // access to node port services from the host. 1240 log.Info("Auto-enabling host reachable services for UDP and TCP as required by BPF NodePort.") 1241 option.Config.EnableHostReachableServices = true 1242 option.Config.EnableHostServicesTCP = true 1243 option.Config.EnableHostServicesUDP = true 1244 } 1245 1246 if option.Config.EnableNodePort && option.Config.Device == "undefined" { 1247 device, err := linuxdatapath.NodeDeviceNameWithDefaultRoute() 1248 if err != nil { 1249 log.WithError(err).Fatal("BPF NodePort's external facing device could not be determined. Use --device to specify.") 1250 } 1251 log.WithField(logfields.Interface, device). 1252 Info("Using auto-derived device for BPF node port") 1253 option.Config.Device = device 1254 } 1255 1256 if option.Config.EnableHostReachableServices { 1257 if option.Config.EnableHostServicesTCP && 1258 (option.Config.EnableIPv4 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_INET4_CONNECT) != nil || 1259 option.Config.EnableIPv6 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_INET6_CONNECT) != nil) { 1260 log.Fatal("BPF host reachable services for TCP needs kernel 4.17.0 or newer.") 1261 } 1262 // NOTE: as host-lb is a hard dependency for NodePort BPF, the following 1263 // probe will catch if the fib_lookup helper is missing (< 4.18), 1264 // which is another hard dependency for NodePort BPF. 1265 if option.Config.EnableHostServicesUDP && 1266 (option.Config.EnableIPv4 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_UDP4_RECVMSG) != nil || 1267 option.Config.EnableIPv6 && bpf.TestDummyProg(bpf.ProgTypeCgroupSockAddr, bpf.BPF_CGROUP_UDP6_RECVMSG) != nil) { 1268 log.Fatal("BPF host reachable services for UDP needs kernel 4.19.57, 5.1.16, 5.2.0 or newer. If you run an older kernel and only need TCP, then specify: --host-reachable-services-protos=tcp") 1269 } 1270 } 1271 1272 // If device has been specified, use it to derive better default 1273 // allocation prefixes 1274 if option.Config.Device != "undefined" { 1275 node.InitDefaultPrefix(option.Config.Device) 1276 } 1277 1278 if option.Config.IPv6NodeAddr != "auto" { 1279 if ip := net.ParseIP(option.Config.IPv6NodeAddr); ip == nil { 1280 log.WithField(logfields.IPAddr, option.Config.IPv6NodeAddr).Fatal("Invalid IPv6 node address") 1281 } else { 1282 if !ip.IsGlobalUnicast() { 1283 log.WithField(logfields.IPAddr, ip).Fatal("Invalid IPv6 node address: not a global unicast address") 1284 } 1285 1286 node.SetIPv6(ip) 1287 } 1288 } 1289 1290 if option.Config.IPv4NodeAddr != "auto" { 1291 if ip := net.ParseIP(option.Config.IPv4NodeAddr); ip == nil { 1292 log.WithField(logfields.IPAddr, option.Config.IPv4NodeAddr).Fatal("Invalid IPv4 node address") 1293 } else { 1294 node.SetExternalIPv4(ip) 1295 } 1296 } 1297 1298 if option.Config.SidecarHTTPProxy { 1299 log.Warn(`"sidecar-http-proxy" flag is deprecated and has no effect`) 1300 } 1301 1302 k8s.SidecarIstioProxyImageRegexp, err = regexp.Compile(option.Config.SidecarIstioProxyImage) 1303 if err != nil { 1304 log.WithError(err).Fatal("Invalid sidecar-istio-proxy-image regular expression") 1305 return 1306 } 1307 } 1308 1309 // waitForHostDeviceWhenReady waits the given ifaceName to be up and ready. If 1310 // ifaceName is not found, then it will wait forever until the device is 1311 // created. 1312 func waitForHostDeviceWhenReady(ifaceName string) error { 1313 for i := 0; ; i++ { 1314 if i%10 == 0 { 1315 log.WithField(logfields.Interface, ifaceName). 1316 Info("Waiting for the underlying interface to be initialized with containers") 1317 } 1318 _, err := netlink.LinkByName(ifaceName) 1319 if err == nil { 1320 log.WithField(logfields.Interface, ifaceName). 1321 Info("Underlying interface initialized with containers!") 1322 break 1323 } 1324 select { 1325 case <-cleanUPSig: 1326 return errors.New("clean up signal triggered") 1327 default: 1328 time.Sleep(time.Second) 1329 } 1330 } 1331 return nil 1332 } 1333 1334 func endParallelMapMode() { 1335 ipcachemap.IPCache.EndParallelMode() 1336 } 1337 1338 func (d *Daemon) initKVStore() { 1339 goopts := &kvstore.ExtraOptions{ 1340 ClusterSizeDependantInterval: d.nodeDiscovery.Manager.ClusterSizeDependantInterval, 1341 } 1342 1343 controller.NewManager().UpdateController("kvstore-locks-gc", 1344 controller.ControllerParams{ 1345 DoFunc: func(ctx context.Context) error { 1346 kvstore.RunLockGC() 1347 return nil 1348 }, 1349 RunInterval: defaults.KVStoreStaleLockTimeout, 1350 }, 1351 ) 1352 1353 // If K8s is enabled we can do the service translation automagically by 1354 // looking at services from k8s and retrieve the service IP from that. 1355 // This makes cilium to not depend on kube dns to interact with etcd 1356 _, isETCDOperator := kvstore.IsEtcdOperator(option.Config.KVStore, option.Config.KVStoreOpt, option.Config.K8sNamespace) 1357 if k8s.IsEnabled() && isETCDOperator { 1358 // Wait services and endpoints cache are synced with k8s before setting 1359 // up etcd so we can perform the name resolution for etcd-operator 1360 // to the service IP as well perform the service -> backend IPs for 1361 // that service IP. 1362 d.waitForCacheSync(k8sAPIGroupServiceV1Core, k8sAPIGroupEndpointV1Core) 1363 log := log.WithField(logfields.LogSubsys, "etcd") 1364 goopts.DialOption = []grpc.DialOption{ 1365 grpc.WithDialer(k8s.CreateCustomDialer(&d.k8sSvcCache, log)), 1366 } 1367 } 1368 1369 if err := kvstore.Setup(option.Config.KVStore, option.Config.KVStoreOpt, goopts); err != nil { 1370 addrkey := fmt.Sprintf("%s.address", option.Config.KVStore) 1371 addr := option.Config.KVStoreOpt[addrkey] 1372 1373 log.WithError(err).WithFields(logrus.Fields{ 1374 "kvstore": option.Config.KVStore, 1375 "address": addr, 1376 }).Fatal("Unable to setup kvstore") 1377 } 1378 } 1379 1380 func runDaemon() { 1381 datapathConfig := linuxdatapath.DatapathConfiguration{ 1382 HostDevice: option.Config.HostDevice, 1383 EncryptInterface: option.Config.EncryptInterface, 1384 } 1385 1386 log.Info("Initializing daemon") 1387 1388 // Since flannel doesn't create the cni0 interface until the first container 1389 // is initialized we need to wait until it is initialized so we can attach 1390 // the BPF program to it. If Cilium is running as a Kubernetes DaemonSet, 1391 // there is also a script waiting for the interface to be created. 1392 if option.Config.IsFlannelMasterDeviceSet() { 1393 err := waitForHostDeviceWhenReady(option.Config.FlannelMasterDevice) 1394 if err != nil { 1395 log.WithError(err).WithFields(logrus.Fields{ 1396 logfields.Interface: option.Config.FlannelMasterDevice, 1397 }).Error("unable to check for host device") 1398 return 1399 } 1400 } 1401 1402 option.Config.RunMonitorAgent = true 1403 1404 if err := enableIPForwarding(); err != nil { 1405 log.WithError(err).Fatal("Error when enabling sysctl parameters") 1406 } 1407 1408 iptablesManager := &iptables.IptablesManager{} 1409 iptablesManager.Init() 1410 1411 d, restoredEndpoints, err := NewDaemon(linuxdatapath.NewDatapath(datapathConfig, iptablesManager), iptablesManager) 1412 if err != nil { 1413 log.WithError(err).Fatal("Error while creating daemon") 1414 return 1415 } 1416 1417 // This validation needs to be done outside of the agent until 1418 // datapath.NodeAddressing is used consistently across the code base. 1419 log.Info("Validating configured node address ranges") 1420 if err := node.ValidatePostInit(); err != nil { 1421 log.WithError(err).Fatal("postinit failed") 1422 } 1423 1424 if option.Config.IsFlannelMasterDeviceSet() && option.Config.FlannelUninstallOnExit { 1425 cleanup.DeferTerminationCleanupFunction(cleanUPWg, cleanUPSig, func() { 1426 d.compilationMutex.Lock() 1427 loader.DeleteDatapath(context.Background(), option.FlannelMasterDevice, "egress") 1428 d.compilationMutex.Unlock() 1429 }) 1430 } 1431 1432 bootstrapStats.enableConntrack.Start() 1433 log.Info("Starting connection tracking garbage collector") 1434 endpointmanager.EnableConntrackGC(option.Config.EnableIPv4, option.Config.EnableIPv6, 1435 restoredEndpoints.restored) 1436 bootstrapStats.enableConntrack.End(true) 1437 1438 endpointmanager.EndpointSynchronizer = &endpointsynchronizer.EndpointSynchronizer{} 1439 1440 bootstrapStats.k8sInit.Start() 1441 1442 // We need to set up etcd in parallel so we will initialize the k8s 1443 // subsystem as well in parallel so caches will start to be synchronized 1444 // with k8s. 1445 k8sCachesSynced := d.initK8sSubsystem() 1446 if option.Config.KVStore == "" { 1447 log.Info("Skipping kvstore configuration") 1448 } else { 1449 d.initKVStore() 1450 } 1451 1452 // Wait only for certain caches, but not all! 1453 // (Check Daemon.initK8sSubsystem() for more info) 1454 <-k8sCachesSynced 1455 bootstrapStats.k8sInit.End(true) 1456 restoreComplete := d.initRestore(restoredEndpoints) 1457 1458 if option.Config.IsFlannelMasterDeviceSet() { 1459 if option.Config.EnableEndpointHealthChecking { 1460 log.Warn("Running Cilium in flannel mode doesn't support endpoint connectivity health checking. Disabling endpoint connectivity health check.") 1461 option.Config.EnableEndpointHealthChecking = false 1462 } 1463 1464 err := node.SetInternalIPv4From(option.Config.FlannelMasterDevice) 1465 if err != nil { 1466 log.WithError(err).WithField("device", option.Config.FlannelMasterDevice).Fatal("Unable to set internal IPv4") 1467 } 1468 if option.Config.FlannelManageExistingContainers { 1469 log.Info("Searching for existing containers...") 1470 d.attachExistingInfraContainers() 1471 } 1472 } 1473 1474 if !option.Config.DryMode { 1475 go func() { 1476 if restoreComplete != nil { 1477 <-restoreComplete 1478 } 1479 d.dnsNameManager.CompleteBootstrap() 1480 maps.CollectStaleMapGarbage() 1481 maps.RemoveDisabledMaps() 1482 }() 1483 } 1484 1485 // The workload event listener *must* be enabled *after* restored endpoints 1486 // are added into the endpoint manager; otherwise, updates to important 1487 // endpoint metadata, such as Kubernetes pod name and namespace, will not 1488 // be performed on the endpoint. 1489 eventsCh, err := workloads.EnableEventListener() 1490 if err != nil { 1491 log.WithError(err).Fatal("Error while enabling workload event watcher") 1492 } else { 1493 d.workloadsEventsCh = eventsCh 1494 } 1495 1496 bootstrapStats.healthCheck.Start() 1497 if option.Config.EnableHealthChecking { 1498 d.initHealth() 1499 } 1500 bootstrapStats.healthCheck.End(true) 1501 1502 d.startStatusCollector() 1503 1504 metricsErrs := initMetrics() 1505 1506 bootstrapStats.initAPI.Start() 1507 api := d.instantiateAPI() 1508 1509 server := server.NewServer(api) 1510 server.EnabledListeners = []string{"unix"} 1511 server.SocketPath = flags.Filename(option.Config.SocketPath) 1512 server.ReadTimeout = apiTimeout 1513 server.WriteTimeout = apiTimeout 1514 defer server.Shutdown() 1515 1516 server.ConfigureAPI() 1517 bootstrapStats.initAPI.End(true) 1518 1519 repr, err := monitorAPI.TimeRepr(time.Now()) 1520 if err != nil { 1521 log.WithError(err).Warn("Failed to generate agent start monitor message") 1522 } else { 1523 d.SendNotification(monitorAPI.AgentNotifyStart, repr) 1524 } 1525 1526 log.WithField("bootstrapTime", time.Since(bootstrapTimestamp)). 1527 Info("Daemon initialization completed") 1528 1529 if option.Config.WriteCNIConfigurationWhenReady != "" { 1530 input, err := ioutil.ReadFile(option.Config.ReadCNIConfiguration) 1531 if err != nil { 1532 log.WithError(err).Fatal("Unable to read CNI configuration file") 1533 } 1534 1535 if err = ioutil.WriteFile(option.Config.WriteCNIConfigurationWhenReady, input, 0644); err != nil { 1536 log.WithError(err).Fatalf("Unable to write CNI configuration file to %s", option.Config.WriteCNIConfigurationWhenReady) 1537 } else { 1538 log.Infof("Wrote CNI configuration file to %s", option.Config.WriteCNIConfigurationWhenReady) 1539 } 1540 } 1541 1542 errs := make(chan error, 1) 1543 1544 go func() { 1545 errs <- server.Serve() 1546 }() 1547 1548 if k8s.IsEnabled() { 1549 bootstrapStats.k8sInit.Start() 1550 k8s.Client().MarkNodeReady(node.GetName()) 1551 bootstrapStats.k8sInit.End(true) 1552 } 1553 1554 bootstrapStats.overall.End(true) 1555 bootstrapStats.updateMetrics() 1556 1557 select { 1558 case err := <-metricsErrs: 1559 if err != nil { 1560 log.WithError(err).Fatal("Cannot start metrics server") 1561 } 1562 case err := <-errs: 1563 if err != nil { 1564 log.WithError(err).Fatal("Error returned from non-returning Serve() call") 1565 } 1566 } 1567 } 1568 1569 func (d *Daemon) instantiateAPI() *restapi.CiliumAPI { 1570 1571 swaggerSpec, err := loads.Analyzed(server.SwaggerJSON, "") 1572 if err != nil { 1573 log.WithError(err).Fatal("Cannot load swagger spec") 1574 } 1575 1576 log.Info("Initializing Cilium API") 1577 api := restapi.NewCiliumAPI(swaggerSpec) 1578 1579 api.Logger = log.Infof 1580 1581 // /healthz/ 1582 api.DaemonGetHealthzHandler = NewGetHealthzHandler(d) 1583 1584 // /cluster/nodes 1585 api.DaemonGetClusterNodesHandler = NewGetClusterNodesHandler(d) 1586 1587 // /config/ 1588 api.DaemonGetConfigHandler = NewGetConfigHandler(d) 1589 api.DaemonPatchConfigHandler = NewPatchConfigHandler(d) 1590 1591 // /endpoint/ 1592 api.EndpointGetEndpointHandler = NewGetEndpointHandler(d) 1593 1594 // /endpoint/{id} 1595 api.EndpointGetEndpointIDHandler = NewGetEndpointIDHandler(d) 1596 api.EndpointPutEndpointIDHandler = NewPutEndpointIDHandler(d) 1597 api.EndpointPatchEndpointIDHandler = NewPatchEndpointIDHandler(d) 1598 api.EndpointDeleteEndpointIDHandler = NewDeleteEndpointIDHandler(d) 1599 1600 // /endpoint/{id}config/ 1601 api.EndpointGetEndpointIDConfigHandler = NewGetEndpointIDConfigHandler(d) 1602 api.EndpointPatchEndpointIDConfigHandler = NewPatchEndpointIDConfigHandler(d) 1603 1604 // /endpoint/{id}/labels/ 1605 api.EndpointGetEndpointIDLabelsHandler = NewGetEndpointIDLabelsHandler(d) 1606 api.EndpointPatchEndpointIDLabelsHandler = NewPatchEndpointIDLabelsHandler(d) 1607 1608 // /endpoint/{id}/log/ 1609 api.EndpointGetEndpointIDLogHandler = NewGetEndpointIDLogHandler(d) 1610 1611 // /endpoint/{id}/healthz 1612 api.EndpointGetEndpointIDHealthzHandler = NewGetEndpointIDHealthzHandler(d) 1613 1614 // /identity/ 1615 api.PolicyGetIdentityHandler = newGetIdentityHandler(d) 1616 api.PolicyGetIdentityIDHandler = newGetIdentityIDHandler(d) 1617 1618 // /identity/endpoints 1619 api.PolicyGetIdentityEndpointsHandler = newGetIdentityEndpointsIDHandler(d) 1620 1621 // /policy/ 1622 api.PolicyGetPolicyHandler = newGetPolicyHandler(d) 1623 api.PolicyPutPolicyHandler = newPutPolicyHandler(d) 1624 api.PolicyDeletePolicyHandler = newDeletePolicyHandler(d) 1625 api.PolicyGetPolicySelectorsHandler = newGetPolicyCacheHandler(d) 1626 1627 // /policy/resolve/ 1628 api.PolicyGetPolicyResolveHandler = NewGetPolicyResolveHandler(d) 1629 1630 // /service/{id}/ 1631 api.ServiceGetServiceIDHandler = NewGetServiceIDHandler(d) 1632 api.ServiceDeleteServiceIDHandler = NewDeleteServiceIDHandler(d) 1633 api.ServicePutServiceIDHandler = NewPutServiceIDHandler(d) 1634 1635 // /service/ 1636 api.ServiceGetServiceHandler = NewGetServiceHandler(d) 1637 1638 // /prefilter/ 1639 api.PrefilterGetPrefilterHandler = NewGetPrefilterHandler(d) 1640 api.PrefilterDeletePrefilterHandler = NewDeletePrefilterHandler(d) 1641 api.PrefilterPatchPrefilterHandler = NewPatchPrefilterHandler(d) 1642 1643 // /ipam/{ip}/ 1644 api.IPAMPostIPAMHandler = NewPostIPAMHandler(d) 1645 api.IPAMPostIPAMIPHandler = NewPostIPAMIPHandler(d) 1646 api.IPAMDeleteIPAMIPHandler = NewDeleteIPAMIPHandler(d) 1647 1648 // /debuginfo 1649 api.DaemonGetDebuginfoHandler = NewGetDebugInfoHandler(d) 1650 1651 // /map 1652 api.DaemonGetMapHandler = NewGetMapHandler(d) 1653 api.DaemonGetMapNameHandler = NewGetMapNameHandler(d) 1654 1655 // metrics 1656 api.MetricsGetMetricsHandler = NewGetMetricsHandler(d) 1657 1658 // /fqdn/cache 1659 api.PolicyGetFqdnCacheHandler = NewGetFqdnCacheHandler(d) 1660 api.PolicyDeleteFqdnCacheHandler = NewDeleteFqdnCacheHandler(d) 1661 api.PolicyGetFqdnCacheIDHandler = NewGetFqdnCacheIDHandler(d) 1662 1663 return api 1664 }