github.com/moby/docker@v26.1.3+incompatible/cmd/dockerd/daemon.go (about) 1 package main 2 3 import ( 4 "context" 5 "crypto/tls" 6 "fmt" 7 "net" 8 "net/http" 9 "os" 10 "path/filepath" 11 "runtime" 12 "sort" 13 "strings" 14 "sync" 15 "time" 16 17 containerddefaults "github.com/containerd/containerd/defaults" 18 "github.com/containerd/containerd/tracing" 19 "github.com/containerd/log" 20 "github.com/docker/docker/api" 21 apiserver "github.com/docker/docker/api/server" 22 buildbackend "github.com/docker/docker/api/server/backend/build" 23 "github.com/docker/docker/api/server/middleware" 24 "github.com/docker/docker/api/server/router" 25 "github.com/docker/docker/api/server/router/build" 26 checkpointrouter "github.com/docker/docker/api/server/router/checkpoint" 27 "github.com/docker/docker/api/server/router/container" 28 distributionrouter "github.com/docker/docker/api/server/router/distribution" 29 grpcrouter "github.com/docker/docker/api/server/router/grpc" 30 "github.com/docker/docker/api/server/router/image" 31 "github.com/docker/docker/api/server/router/network" 32 pluginrouter "github.com/docker/docker/api/server/router/plugin" 33 sessionrouter "github.com/docker/docker/api/server/router/session" 34 swarmrouter "github.com/docker/docker/api/server/router/swarm" 35 systemrouter "github.com/docker/docker/api/server/router/system" 36 "github.com/docker/docker/api/server/router/volume" 37 buildkit "github.com/docker/docker/builder/builder-next" 38 "github.com/docker/docker/builder/dockerfile" 39 "github.com/docker/docker/cli/debug" 40 "github.com/docker/docker/cmd/dockerd/trap" 41 "github.com/docker/docker/daemon" 42 "github.com/docker/docker/daemon/cluster" 43 "github.com/docker/docker/daemon/config" 44 "github.com/docker/docker/daemon/listeners" 45 "github.com/docker/docker/dockerversion" 46 "github.com/docker/docker/libcontainerd/supervisor" 47 dopts "github.com/docker/docker/opts" 48 "github.com/docker/docker/pkg/authorization" 49 "github.com/docker/docker/pkg/homedir" 50 "github.com/docker/docker/pkg/pidfile" 51 "github.com/docker/docker/pkg/plugingetter" 52 "github.com/docker/docker/pkg/rootless" 53 "github.com/docker/docker/pkg/sysinfo" 54 "github.com/docker/docker/pkg/system" 55 "github.com/docker/docker/plugin" 56 "github.com/docker/docker/runconfig" 57 "github.com/docker/go-connections/tlsconfig" 58 "github.com/moby/buildkit/session" 59 "github.com/moby/buildkit/util/tracing/detect" 60 swarmapi "github.com/moby/swarmkit/v2/api" 61 "github.com/pkg/errors" 62 "github.com/sirupsen/logrus" 63 "github.com/spf13/pflag" 64 "go.opentelemetry.io/otel" 65 "go.opentelemetry.io/otel/propagation" 66 "go.opentelemetry.io/otel/sdk/resource" 67 "tags.cncf.io/container-device-interface/pkg/cdi" 68 ) 69 70 // DaemonCli represents the daemon CLI. 71 type DaemonCli struct { 72 *config.Config 73 configFile *string 74 flags *pflag.FlagSet 75 76 d *daemon.Daemon 77 authzMiddleware *authorization.Middleware // authzMiddleware enables to dynamically reload the authorization plugins 78 79 stopOnce sync.Once 80 apiShutdown chan struct{} 81 } 82 83 // NewDaemonCli returns a daemon CLI 84 func NewDaemonCli() *DaemonCli { 85 return &DaemonCli{ 86 apiShutdown: make(chan struct{}), 87 } 88 } 89 90 func (cli *DaemonCli) start(opts *daemonOptions) (err error) { 91 ctx := context.TODO() 92 93 if cli.Config, err = loadDaemonCliConfig(opts); err != nil { 94 return err 95 } 96 97 tlsConfig, err := newAPIServerTLSConfig(cli.Config) 98 if err != nil { 99 return err 100 } 101 102 if opts.Validate { 103 // If config wasn't OK we wouldn't have made it this far. 104 _, _ = fmt.Fprintln(os.Stderr, "configuration OK") 105 return nil 106 } 107 108 configureProxyEnv(cli.Config) 109 configureDaemonLogs(cli.Config) 110 111 log.G(ctx).Info("Starting up") 112 113 cli.configFile = &opts.configFile 114 cli.flags = opts.flags 115 116 if cli.Config.Debug { 117 debug.Enable() 118 } 119 120 if cli.Config.Experimental { 121 log.G(ctx).Warn("Running experimental build") 122 } 123 124 if cli.Config.IsRootless() { 125 log.G(ctx).Warn("Running in rootless mode. This mode has feature limitations.") 126 } 127 if rootless.RunningWithRootlessKit() { 128 log.G(ctx).Info("Running with RootlessKit integration") 129 if !cli.Config.IsRootless() { 130 return fmt.Errorf("rootless mode needs to be enabled for running with RootlessKit") 131 } 132 } 133 134 // return human-friendly error before creating files 135 if runtime.GOOS == "linux" && os.Geteuid() != 0 { 136 return fmt.Errorf("dockerd needs to be started with root privileges. To run dockerd in rootless mode as an unprivileged user, see https://docs.docker.com/go/rootless/") 137 } 138 139 if err := setDefaultUmask(); err != nil { 140 return err 141 } 142 143 // Create the daemon root before we create ANY other files (PID, or migrate keys) 144 // to ensure the appropriate ACL is set (particularly relevant on Windows) 145 if err := daemon.CreateDaemonRoot(cli.Config); err != nil { 146 return err 147 } 148 149 if err := system.MkdirAll(cli.Config.ExecRoot, 0o700); err != nil { 150 return err 151 } 152 153 potentiallyUnderRuntimeDir := []string{cli.Config.ExecRoot} 154 155 if cli.Pidfile != "" { 156 if err = system.MkdirAll(filepath.Dir(cli.Pidfile), 0o755); err != nil { 157 return errors.Wrap(err, "failed to create pidfile directory") 158 } 159 if err = pidfile.Write(cli.Pidfile, os.Getpid()); err != nil { 160 return errors.Wrapf(err, "failed to start daemon, ensure docker is not running or delete %s", cli.Pidfile) 161 } 162 potentiallyUnderRuntimeDir = append(potentiallyUnderRuntimeDir, cli.Pidfile) 163 defer func() { 164 if err := os.Remove(cli.Pidfile); err != nil { 165 log.G(ctx).Error(err) 166 } 167 }() 168 } 169 170 if cli.Config.IsRootless() { 171 // Set sticky bit if XDG_RUNTIME_DIR is set && the file is actually under XDG_RUNTIME_DIR 172 if _, err := homedir.StickRuntimeDirContents(potentiallyUnderRuntimeDir); err != nil { 173 // StickRuntimeDirContents returns nil error if XDG_RUNTIME_DIR is just unset 174 log.G(ctx).WithError(err).Warn("cannot set sticky bit on files under XDG_RUNTIME_DIR") 175 } 176 } 177 178 lss, hosts, err := loadListeners(cli.Config, tlsConfig) 179 if err != nil { 180 return errors.Wrap(err, "failed to load listeners") 181 } 182 183 ctx, cancel := context.WithCancel(context.Background()) 184 waitForContainerDShutdown, err := cli.initContainerd(ctx) 185 if waitForContainerDShutdown != nil { 186 defer waitForContainerDShutdown(10 * time.Second) 187 } 188 if err != nil { 189 cancel() 190 return err 191 } 192 defer cancel() 193 194 httpServer := &http.Server{ 195 ReadHeaderTimeout: 5 * time.Minute, // "G112: Potential Slowloris Attack (gosec)"; not a real concern for our use, so setting a long timeout. 196 } 197 apiShutdownCtx, apiShutdownCancel := context.WithCancel(context.Background()) 198 apiShutdownDone := make(chan struct{}) 199 trap.Trap(cli.stop) 200 go func() { 201 // Block until cli.stop() has been called. 202 // It may have already been called, and that's okay. 203 // Any httpServer.Serve() calls made after 204 // httpServer.Shutdown() will return immediately, 205 // which is what we want. 206 <-cli.apiShutdown 207 err := httpServer.Shutdown(apiShutdownCtx) 208 if err != nil { 209 log.G(ctx).WithError(err).Error("Error shutting down http server") 210 } 211 close(apiShutdownDone) 212 }() 213 defer func() { 214 select { 215 case <-cli.apiShutdown: 216 // cli.stop() has been called and the daemon has completed 217 // shutting down. Give the HTTP server a little more time to 218 // finish handling any outstanding requests if needed. 219 tmr := time.AfterFunc(5*time.Second, apiShutdownCancel) 220 defer tmr.Stop() 221 <-apiShutdownDone 222 default: 223 // cli.start() has returned without cli.stop() being called, 224 // e.g. because the daemon failed to start. 225 // Stop the HTTP server with no grace period. 226 if closeErr := httpServer.Close(); closeErr != nil { 227 log.G(ctx).WithError(closeErr).Error("Error closing http server") 228 } 229 } 230 }() 231 232 // Notify that the API is active, but before daemon is set up. 233 preNotifyReady() 234 235 const otelServiceNameEnv = "OTEL_SERVICE_NAME" 236 if _, ok := os.LookupEnv(otelServiceNameEnv); !ok { 237 os.Setenv(otelServiceNameEnv, filepath.Base(os.Args[0])) 238 } 239 240 setOTLPProtoDefault() 241 otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) 242 243 // Override BuildKit's default Resource so that it matches the semconv 244 // version that is used in our code. 245 detect.OverrideResource(resource.Default()) 246 detect.Recorder = detect.NewTraceRecorder() 247 248 tp, err := detect.TracerProvider() 249 if err != nil { 250 log.G(ctx).WithError(err).Warn("Failed to initialize tracing, skipping") 251 } else { 252 otel.SetTracerProvider(tp) 253 log.G(ctx).Logger.AddHook(tracing.NewLogrusHook()) 254 } 255 256 pluginStore := plugin.NewStore() 257 258 var apiServer apiserver.Server 259 cli.authzMiddleware, err = initMiddlewares(&apiServer, cli.Config, pluginStore) 260 if err != nil { 261 return errors.Wrap(err, "failed to start API server") 262 } 263 264 d, err := daemon.NewDaemon(ctx, cli.Config, pluginStore, cli.authzMiddleware) 265 if err != nil { 266 return errors.Wrap(err, "failed to start daemon") 267 } 268 269 d.StoreHosts(hosts) 270 271 // validate after NewDaemon has restored enabled plugins. Don't change order. 272 if err := validateAuthzPlugins(cli.Config.AuthorizationPlugins, pluginStore); err != nil { 273 return errors.Wrap(err, "failed to validate authorization plugin") 274 } 275 276 // Note that CDI is not inherently linux-specific, there are some linux-specific assumptions / implementations in the code that 277 // queries the properties of device on the host as wel as performs the injection of device nodes and their access permissions into the OCI spec. 278 // 279 // In order to lift this restriction the following would have to be addressed: 280 // - Support needs to be added to the cdi package for injecting Windows devices: https://tags.cncf.io/container-device-interface/issues/28 281 // - The DeviceRequests API must be extended to non-linux platforms. 282 if runtime.GOOS == "linux" && cli.Config.Features["cdi"] { 283 daemon.RegisterCDIDriver(cli.Config.CDISpecDirs...) 284 } 285 286 cli.d = d 287 288 if err := startMetricsServer(cli.Config.MetricsAddress); err != nil { 289 return errors.Wrap(err, "failed to start metrics server") 290 } 291 292 c, err := createAndStartCluster(cli, d) 293 if err != nil { 294 log.G(ctx).Fatalf("Error starting cluster component: %v", err) 295 } 296 297 // Restart all autostart containers which has a swarm endpoint 298 // and is not yet running now that we have successfully 299 // initialized the cluster. 300 d.RestartSwarmContainers() 301 302 log.G(ctx).Info("Daemon has completed initialization") 303 304 routerCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 305 defer cancel() 306 307 // Get a the current daemon config, because the daemon sets up config 308 // during initialization. We cannot user the cli.Config for that reason, 309 // as that only holds the config that was set by the user. 310 // 311 // FIXME(thaJeztah): better separate runtime and config data? 312 daemonCfg := d.Config() 313 routerOpts, err := newRouterOptions(routerCtx, &daemonCfg, d, c) 314 if err != nil { 315 return err 316 } 317 318 httpServer.Handler = apiServer.CreateMux(routerOpts.Build()...) 319 320 go d.ProcessClusterNotifications(ctx, c.GetWatchStream()) 321 322 cli.setupConfigReloadTrap() 323 324 // after the daemon is done setting up we can notify systemd api 325 notifyReady() 326 327 // Daemon is fully initialized. Start handling API traffic 328 // and wait for serve API to complete. 329 var ( 330 apiWG sync.WaitGroup 331 errAPI = make(chan error, 1) 332 ) 333 for _, ls := range lss { 334 apiWG.Add(1) 335 go func(ls net.Listener) { 336 defer apiWG.Done() 337 log.G(ctx).Infof("API listen on %s", ls.Addr()) 338 if err := httpServer.Serve(ls); err != http.ErrServerClosed { 339 log.G(ctx).WithFields(log.Fields{ 340 "error": err, 341 "listener": ls.Addr(), 342 }).Error("ServeAPI error") 343 344 select { 345 case errAPI <- err: 346 default: 347 } 348 } 349 }(ls) 350 } 351 apiWG.Wait() 352 close(errAPI) 353 354 c.Cleanup() 355 356 // notify systemd that we're shutting down 357 notifyStopping() 358 shutdownDaemon(ctx, d) 359 360 if err := routerOpts.buildkit.Close(); err != nil { 361 log.G(ctx).WithError(err).Error("Failed to close buildkit") 362 } 363 364 // Stop notification processing and any background processes 365 cancel() 366 367 if err, ok := <-errAPI; ok { 368 return errors.Wrap(err, "shutting down due to ServeAPI error") 369 } 370 371 detect.Shutdown(context.Background()) 372 373 log.G(ctx).Info("Daemon shutdown complete") 374 return nil 375 } 376 377 // The buildkit "detect" package uses grpc as the default proto, which is in conformance with the old spec. 378 // For a little while now http/protobuf is the default spec, so this function sets the protocol to http/protobuf when the env var is unset 379 // so that the detect package will use http/protobuf as a default. 380 // TODO: This can be removed after buildkit is updated to use http/protobuf as the default. 381 func setOTLPProtoDefault() { 382 const ( 383 tracesEnv = "OTEL_EXPORTER_OTLP_TRACES_PROTOCOL" 384 metricsEnv = "OTEL_EXPORTER_OTLP_METRICS_PROTOCOL" 385 protoEnv = "OTEL_EXPORTER_OTLP_PROTOCOL" 386 387 defaultProto = "http/protobuf" 388 ) 389 390 if os.Getenv(protoEnv) == "" { 391 if os.Getenv(tracesEnv) == "" { 392 os.Setenv(tracesEnv, defaultProto) 393 } 394 if os.Getenv(metricsEnv) == "" { 395 os.Setenv(metricsEnv, defaultProto) 396 } 397 } 398 } 399 400 type routerOptions struct { 401 sessionManager *session.Manager 402 buildBackend *buildbackend.Backend 403 features func() map[string]bool 404 buildkit *buildkit.Builder 405 daemon *daemon.Daemon 406 cluster *cluster.Cluster 407 } 408 409 func newRouterOptions(ctx context.Context, config *config.Config, d *daemon.Daemon, c *cluster.Cluster) (routerOptions, error) { 410 sm, err := session.NewManager() 411 if err != nil { 412 return routerOptions{}, errors.Wrap(err, "failed to create sessionmanager") 413 } 414 415 manager, err := dockerfile.NewBuildManager(d.BuilderBackend(), d.IdentityMapping()) 416 if err != nil { 417 return routerOptions{}, err 418 } 419 cgroupParent := newCgroupParent(config) 420 421 bk, err := buildkit.New(ctx, buildkit.Opt{ 422 SessionManager: sm, 423 Root: filepath.Join(config.Root, "buildkit"), 424 EngineID: d.ID(), 425 Dist: d.DistributionServices(), 426 ImageTagger: d.ImageService(), 427 NetworkController: d.NetworkController(), 428 DefaultCgroupParent: cgroupParent, 429 RegistryHosts: d.RegistryHosts, 430 BuilderConfig: config.Builder, 431 Rootless: daemon.Rootless(config), 432 IdentityMapping: d.IdentityMapping(), 433 DNSConfig: config.DNSConfig, 434 ApparmorProfile: daemon.DefaultApparmorProfile(), 435 UseSnapshotter: d.UsesSnapshotter(), 436 Snapshotter: d.ImageService().StorageDriver(), 437 ContainerdAddress: config.ContainerdAddr, 438 ContainerdNamespace: config.ContainerdNamespace, 439 }) 440 if err != nil { 441 return routerOptions{}, err 442 } 443 444 bb, err := buildbackend.NewBackend(d.ImageService(), manager, bk, d.EventsService) 445 if err != nil { 446 return routerOptions{}, errors.Wrap(err, "failed to create buildmanager") 447 } 448 449 return routerOptions{ 450 sessionManager: sm, 451 buildBackend: bb, 452 features: d.Features, 453 buildkit: bk, 454 daemon: d, 455 cluster: c, 456 }, nil 457 } 458 459 func (cli *DaemonCli) reloadConfig() { 460 ctx := context.TODO() 461 reload := func(c *config.Config) { 462 if err := validateAuthzPlugins(c.AuthorizationPlugins, cli.d.PluginStore); err != nil { 463 log.G(ctx).Fatalf("Error validating authorization plugin: %v", err) 464 return 465 } 466 467 if err := cli.d.Reload(c); err != nil { 468 log.G(ctx).Errorf("Error reconfiguring the daemon: %v", err) 469 return 470 } 471 472 // Apply our own configuration only after the daemon reload has succeeded. We 473 // don't want to partially apply the config if the daemon is unhappy with it. 474 475 cli.authzMiddleware.SetPlugins(c.AuthorizationPlugins) 476 477 if c.IsValueSet("debug") { 478 debugEnabled := debug.IsEnabled() 479 switch { 480 case debugEnabled && !c.Debug: // disable debug 481 debug.Disable() 482 case c.Debug && !debugEnabled: // enable debug 483 debug.Enable() 484 } 485 } 486 } 487 488 if err := config.Reload(*cli.configFile, cli.flags, reload); err != nil { 489 log.G(ctx).Error(err) 490 } 491 } 492 493 func (cli *DaemonCli) stop() { 494 // Signal that the API server should shut down as soon as possible. 495 // This construct is used rather than directly shutting down the HTTP 496 // server to avoid any issues if this method is called before the server 497 // has been instantiated in cli.start(). If this method is called first, 498 // the HTTP server will be shut down immediately upon instantiation. 499 cli.stopOnce.Do(func() { 500 close(cli.apiShutdown) 501 }) 502 } 503 504 // shutdownDaemon just wraps daemon.Shutdown() to handle a timeout in case 505 // d.Shutdown() is waiting too long to kill container or worst it's 506 // blocked there 507 func shutdownDaemon(ctx context.Context, d *daemon.Daemon) { 508 var cancel context.CancelFunc 509 if timeout := d.ShutdownTimeout(); timeout >= 0 { 510 ctx, cancel = context.WithTimeout(ctx, time.Duration(timeout)*time.Second) 511 } else { 512 ctx, cancel = context.WithCancel(ctx) 513 } 514 515 go func() { 516 defer cancel() 517 d.Shutdown(ctx) 518 }() 519 520 <-ctx.Done() 521 if errors.Is(ctx.Err(), context.DeadlineExceeded) { 522 log.G(ctx).Error("Force shutdown daemon") 523 } else { 524 log.G(ctx).Debug("Clean shutdown succeeded") 525 } 526 } 527 528 func loadDaemonCliConfig(opts *daemonOptions) (*config.Config, error) { 529 if !opts.flags.Parsed() { 530 return nil, errors.New(`cannot load CLI config before flags are parsed`) 531 } 532 opts.setDefaultOptions() 533 534 conf := opts.daemonConfig 535 flags := opts.flags 536 conf.Debug = opts.Debug 537 conf.Hosts = opts.Hosts 538 conf.LogLevel = opts.LogLevel 539 conf.LogFormat = log.OutputFormat(opts.LogFormat) 540 541 // The DOCKER_MIN_API_VERSION env-var allows overriding the minimum API 542 // version provided by the daemon within constraints of the minimum and 543 // maximum (current) supported API versions. 544 // 545 // API versions older than [config.defaultMinAPIVersion] are deprecated and 546 // to be removed in a future release. The "DOCKER_MIN_API_VERSION" env-var 547 // should only be used for exceptional cases. 548 if ver := os.Getenv("DOCKER_MIN_API_VERSION"); ver != "" { 549 if err := config.ValidateMinAPIVersion(ver); err != nil { 550 return nil, errors.Wrap(err, "invalid DOCKER_MIN_API_VERSION") 551 } 552 conf.MinAPIVersion = ver 553 } 554 555 if flags.Changed(FlagTLS) { 556 conf.TLS = &opts.TLS 557 } 558 if flags.Changed(FlagTLSVerify) { 559 conf.TLSVerify = &opts.TLSVerify 560 v := true 561 conf.TLS = &v 562 } 563 564 if opts.TLSOptions != nil { 565 conf.TLSOptions = config.TLSOptions{ 566 CAFile: opts.TLSOptions.CAFile, 567 CertFile: opts.TLSOptions.CertFile, 568 KeyFile: opts.TLSOptions.KeyFile, 569 } 570 } else { 571 conf.TLSOptions = config.TLSOptions{} 572 } 573 574 if opts.configFile != "" { 575 c, err := config.MergeDaemonConfigurations(conf, flags, opts.configFile) 576 if err != nil { 577 if flags.Changed("config-file") || !os.IsNotExist(err) { 578 return nil, errors.Wrapf(err, "unable to configure the Docker daemon with file %s", opts.configFile) 579 } 580 } 581 582 // the merged configuration can be nil if the config file didn't exist. 583 // leave the current configuration as it is if when that happens. 584 if c != nil { 585 conf = c 586 } 587 } 588 589 if err := normalizeHosts(conf); err != nil { 590 return nil, err 591 } 592 593 if err := config.Validate(conf); err != nil { 594 return nil, err 595 } 596 597 // Check if duplicate label-keys with different values are found 598 newLabels, err := config.GetConflictFreeLabels(conf.Labels) 599 if err != nil { 600 return nil, err 601 } 602 conf.Labels = newLabels 603 604 // Regardless of whether the user sets it to true or false, if they 605 // specify TLSVerify at all then we need to turn on TLS 606 if conf.IsValueSet(FlagTLSVerify) { 607 v := true 608 conf.TLS = &v 609 } 610 611 if conf.TLSVerify == nil && conf.TLS != nil { 612 conf.TLSVerify = conf.TLS 613 } 614 615 err = validateCPURealtimeOptions(conf) 616 if err != nil { 617 return nil, err 618 } 619 620 if conf.CDISpecDirs == nil { 621 // If the CDISpecDirs is not set at this stage, we set it to the default. 622 conf.CDISpecDirs = append([]string(nil), cdi.DefaultSpecDirs...) 623 } else if len(conf.CDISpecDirs) == 1 && conf.CDISpecDirs[0] == "" { 624 // If CDISpecDirs is set to an empty string, we clear it to ensure that CDI is disabled. 625 conf.CDISpecDirs = nil 626 } 627 if !conf.Features["cdi"] { 628 // If the CDI feature is not enabled, we clear the CDISpecDirs to ensure that CDI is disabled. 629 conf.CDISpecDirs = nil 630 } 631 632 if err := loadCLIPlatformConfig(conf); err != nil { 633 return nil, err 634 } 635 636 return conf, nil 637 } 638 639 // normalizeHosts normalizes the configured config.Hosts and remove duplicates. 640 // It returns an error if it fails to parse a host. 641 func normalizeHosts(config *config.Config) error { 642 if len(config.Hosts) == 0 { 643 // if no hosts are configured, create a single entry slice, so that the 644 // default is used. 645 // 646 // TODO(thaJeztah) implement a cleaner way for this; this depends on a 647 // side-effect of how we parse empty/partial hosts. 648 config.Hosts = make([]string, 1) 649 } 650 hosts := make([]string, 0, len(config.Hosts)) 651 seen := make(map[string]struct{}, len(config.Hosts)) 652 653 useTLS := DefaultTLSValue 654 if config.TLS != nil { 655 useTLS = *config.TLS 656 } 657 658 for _, h := range config.Hosts { 659 host, err := dopts.ParseHost(useTLS, honorXDG, h) 660 if err != nil { 661 return err 662 } 663 if _, ok := seen[host]; ok { 664 continue 665 } 666 seen[host] = struct{}{} 667 hosts = append(hosts, host) 668 } 669 sort.Strings(hosts) 670 config.Hosts = hosts 671 return nil 672 } 673 674 func (opts routerOptions) Build() []router.Router { 675 decoder := runconfig.ContainerDecoder{ 676 GetSysInfo: func() *sysinfo.SysInfo { 677 return opts.daemon.RawSysInfo() 678 }, 679 } 680 681 routers := []router.Router{ 682 // we need to add the checkpoint router before the container router or the DELETE gets masked 683 checkpointrouter.NewRouter(opts.daemon, decoder), 684 container.NewRouter(opts.daemon, decoder, opts.daemon.RawSysInfo().CgroupUnified), 685 image.NewRouter( 686 opts.daemon.ImageService(), 687 opts.daemon.RegistryService(), 688 opts.daemon.ReferenceStore, 689 opts.daemon.ImageService().DistributionServices().ImageStore, 690 opts.daemon.ImageService().DistributionServices().LayerStore, 691 ), 692 systemrouter.NewRouter(opts.daemon, opts.cluster, opts.buildkit, opts.daemon.Features), 693 volume.NewRouter(opts.daemon.VolumesService(), opts.cluster), 694 build.NewRouter(opts.buildBackend, opts.daemon), 695 sessionrouter.NewRouter(opts.sessionManager), 696 swarmrouter.NewRouter(opts.cluster), 697 pluginrouter.NewRouter(opts.daemon.PluginManager()), 698 distributionrouter.NewRouter(opts.daemon.ImageBackend()), 699 } 700 701 if opts.buildBackend != nil { 702 routers = append(routers, grpcrouter.NewRouter(opts.buildBackend)) 703 } 704 705 if opts.daemon.NetworkControllerEnabled() { 706 routers = append(routers, network.NewRouter(opts.daemon, opts.cluster)) 707 } 708 709 if opts.daemon.HasExperimental() { 710 for _, r := range routers { 711 for _, route := range r.Routes() { 712 if experimental, ok := route.(router.ExperimentalRoute); ok { 713 experimental.Enable() 714 } 715 } 716 } 717 } 718 719 return routers 720 } 721 722 func initMiddlewares(s *apiserver.Server, cfg *config.Config, pluginStore plugingetter.PluginGetter) (*authorization.Middleware, error) { 723 exp := middleware.NewExperimentalMiddleware(cfg.Experimental) 724 s.UseMiddleware(exp) 725 726 vm, err := middleware.NewVersionMiddleware(dockerversion.Version, api.DefaultVersion, cfg.MinAPIVersion) 727 if err != nil { 728 return nil, err 729 } 730 s.UseMiddleware(*vm) 731 732 if cfg.CorsHeaders != "" { 733 c := middleware.NewCORSMiddleware(cfg.CorsHeaders) 734 s.UseMiddleware(c) 735 } 736 737 authzMiddleware := authorization.NewMiddleware(cfg.AuthorizationPlugins, pluginStore) 738 s.UseMiddleware(authzMiddleware) 739 return authzMiddleware, nil 740 } 741 742 func (cli *DaemonCli) getContainerdDaemonOpts() ([]supervisor.DaemonOpt, error) { 743 var opts []supervisor.DaemonOpt 744 if cli.Debug { 745 opts = append(opts, supervisor.WithLogLevel("debug")) 746 } else { 747 opts = append(opts, supervisor.WithLogLevel(cli.LogLevel)) 748 } 749 750 if logFormat := cli.Config.LogFormat; logFormat != "" { 751 opts = append(opts, supervisor.WithLogFormat(logFormat)) 752 } 753 754 if !cli.CriContainerd { 755 // CRI support in the managed daemon is currently opt-in. 756 // 757 // It's disabled by default, originally because it was listening on 758 // a TCP connection at 0.0.0.0:10010, which was considered a security 759 // risk, and could conflict with user's container ports. 760 // 761 // Current versions of containerd started now listen on localhost on 762 // an ephemeral port instead, but could still conflict with container 763 // ports, and running kubernetes using the static binaries is not a 764 // common scenario, so we (for now) continue disabling it by default. 765 // 766 // Also see https://github.com/containerd/containerd/issues/2483#issuecomment-407530608 767 opts = append(opts, supervisor.WithCRIDisabled()) 768 } 769 770 return opts, nil 771 } 772 773 func newAPIServerTLSConfig(config *config.Config) (*tls.Config, error) { 774 var tlsConfig *tls.Config 775 if config.TLS != nil && *config.TLS { 776 var ( 777 clientAuth tls.ClientAuthType 778 err error 779 ) 780 if config.TLSVerify == nil || *config.TLSVerify { 781 // server requires and verifies client's certificate 782 clientAuth = tls.RequireAndVerifyClientCert 783 } 784 tlsConfig, err = tlsconfig.Server(tlsconfig.Options{ 785 CAFile: config.TLSOptions.CAFile, 786 CertFile: config.TLSOptions.CertFile, 787 KeyFile: config.TLSOptions.KeyFile, 788 ExclusiveRootPools: true, 789 ClientAuth: clientAuth, 790 }) 791 if err != nil { 792 return nil, errors.Wrap(err, "invalid TLS configuration") 793 } 794 } 795 796 return tlsConfig, nil 797 } 798 799 // checkTLSAuthOK checks basically for an explicitly disabled TLS/TLSVerify 800 // Going forward we do not want to support a scenario where dockerd listens 801 // on TCP without either TLS client auth (or an explicit opt-in to disable it) 802 func checkTLSAuthOK(c *config.Config) bool { 803 if c.TLS == nil { 804 // Either TLS is enabled by default, in which case TLS verification should be enabled by default, or explicitly disabled 805 // Or TLS is disabled by default... in any of these cases, we can just take the default value as to how to proceed 806 return DefaultTLSValue 807 } 808 809 if !*c.TLS { 810 // TLS is explicitly disabled, which is supported 811 return true 812 } 813 814 if c.TLSVerify == nil { 815 // this actually shouldn't happen since we set TLSVerify on the config object anyway 816 // But in case it does get here, be cautious and assume this is not supported. 817 return false 818 } 819 820 // Either TLSVerify is explicitly enabled or disabled, both cases are supported 821 return true 822 } 823 824 func loadListeners(cfg *config.Config, tlsConfig *tls.Config) ([]net.Listener, []string, error) { 825 ctx := context.TODO() 826 827 if len(cfg.Hosts) == 0 { 828 return nil, nil, errors.New("no hosts configured") 829 } 830 var ( 831 hosts []string 832 lss []net.Listener 833 ) 834 835 for i := 0; i < len(cfg.Hosts); i++ { 836 protoAddr := cfg.Hosts[i] 837 proto, addr, ok := strings.Cut(protoAddr, "://") 838 if !ok { 839 return nil, nil, fmt.Errorf("bad format %s, expected PROTO://ADDR", protoAddr) 840 } 841 842 // It's a bad idea to bind to TCP without tlsverify. 843 authEnabled := tlsConfig != nil && tlsConfig.ClientAuth == tls.RequireAndVerifyClientCert 844 if proto == "tcp" && !authEnabled { 845 log.G(ctx).WithField("host", protoAddr).Warn("Binding to IP address without --tlsverify is insecure and gives root access on this machine to everyone who has access to your network.") 846 log.G(ctx).WithField("host", protoAddr).Warn("Binding to an IP address, even on localhost, can also give access to scripts run in a browser. Be safe out there!") 847 log.G(ctx).WithField("host", protoAddr).Warn("[DEPRECATION NOTICE] In future versions this will be a hard failure preventing the daemon from starting! Learn more at: https://docs.docker.com/go/api-security/") 848 time.Sleep(time.Second) 849 850 // If TLSVerify is explicitly set to false we'll take that as "Please let me shoot myself in the foot" 851 // We do not want to continue to support a default mode where tls verification is disabled, so we do some extra warnings here and eventually remove support 852 if !checkTLSAuthOK(cfg) { 853 ipAddr, _, err := net.SplitHostPort(addr) 854 if err != nil { 855 return nil, nil, errors.Wrap(err, "error parsing tcp address") 856 } 857 858 // shortcut all this extra stuff for literal "localhost" 859 // -H supports specifying hostnames, since we want to bypass this on loopback interfaces we'll look it up here. 860 if ipAddr != "localhost" { 861 ip := net.ParseIP(ipAddr) 862 if ip == nil { 863 ipA, err := net.ResolveIPAddr("ip", ipAddr) 864 if err != nil { 865 log.G(ctx).WithError(err).WithField("host", ipAddr).Error("Error looking up specified host address") 866 } 867 if ipA != nil { 868 ip = ipA.IP 869 } 870 } 871 if ip == nil || !ip.IsLoopback() { 872 log.G(ctx).WithField("host", protoAddr).Warn("Binding to an IP address without --tlsverify is deprecated. Startup is intentionally being slowed down to show this message") 873 log.G(ctx).WithField("host", protoAddr).Warn("Please consider generating tls certificates with client validation to prevent exposing unauthenticated root access to your network") 874 log.G(ctx).WithField("host", protoAddr).Warnf("You can override this by explicitly specifying '--%s=false' or '--%s=false'", FlagTLS, FlagTLSVerify) 875 log.G(ctx).WithField("host", protoAddr).Warnf("Support for listening on TCP without authentication or explicit intent to run without authentication will be removed in the next release") 876 877 time.Sleep(15 * time.Second) 878 } 879 } 880 } 881 } 882 // If we're binding to a TCP port, make sure that a container doesn't try to use it. 883 if proto == "tcp" { 884 if err := allocateDaemonPort(addr); err != nil { 885 return nil, nil, err 886 } 887 } 888 ls, err := listeners.Init(proto, addr, cfg.SocketGroup, tlsConfig) 889 if err != nil { 890 return nil, nil, err 891 } 892 log.G(ctx).Debugf("Listener created for HTTP on %s (%s)", proto, addr) 893 hosts = append(hosts, addr) 894 lss = append(lss, ls...) 895 } 896 897 return lss, hosts, nil 898 } 899 900 func createAndStartCluster(cli *DaemonCli, d *daemon.Daemon) (*cluster.Cluster, error) { 901 name, _ := os.Hostname() 902 903 // Use a buffered channel to pass changes from store watch API to daemon 904 // A buffer allows store watch API and daemon processing to not wait for each other 905 watchStream := make(chan *swarmapi.WatchMessage, 32) 906 907 c, err := cluster.New(cluster.Config{ 908 Root: cli.Config.Root, 909 Name: name, 910 Backend: d, 911 VolumeBackend: d.VolumesService(), 912 ImageBackend: d.ImageBackend(), 913 PluginBackend: d.PluginManager(), 914 NetworkSubnetsProvider: d, 915 DefaultAdvertiseAddr: cli.Config.SwarmDefaultAdvertiseAddr, 916 RaftHeartbeatTick: cli.Config.SwarmRaftHeartbeatTick, 917 RaftElectionTick: cli.Config.SwarmRaftElectionTick, 918 RuntimeRoot: cli.getSwarmRunRoot(), 919 WatchStream: watchStream, 920 }) 921 if err != nil { 922 return nil, err 923 } 924 d.SetCluster(c) 925 err = c.Start() 926 927 return c, err 928 } 929 930 // validates that the plugins requested with the --authorization-plugin flag are valid AuthzDriver 931 // plugins present on the host and available to the daemon 932 func validateAuthzPlugins(requestedPlugins []string, pg plugingetter.PluginGetter) error { 933 for _, reqPlugin := range requestedPlugins { 934 if _, err := pg.Get(reqPlugin, authorization.AuthZApiImplements, plugingetter.Lookup); err != nil { 935 return err 936 } 937 } 938 return nil 939 } 940 941 func systemContainerdRunning(honorXDG bool) (string, bool, error) { 942 addr := containerddefaults.DefaultAddress 943 if honorXDG { 944 runtimeDir, err := homedir.GetRuntimeDir() 945 if err != nil { 946 return "", false, err 947 } 948 addr = filepath.Join(runtimeDir, "containerd", "containerd.sock") 949 } 950 _, err := os.Lstat(addr) 951 return addr, err == nil, nil 952 } 953 954 // configureDaemonLogs sets the logging level and formatting. It expects 955 // the passed configuration to already be validated, and ignores invalid options. 956 func configureDaemonLogs(conf *config.Config) { 957 switch conf.LogFormat { 958 case log.JSONFormat: 959 if err := log.SetFormat(log.JSONFormat); err != nil { 960 panic(err.Error()) 961 } 962 case log.TextFormat, "": 963 if err := log.SetFormat(log.TextFormat); err != nil { 964 panic(err.Error()) 965 } 966 if conf.RawLogs { 967 // FIXME(thaJeztah): this needs a better solution: containerd doesn't allow disabling colors, and this code is depending on internal knowledge of "log.SetFormat" 968 if l, ok := log.L.Logger.Formatter.(*logrus.TextFormatter); ok { 969 l.DisableColors = true 970 } 971 } 972 default: 973 panic("unsupported log format " + conf.LogFormat) 974 } 975 976 logLevel := conf.LogLevel 977 if logLevel == "" { 978 logLevel = "info" 979 } 980 if err := log.SetLevel(logLevel); err != nil { 981 log.G(context.TODO()).WithError(err).Warn("configure log level") 982 } 983 } 984 985 func configureProxyEnv(conf *config.Config) { 986 if p := conf.HTTPProxy; p != "" { 987 overrideProxyEnv("HTTP_PROXY", p) 988 overrideProxyEnv("http_proxy", p) 989 } 990 if p := conf.HTTPSProxy; p != "" { 991 overrideProxyEnv("HTTPS_PROXY", p) 992 overrideProxyEnv("https_proxy", p) 993 } 994 if p := conf.NoProxy; p != "" { 995 overrideProxyEnv("NO_PROXY", p) 996 overrideProxyEnv("no_proxy", p) 997 } 998 } 999 1000 func overrideProxyEnv(name, val string) { 1001 if oldVal := os.Getenv(name); oldVal != "" && oldVal != val { 1002 log.G(context.TODO()).WithFields(log.Fields{ 1003 "name": name, 1004 "old-value": config.MaskCredentials(oldVal), 1005 "new-value": config.MaskCredentials(val), 1006 }).Warn("overriding existing proxy variable with value from configuration") 1007 } 1008 _ = os.Setenv(name, val) 1009 }