github.com/jiasir/docker@v1.3.3-0.20170609024000-252e610103e7/daemon/daemon.go (about) 1 // Package daemon exposes the functions that occur on the host server 2 // that the Docker daemon is running. 3 // 4 // In implementing the various functions of the daemon, there is often 5 // a method-specific struct for configuring the runtime behavior. 6 package daemon 7 8 import ( 9 "context" 10 "fmt" 11 "io/ioutil" 12 "net" 13 "os" 14 "path" 15 "path/filepath" 16 "runtime" 17 "strings" 18 "sync" 19 "time" 20 21 "github.com/Sirupsen/logrus" 22 containerd "github.com/containerd/containerd/api/grpc/types" 23 "github.com/docker/docker/api" 24 "github.com/docker/docker/api/types" 25 containertypes "github.com/docker/docker/api/types/container" 26 "github.com/docker/docker/container" 27 "github.com/docker/docker/daemon/config" 28 "github.com/docker/docker/daemon/discovery" 29 "github.com/docker/docker/daemon/events" 30 "github.com/docker/docker/daemon/exec" 31 "github.com/docker/docker/daemon/logger" 32 // register graph drivers 33 _ "github.com/docker/docker/daemon/graphdriver/register" 34 "github.com/docker/docker/daemon/initlayer" 35 "github.com/docker/docker/daemon/stats" 36 dmetadata "github.com/docker/docker/distribution/metadata" 37 "github.com/docker/docker/distribution/xfer" 38 "github.com/docker/docker/dockerversion" 39 "github.com/docker/docker/image" 40 "github.com/docker/docker/layer" 41 "github.com/docker/docker/libcontainerd" 42 "github.com/docker/docker/migrate/v1" 43 "github.com/docker/docker/pkg/fileutils" 44 "github.com/docker/docker/pkg/idtools" 45 "github.com/docker/docker/pkg/plugingetter" 46 "github.com/docker/docker/pkg/registrar" 47 "github.com/docker/docker/pkg/signal" 48 "github.com/docker/docker/pkg/sysinfo" 49 "github.com/docker/docker/pkg/system" 50 "github.com/docker/docker/pkg/truncindex" 51 "github.com/docker/docker/plugin" 52 refstore "github.com/docker/docker/reference" 53 "github.com/docker/docker/registry" 54 "github.com/docker/docker/runconfig" 55 volumedrivers "github.com/docker/docker/volume/drivers" 56 "github.com/docker/docker/volume/local" 57 "github.com/docker/docker/volume/store" 58 "github.com/docker/libnetwork" 59 "github.com/docker/libnetwork/cluster" 60 nwconfig "github.com/docker/libnetwork/config" 61 "github.com/docker/libtrust" 62 "github.com/pkg/errors" 63 ) 64 65 var ( 66 // DefaultRuntimeBinary is the default runtime to be used by 67 // containerd if none is specified 68 DefaultRuntimeBinary = "docker-runc" 69 70 errSystemNotSupported = errors.New("The Docker daemon is not supported on this platform.") 71 ) 72 73 // Daemon holds information about the Docker daemon. 74 type Daemon struct { 75 ID string 76 repository string 77 containers container.Store 78 execCommands *exec.Store 79 referenceStore refstore.Store 80 downloadManager *xfer.LayerDownloadManager 81 uploadManager *xfer.LayerUploadManager 82 distributionMetadataStore dmetadata.Store 83 trustKey libtrust.PrivateKey 84 idIndex *truncindex.TruncIndex 85 configStore *config.Config 86 statsCollector *stats.Collector 87 defaultLogConfig containertypes.LogConfig 88 RegistryService registry.Service 89 EventsService *events.Events 90 netController libnetwork.NetworkController 91 volumes *store.VolumeStore 92 discoveryWatcher discovery.Reloader 93 root string 94 seccompEnabled bool 95 apparmorEnabled bool 96 shutdown bool 97 idMappings *idtools.IDMappings 98 layerStore layer.Store 99 imageStore image.Store 100 PluginStore *plugin.Store // todo: remove 101 pluginManager *plugin.Manager 102 nameIndex *registrar.Registrar 103 linkIndex *linkIndex 104 containerd libcontainerd.Client 105 containerdRemote libcontainerd.Remote 106 defaultIsolation containertypes.Isolation // Default isolation mode on Windows 107 clusterProvider cluster.Provider 108 cluster Cluster 109 metricsPluginListener net.Listener 110 111 machineMemory uint64 112 113 seccompProfile []byte 114 seccompProfilePath string 115 116 diskUsageRunning int32 117 pruneRunning int32 118 hosts map[string]bool // hosts stores the addresses the daemon is listening on 119 } 120 121 // StoreHosts stores the addresses the daemon is listening on 122 func (daemon *Daemon) StoreHosts(hosts []string) { 123 if daemon.hosts == nil { 124 daemon.hosts = make(map[string]bool) 125 } 126 for _, h := range hosts { 127 daemon.hosts[h] = true 128 } 129 } 130 131 // HasExperimental returns whether the experimental features of the daemon are enabled or not 132 func (daemon *Daemon) HasExperimental() bool { 133 if daemon.configStore != nil && daemon.configStore.Experimental { 134 return true 135 } 136 return false 137 } 138 139 func (daemon *Daemon) restore() error { 140 var ( 141 currentDriver = daemon.GraphDriverName() 142 containers = make(map[string]*container.Container) 143 ) 144 145 logrus.Info("Loading containers: start.") 146 147 dir, err := ioutil.ReadDir(daemon.repository) 148 if err != nil { 149 return err 150 } 151 152 for _, v := range dir { 153 id := v.Name() 154 container, err := daemon.load(id) 155 if err != nil { 156 logrus.Errorf("Failed to load container %v: %v", id, err) 157 continue 158 } 159 160 // Ignore the container if it does not support the current driver being used by the graph 161 if (container.Driver == "" && currentDriver == "aufs") || container.Driver == currentDriver { 162 rwlayer, err := daemon.layerStore.GetRWLayer(container.ID) 163 if err != nil { 164 logrus.Errorf("Failed to load container mount %v: %v", id, err) 165 continue 166 } 167 container.RWLayer = rwlayer 168 logrus.Debugf("Loaded container %v", container.ID) 169 170 containers[container.ID] = container 171 } else { 172 logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) 173 } 174 } 175 176 removeContainers := make(map[string]*container.Container) 177 restartContainers := make(map[*container.Container]chan struct{}) 178 activeSandboxes := make(map[string]interface{}) 179 for id, c := range containers { 180 if err := daemon.registerName(c); err != nil { 181 logrus.Errorf("Failed to register container %s: %s", c.ID, err) 182 delete(containers, id) 183 continue 184 } 185 daemon.Register(c) 186 187 // verify that all volumes valid and have been migrated from the pre-1.7 layout 188 if err := daemon.verifyVolumesInfo(c); err != nil { 189 // don't skip the container due to error 190 logrus.Errorf("Failed to verify volumes for container '%s': %v", c.ID, err) 191 } 192 193 // The LogConfig.Type is empty if the container was created before docker 1.12 with default log driver. 194 // We should rewrite it to use the daemon defaults. 195 // Fixes https://github.com/docker/docker/issues/22536 196 if c.HostConfig.LogConfig.Type == "" { 197 if err := daemon.mergeAndVerifyLogConfig(&c.HostConfig.LogConfig); err != nil { 198 logrus.Errorf("Failed to verify log config for container %s: %q", c.ID, err) 199 continue 200 } 201 } 202 } 203 204 var wg sync.WaitGroup 205 var mapLock sync.Mutex 206 for _, c := range containers { 207 wg.Add(1) 208 go func(c *container.Container) { 209 defer wg.Done() 210 daemon.backportMountSpec(c) 211 if err := c.ToDiskLocking(); err != nil { 212 logrus.WithError(err).WithField("container", c.ID).Error("error saving backported mountspec to disk") 213 } 214 215 daemon.setStateCounter(c) 216 if c.IsRunning() || c.IsPaused() { 217 c.RestartManager().Cancel() // manually start containers because some need to wait for swarm networking 218 if err := daemon.containerd.Restore(c.ID, c.InitializeStdio); err != nil { 219 logrus.Errorf("Failed to restore %s with containerd: %s", c.ID, err) 220 return 221 } 222 223 // we call Mount and then Unmount to get BaseFs of the container 224 if err := daemon.Mount(c); err != nil { 225 // The mount is unlikely to fail. However, in case mount fails 226 // the container should be allowed to restore here. Some functionalities 227 // (like docker exec -u user) might be missing but container is able to be 228 // stopped/restarted/removed. 229 // See #29365 for related information. 230 // The error is only logged here. 231 logrus.Warnf("Failed to mount container on getting BaseFs path %v: %v", c.ID, err) 232 } else { 233 if err := daemon.Unmount(c); err != nil { 234 logrus.Warnf("Failed to umount container on getting BaseFs path %v: %v", c.ID, err) 235 } 236 } 237 238 c.ResetRestartManager(false) 239 if !c.HostConfig.NetworkMode.IsContainer() && c.IsRunning() { 240 options, err := daemon.buildSandboxOptions(c) 241 if err != nil { 242 logrus.Warnf("Failed build sandbox option to restore container %s: %v", c.ID, err) 243 } 244 mapLock.Lock() 245 activeSandboxes[c.NetworkSettings.SandboxID] = options 246 mapLock.Unlock() 247 } 248 249 } 250 // fixme: only if not running 251 // get list of containers we need to restart 252 if !c.IsRunning() && !c.IsPaused() { 253 // Do not autostart containers which 254 // has endpoints in a swarm scope 255 // network yet since the cluster is 256 // not initialized yet. We will start 257 // it after the cluster is 258 // initialized. 259 if daemon.configStore.AutoRestart && c.ShouldRestart() && !c.NetworkSettings.HasSwarmEndpoint { 260 mapLock.Lock() 261 restartContainers[c] = make(chan struct{}) 262 mapLock.Unlock() 263 } else if c.HostConfig != nil && c.HostConfig.AutoRemove { 264 mapLock.Lock() 265 removeContainers[c.ID] = c 266 mapLock.Unlock() 267 } 268 } 269 270 if c.RemovalInProgress { 271 // We probably crashed in the middle of a removal, reset 272 // the flag. 273 // 274 // We DO NOT remove the container here as we do not 275 // know if the user had requested for either the 276 // associated volumes, network links or both to also 277 // be removed. So we put the container in the "dead" 278 // state and leave further processing up to them. 279 logrus.Debugf("Resetting RemovalInProgress flag from %v", c.ID) 280 c.ResetRemovalInProgress() 281 c.SetDead() 282 c.ToDisk() 283 } 284 }(c) 285 } 286 wg.Wait() 287 daemon.netController, err = daemon.initNetworkController(daemon.configStore, activeSandboxes) 288 if err != nil { 289 return fmt.Errorf("Error initializing network controller: %v", err) 290 } 291 292 // Now that all the containers are registered, register the links 293 for _, c := range containers { 294 if err := daemon.registerLinks(c, c.HostConfig); err != nil { 295 logrus.Errorf("failed to register link for container %s: %v", c.ID, err) 296 } 297 } 298 299 group := sync.WaitGroup{} 300 for c, notifier := range restartContainers { 301 group.Add(1) 302 303 go func(c *container.Container, chNotify chan struct{}) { 304 defer group.Done() 305 306 logrus.Debugf("Starting container %s", c.ID) 307 308 // ignore errors here as this is a best effort to wait for children to be 309 // running before we try to start the container 310 children := daemon.children(c) 311 timeout := time.After(5 * time.Second) 312 for _, child := range children { 313 if notifier, exists := restartContainers[child]; exists { 314 select { 315 case <-notifier: 316 case <-timeout: 317 } 318 } 319 } 320 321 // Make sure networks are available before starting 322 daemon.waitForNetworks(c) 323 if err := daemon.containerStart(c, "", "", true); err != nil { 324 logrus.Errorf("Failed to start container %s: %s", c.ID, err) 325 } 326 close(chNotify) 327 }(c, notifier) 328 329 } 330 group.Wait() 331 332 removeGroup := sync.WaitGroup{} 333 for id := range removeContainers { 334 removeGroup.Add(1) 335 go func(cid string) { 336 if err := daemon.ContainerRm(cid, &types.ContainerRmConfig{ForceRemove: true, RemoveVolume: true}); err != nil { 337 logrus.Errorf("Failed to remove container %s: %s", cid, err) 338 } 339 removeGroup.Done() 340 }(id) 341 } 342 removeGroup.Wait() 343 344 // any containers that were started above would already have had this done, 345 // however we need to now prepare the mountpoints for the rest of the containers as well. 346 // This shouldn't cause any issue running on the containers that already had this run. 347 // This must be run after any containers with a restart policy so that containerized plugins 348 // can have a chance to be running before we try to initialize them. 349 for _, c := range containers { 350 // if the container has restart policy, do not 351 // prepare the mountpoints since it has been done on restarting. 352 // This is to speed up the daemon start when a restart container 353 // has a volume and the volume driver is not available. 354 if _, ok := restartContainers[c]; ok { 355 continue 356 } else if _, ok := removeContainers[c.ID]; ok { 357 // container is automatically removed, skip it. 358 continue 359 } 360 361 group.Add(1) 362 go func(c *container.Container) { 363 defer group.Done() 364 if err := daemon.prepareMountPoints(c); err != nil { 365 logrus.Error(err) 366 } 367 }(c) 368 } 369 370 group.Wait() 371 372 logrus.Info("Loading containers: done.") 373 374 return nil 375 } 376 377 // RestartSwarmContainers restarts any autostart container which has a 378 // swarm endpoint. 379 func (daemon *Daemon) RestartSwarmContainers() { 380 group := sync.WaitGroup{} 381 for _, c := range daemon.List() { 382 if !c.IsRunning() && !c.IsPaused() { 383 // Autostart all the containers which has a 384 // swarm endpoint now that the cluster is 385 // initialized. 386 if daemon.configStore.AutoRestart && c.ShouldRestart() && c.NetworkSettings.HasSwarmEndpoint { 387 group.Add(1) 388 go func(c *container.Container) { 389 defer group.Done() 390 if err := daemon.containerStart(c, "", "", true); err != nil { 391 logrus.Error(err) 392 } 393 }(c) 394 } 395 } 396 397 } 398 group.Wait() 399 } 400 401 // waitForNetworks is used during daemon initialization when starting up containers 402 // It ensures that all of a container's networks are available before the daemon tries to start the container. 403 // In practice it just makes sure the discovery service is available for containers which use a network that require discovery. 404 func (daemon *Daemon) waitForNetworks(c *container.Container) { 405 if daemon.discoveryWatcher == nil { 406 return 407 } 408 // Make sure if the container has a network that requires discovery that the discovery service is available before starting 409 for netName := range c.NetworkSettings.Networks { 410 // If we get `ErrNoSuchNetwork` here, we can assume that it is due to discovery not being ready 411 // Most likely this is because the K/V store used for discovery is in a container and needs to be started 412 if _, err := daemon.netController.NetworkByName(netName); err != nil { 413 if _, ok := err.(libnetwork.ErrNoSuchNetwork); !ok { 414 continue 415 } 416 // use a longish timeout here due to some slowdowns in libnetwork if the k/v store is on anything other than --net=host 417 // FIXME: why is this slow??? 418 logrus.Debugf("Container %s waiting for network to be ready", c.Name) 419 select { 420 case <-daemon.discoveryWatcher.ReadyCh(): 421 case <-time.After(60 * time.Second): 422 } 423 return 424 } 425 } 426 } 427 428 func (daemon *Daemon) children(c *container.Container) map[string]*container.Container { 429 return daemon.linkIndex.children(c) 430 } 431 432 // parents returns the names of the parent containers of the container 433 // with the given name. 434 func (daemon *Daemon) parents(c *container.Container) map[string]*container.Container { 435 return daemon.linkIndex.parents(c) 436 } 437 438 func (daemon *Daemon) registerLink(parent, child *container.Container, alias string) error { 439 fullName := path.Join(parent.Name, alias) 440 if err := daemon.nameIndex.Reserve(fullName, child.ID); err != nil { 441 if err == registrar.ErrNameReserved { 442 logrus.Warnf("error registering link for %s, to %s, as alias %s, ignoring: %v", parent.ID, child.ID, alias, err) 443 return nil 444 } 445 return err 446 } 447 daemon.linkIndex.link(parent, child, fullName) 448 return nil 449 } 450 451 // DaemonJoinsCluster informs the daemon has joined the cluster and provides 452 // the handler to query the cluster component 453 func (daemon *Daemon) DaemonJoinsCluster(clusterProvider cluster.Provider) { 454 daemon.setClusterProvider(clusterProvider) 455 } 456 457 // DaemonLeavesCluster informs the daemon has left the cluster 458 func (daemon *Daemon) DaemonLeavesCluster() { 459 // Daemon is in charge of removing the attachable networks with 460 // connected containers when the node leaves the swarm 461 daemon.clearAttachableNetworks() 462 // We no longer need the cluster provider, stop it now so that 463 // the network agent will stop listening to cluster events. 464 daemon.setClusterProvider(nil) 465 // Wait for the networking cluster agent to stop 466 daemon.netController.AgentStopWait() 467 // Daemon is in charge of removing the ingress network when the 468 // node leaves the swarm. Wait for job to be done or timeout. 469 // This is called also on graceful daemon shutdown. We need to 470 // wait, because the ingress release has to happen before the 471 // network controller is stopped. 472 if done, err := daemon.ReleaseIngress(); err == nil { 473 select { 474 case <-done: 475 case <-time.After(5 * time.Second): 476 logrus.Warnf("timeout while waiting for ingress network removal") 477 } 478 } else { 479 logrus.Warnf("failed to initiate ingress network removal: %v", err) 480 } 481 } 482 483 // setClusterProvider sets a component for querying the current cluster state. 484 func (daemon *Daemon) setClusterProvider(clusterProvider cluster.Provider) { 485 daemon.clusterProvider = clusterProvider 486 daemon.netController.SetClusterProvider(clusterProvider) 487 } 488 489 // IsSwarmCompatible verifies if the current daemon 490 // configuration is compatible with the swarm mode 491 func (daemon *Daemon) IsSwarmCompatible() error { 492 if daemon.configStore == nil { 493 return nil 494 } 495 return daemon.configStore.IsSwarmCompatible() 496 } 497 498 // NewDaemon sets up everything for the daemon to be able to service 499 // requests from the webserver. 500 func NewDaemon(config *config.Config, registryService registry.Service, containerdRemote libcontainerd.Remote, pluginStore *plugin.Store) (daemon *Daemon, err error) { 501 setDefaultMtu(config) 502 503 // Ensure that we have a correct root key limit for launching containers. 504 if err := ModifyRootKeyLimit(); err != nil { 505 logrus.Warnf("unable to modify root key limit, number of containers could be limited by this quota: %v", err) 506 } 507 508 // Ensure we have compatible and valid configuration options 509 if err := verifyDaemonSettings(config); err != nil { 510 return nil, err 511 } 512 513 // Do we have a disabled network? 514 config.DisableBridge = isBridgeNetworkDisabled(config) 515 516 // Verify the platform is supported as a daemon 517 if !platformSupported { 518 return nil, errSystemNotSupported 519 } 520 521 // Validate platform-specific requirements 522 if err := checkSystem(); err != nil { 523 return nil, err 524 } 525 526 idMappings, err := setupRemappedRoot(config) 527 if err != nil { 528 return nil, err 529 } 530 rootIDs := idMappings.RootPair() 531 if err := setupDaemonProcess(config); err != nil { 532 return nil, err 533 } 534 535 // set up the tmpDir to use a canonical path 536 tmp, err := prepareTempDir(config.Root, rootIDs) 537 if err != nil { 538 return nil, fmt.Errorf("Unable to get the TempDir under %s: %s", config.Root, err) 539 } 540 realTmp, err := fileutils.ReadSymlinkedDirectory(tmp) 541 if err != nil { 542 return nil, fmt.Errorf("Unable to get the full path to the TempDir (%s): %s", tmp, err) 543 } 544 os.Setenv("TMPDIR", realTmp) 545 546 d := &Daemon{configStore: config} 547 // Ensure the daemon is properly shutdown if there is a failure during 548 // initialization 549 defer func() { 550 if err != nil { 551 if err := d.Shutdown(); err != nil { 552 logrus.Error(err) 553 } 554 } 555 }() 556 557 // set up SIGUSR1 handler on Unix-like systems, or a Win32 global event 558 // on Windows to dump Go routine stacks 559 stackDumpDir := config.Root 560 if execRoot := config.GetExecRoot(); execRoot != "" { 561 stackDumpDir = execRoot 562 } 563 d.setupDumpStackTrap(stackDumpDir) 564 565 if err := d.setupSeccompProfile(); err != nil { 566 return nil, err 567 } 568 569 // Set the default isolation mode (only applicable on Windows) 570 if err := d.setDefaultIsolation(); err != nil { 571 return nil, fmt.Errorf("error setting default isolation mode: %v", err) 572 } 573 574 logrus.Debugf("Using default logging driver %s", config.LogConfig.Type) 575 576 if err := configureMaxThreads(config); err != nil { 577 logrus.Warnf("Failed to configure golang's threads limit: %v", err) 578 } 579 580 if err := ensureDefaultAppArmorProfile(); err != nil { 581 logrus.Errorf(err.Error()) 582 } 583 584 daemonRepo := filepath.Join(config.Root, "containers") 585 if err := idtools.MkdirAllAndChown(daemonRepo, 0700, rootIDs); err != nil && !os.IsExist(err) { 586 return nil, err 587 } 588 589 if runtime.GOOS == "windows" { 590 if err := system.MkdirAll(filepath.Join(config.Root, "credentialspecs"), 0); err != nil && !os.IsExist(err) { 591 return nil, err 592 } 593 } 594 595 driverName := os.Getenv("DOCKER_DRIVER") 596 if driverName == "" { 597 driverName = config.GraphDriver 598 } 599 600 d.RegistryService = registryService 601 d.PluginStore = pluginStore 602 logger.RegisterPluginGetter(d.PluginStore) 603 604 metricsSockPath, err := d.listenMetricsSock() 605 if err != nil { 606 return nil, err 607 } 608 registerMetricsPluginCallback(d.PluginStore, metricsSockPath) 609 610 // Plugin system initialization should happen before restore. Do not change order. 611 d.pluginManager, err = plugin.NewManager(plugin.ManagerConfig{ 612 Root: filepath.Join(config.Root, "plugins"), 613 ExecRoot: getPluginExecRoot(config.Root), 614 Store: d.PluginStore, 615 Executor: containerdRemote, 616 RegistryService: registryService, 617 LiveRestoreEnabled: config.LiveRestoreEnabled, 618 LogPluginEvent: d.LogPluginEvent, // todo: make private 619 AuthzMiddleware: config.AuthzMiddleware, 620 }) 621 if err != nil { 622 return nil, errors.Wrap(err, "couldn't create plugin manager") 623 } 624 625 d.layerStore, err = layer.NewStoreFromOptions(layer.StoreOptions{ 626 StorePath: config.Root, 627 MetadataStorePathTemplate: filepath.Join(config.Root, "image", "%s", "layerdb"), 628 GraphDriver: driverName, 629 GraphDriverOptions: config.GraphOptions, 630 IDMappings: idMappings, 631 PluginGetter: d.PluginStore, 632 ExperimentalEnabled: config.Experimental, 633 }) 634 if err != nil { 635 return nil, err 636 } 637 638 graphDriver := d.layerStore.DriverName() 639 imageRoot := filepath.Join(config.Root, "image", graphDriver) 640 641 // Configure and validate the kernels security support 642 if err := configureKernelSecuritySupport(config, graphDriver); err != nil { 643 return nil, err 644 } 645 646 logrus.Debugf("Max Concurrent Downloads: %d", *config.MaxConcurrentDownloads) 647 d.downloadManager = xfer.NewLayerDownloadManager(d.layerStore, *config.MaxConcurrentDownloads) 648 logrus.Debugf("Max Concurrent Uploads: %d", *config.MaxConcurrentUploads) 649 d.uploadManager = xfer.NewLayerUploadManager(*config.MaxConcurrentUploads) 650 651 ifs, err := image.NewFSStoreBackend(filepath.Join(imageRoot, "imagedb")) 652 if err != nil { 653 return nil, err 654 } 655 656 d.imageStore, err = image.NewImageStore(ifs, d.layerStore) 657 if err != nil { 658 return nil, err 659 } 660 661 // Configure the volumes driver 662 volStore, err := d.configureVolumes(rootIDs) 663 if err != nil { 664 return nil, err 665 } 666 667 trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) 668 if err != nil { 669 return nil, err 670 } 671 672 trustDir := filepath.Join(config.Root, "trust") 673 674 if err := system.MkdirAll(trustDir, 0700); err != nil { 675 return nil, err 676 } 677 678 distributionMetadataStore, err := dmetadata.NewFSMetadataStore(filepath.Join(imageRoot, "distribution")) 679 if err != nil { 680 return nil, err 681 } 682 683 eventsService := events.New() 684 685 referenceStore, err := refstore.NewReferenceStore(filepath.Join(imageRoot, "repositories.json")) 686 if err != nil { 687 return nil, fmt.Errorf("Couldn't create Tag store repositories: %s", err) 688 } 689 690 migrationStart := time.Now() 691 if err := v1.Migrate(config.Root, graphDriver, d.layerStore, d.imageStore, referenceStore, distributionMetadataStore); err != nil { 692 logrus.Errorf("Graph migration failed: %q. Your old graph data was found to be too inconsistent for upgrading to content-addressable storage. Some of the old data was probably not upgraded. We recommend starting over with a clean storage directory if possible.", err) 693 } 694 logrus.Infof("Graph migration to content-addressability took %.2f seconds", time.Since(migrationStart).Seconds()) 695 696 // Discovery is only enabled when the daemon is launched with an address to advertise. When 697 // initialized, the daemon is registered and we can store the discovery backend as it's read-only 698 if err := d.initDiscovery(config); err != nil { 699 return nil, err 700 } 701 702 sysInfo := sysinfo.New(false) 703 // Check if Devices cgroup is mounted, it is hard requirement for container security, 704 // on Linux. 705 if runtime.GOOS == "linux" && !sysInfo.CgroupDevicesEnabled { 706 return nil, errors.New("Devices cgroup isn't mounted") 707 } 708 709 d.ID = trustKey.PublicKey().KeyID() 710 d.repository = daemonRepo 711 d.containers = container.NewMemoryStore() 712 d.execCommands = exec.NewStore() 713 d.referenceStore = referenceStore 714 d.distributionMetadataStore = distributionMetadataStore 715 d.trustKey = trustKey 716 d.idIndex = truncindex.NewTruncIndex([]string{}) 717 d.statsCollector = d.newStatsCollector(1 * time.Second) 718 d.defaultLogConfig = containertypes.LogConfig{ 719 Type: config.LogConfig.Type, 720 Config: config.LogConfig.Config, 721 } 722 d.EventsService = eventsService 723 d.volumes = volStore 724 d.root = config.Root 725 d.idMappings = idMappings 726 d.seccompEnabled = sysInfo.Seccomp 727 d.apparmorEnabled = sysInfo.AppArmor 728 729 d.nameIndex = registrar.NewRegistrar() 730 d.linkIndex = newLinkIndex() 731 d.containerdRemote = containerdRemote 732 733 go d.execCommandGC() 734 735 d.containerd, err = containerdRemote.Client(d) 736 if err != nil { 737 return nil, err 738 } 739 740 if err := d.restore(); err != nil { 741 return nil, err 742 } 743 744 // FIXME: this method never returns an error 745 info, _ := d.SystemInfo() 746 747 engineInfo.WithValues( 748 dockerversion.Version, 749 dockerversion.GitCommit, 750 info.Architecture, 751 info.Driver, 752 info.KernelVersion, 753 info.OperatingSystem, 754 info.OSType, 755 info.ID, 756 ).Set(1) 757 engineCpus.Set(float64(info.NCPU)) 758 engineMemory.Set(float64(info.MemTotal)) 759 760 return d, nil 761 } 762 763 func (daemon *Daemon) shutdownContainer(c *container.Container) error { 764 stopTimeout := c.StopTimeout() 765 // TODO(windows): Handle docker restart with paused containers 766 if c.IsPaused() { 767 // To terminate a process in freezer cgroup, we should send 768 // SIGTERM to this process then unfreeze it, and the process will 769 // force to terminate immediately. 770 logrus.Debugf("Found container %s is paused, sending SIGTERM before unpausing it", c.ID) 771 sig, ok := signal.SignalMap["TERM"] 772 if !ok { 773 return errors.New("System does not support SIGTERM") 774 } 775 if err := daemon.kill(c, int(sig)); err != nil { 776 return fmt.Errorf("sending SIGTERM to container %s with error: %v", c.ID, err) 777 } 778 if err := daemon.containerUnpause(c); err != nil { 779 return fmt.Errorf("Failed to unpause container %s with error: %v", c.ID, err) 780 } 781 782 ctx, cancel := context.WithTimeout(context.Background(), time.Duration(stopTimeout)*time.Second) 783 defer cancel() 784 785 // Wait with timeout for container to exit. 786 if status := <-c.Wait(ctx, container.WaitConditionNotRunning); status.Err() != nil { 787 logrus.Debugf("container %s failed to exit in %d second of SIGTERM, sending SIGKILL to force", c.ID, stopTimeout) 788 sig, ok := signal.SignalMap["KILL"] 789 if !ok { 790 return errors.New("System does not support SIGKILL") 791 } 792 if err := daemon.kill(c, int(sig)); err != nil { 793 logrus.Errorf("Failed to SIGKILL container %s", c.ID) 794 } 795 // Wait for exit again without a timeout. 796 // Explicitly ignore the result. 797 _ = <-c.Wait(context.Background(), container.WaitConditionNotRunning) 798 return status.Err() 799 } 800 } 801 // If container failed to exit in stopTimeout seconds of SIGTERM, then using the force 802 if err := daemon.containerStop(c, stopTimeout); err != nil { 803 return fmt.Errorf("Failed to stop container %s with error: %v", c.ID, err) 804 } 805 806 // Wait without timeout for the container to exit. 807 // Ignore the result. 808 _ = <-c.Wait(context.Background(), container.WaitConditionNotRunning) 809 return nil 810 } 811 812 // ShutdownTimeout returns the shutdown timeout based on the max stopTimeout of the containers, 813 // and is limited by daemon's ShutdownTimeout. 814 func (daemon *Daemon) ShutdownTimeout() int { 815 // By default we use daemon's ShutdownTimeout. 816 shutdownTimeout := daemon.configStore.ShutdownTimeout 817 818 graceTimeout := 5 819 if daemon.containers != nil { 820 for _, c := range daemon.containers.List() { 821 if shutdownTimeout >= 0 { 822 stopTimeout := c.StopTimeout() 823 if stopTimeout < 0 { 824 shutdownTimeout = -1 825 } else { 826 if stopTimeout+graceTimeout > shutdownTimeout { 827 shutdownTimeout = stopTimeout + graceTimeout 828 } 829 } 830 } 831 } 832 } 833 return shutdownTimeout 834 } 835 836 // Shutdown stops the daemon. 837 func (daemon *Daemon) Shutdown() error { 838 daemon.shutdown = true 839 // Keep mounts and networking running on daemon shutdown if 840 // we are to keep containers running and restore them. 841 842 if daemon.configStore.LiveRestoreEnabled && daemon.containers != nil { 843 // check if there are any running containers, if none we should do some cleanup 844 if ls, err := daemon.Containers(&types.ContainerListOptions{}); len(ls) != 0 || err != nil { 845 // metrics plugins still need some cleanup 846 daemon.cleanupMetricsPlugins() 847 return nil 848 } 849 } 850 851 if daemon.containers != nil { 852 logrus.Debugf("start clean shutdown of all containers with a %d seconds timeout...", daemon.configStore.ShutdownTimeout) 853 daemon.containers.ApplyAll(func(c *container.Container) { 854 if !c.IsRunning() { 855 return 856 } 857 logrus.Debugf("stopping %s", c.ID) 858 if err := daemon.shutdownContainer(c); err != nil { 859 logrus.Errorf("Stop container error: %v", err) 860 return 861 } 862 if mountid, err := daemon.layerStore.GetMountID(c.ID); err == nil { 863 daemon.cleanupMountsByID(mountid) 864 } 865 logrus.Debugf("container stopped %s", c.ID) 866 }) 867 } 868 869 if daemon.volumes != nil { 870 if err := daemon.volumes.Shutdown(); err != nil { 871 logrus.Errorf("Error shutting down volume store: %v", err) 872 } 873 } 874 875 if daemon.layerStore != nil { 876 if err := daemon.layerStore.Cleanup(); err != nil { 877 logrus.Errorf("Error during layer Store.Cleanup(): %v", err) 878 } 879 } 880 881 // If we are part of a cluster, clean up cluster's stuff 882 if daemon.clusterProvider != nil { 883 logrus.Debugf("start clean shutdown of cluster resources...") 884 daemon.DaemonLeavesCluster() 885 } 886 887 daemon.cleanupMetricsPlugins() 888 889 // Shutdown plugins after containers and layerstore. Don't change the order. 890 daemon.pluginShutdown() 891 892 // trigger libnetwork Stop only if it's initialized 893 if daemon.netController != nil { 894 daemon.netController.Stop() 895 } 896 897 if err := daemon.cleanupMounts(); err != nil { 898 return err 899 } 900 901 return nil 902 } 903 904 // Mount sets container.BaseFS 905 // (is it not set coming in? why is it unset?) 906 func (daemon *Daemon) Mount(container *container.Container) error { 907 dir, err := container.RWLayer.Mount(container.GetMountLabel()) 908 if err != nil { 909 return err 910 } 911 logrus.Debugf("container mounted via layerStore: %v", dir) 912 913 if container.BaseFS != dir { 914 // The mount path reported by the graph driver should always be trusted on Windows, since the 915 // volume path for a given mounted layer may change over time. This should only be an error 916 // on non-Windows operating systems. 917 if container.BaseFS != "" && runtime.GOOS != "windows" { 918 daemon.Unmount(container) 919 return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')", 920 daemon.GraphDriverName(), container.ID, container.BaseFS, dir) 921 } 922 } 923 container.BaseFS = dir // TODO: combine these fields 924 return nil 925 } 926 927 // Unmount unsets the container base filesystem 928 func (daemon *Daemon) Unmount(container *container.Container) error { 929 if err := container.RWLayer.Unmount(); err != nil { 930 logrus.Errorf("Error unmounting container %s: %s", container.ID, err) 931 return err 932 } 933 934 return nil 935 } 936 937 // Subnets return the IPv4 and IPv6 subnets of networks that are manager by Docker. 938 func (daemon *Daemon) Subnets() ([]net.IPNet, []net.IPNet) { 939 var v4Subnets []net.IPNet 940 var v6Subnets []net.IPNet 941 942 managedNetworks := daemon.netController.Networks() 943 944 for _, managedNetwork := range managedNetworks { 945 v4infos, v6infos := managedNetwork.Info().IpamInfo() 946 for _, info := range v4infos { 947 if info.IPAMData.Pool != nil { 948 v4Subnets = append(v4Subnets, *info.IPAMData.Pool) 949 } 950 } 951 for _, info := range v6infos { 952 if info.IPAMData.Pool != nil { 953 v6Subnets = append(v6Subnets, *info.IPAMData.Pool) 954 } 955 } 956 } 957 958 return v4Subnets, v6Subnets 959 } 960 961 // GraphDriverName returns the name of the graph driver used by the layer.Store 962 func (daemon *Daemon) GraphDriverName() string { 963 return daemon.layerStore.DriverName() 964 } 965 966 // prepareTempDir prepares and returns the default directory to use 967 // for temporary files. 968 // If it doesn't exist, it is created. If it exists, its content is removed. 969 func prepareTempDir(rootDir string, rootIDs idtools.IDPair) (string, error) { 970 var tmpDir string 971 if tmpDir = os.Getenv("DOCKER_TMPDIR"); tmpDir == "" { 972 tmpDir = filepath.Join(rootDir, "tmp") 973 newName := tmpDir + "-old" 974 if err := os.Rename(tmpDir, newName); err == nil { 975 go func() { 976 if err := os.RemoveAll(newName); err != nil { 977 logrus.Warnf("failed to delete old tmp directory: %s", newName) 978 } 979 }() 980 } else { 981 logrus.Warnf("failed to rename %s for background deletion: %s. Deleting synchronously", tmpDir, err) 982 if err := os.RemoveAll(tmpDir); err != nil { 983 logrus.Warnf("failed to delete old tmp directory: %s", tmpDir) 984 } 985 } 986 } 987 // We don't remove the content of tmpdir if it's not the default, 988 // it may hold things that do not belong to us. 989 return tmpDir, idtools.MkdirAllAndChown(tmpDir, 0700, rootIDs) 990 } 991 992 func (daemon *Daemon) setupInitLayer(initPath string) error { 993 rootIDs := daemon.idMappings.RootPair() 994 return initlayer.Setup(initPath, rootIDs) 995 } 996 997 func setDefaultMtu(conf *config.Config) { 998 // do nothing if the config does not have the default 0 value. 999 if conf.Mtu != 0 { 1000 return 1001 } 1002 conf.Mtu = config.DefaultNetworkMtu 1003 } 1004 1005 func (daemon *Daemon) configureVolumes(rootIDs idtools.IDPair) (*store.VolumeStore, error) { 1006 volumesDriver, err := local.New(daemon.configStore.Root, rootIDs) 1007 if err != nil { 1008 return nil, err 1009 } 1010 1011 volumedrivers.RegisterPluginGetter(daemon.PluginStore) 1012 1013 if !volumedrivers.Register(volumesDriver, volumesDriver.Name()) { 1014 return nil, errors.New("local volume driver could not be registered") 1015 } 1016 return store.New(daemon.configStore.Root) 1017 } 1018 1019 // IsShuttingDown tells whether the daemon is shutting down or not 1020 func (daemon *Daemon) IsShuttingDown() bool { 1021 return daemon.shutdown 1022 } 1023 1024 // initDiscovery initializes the discovery watcher for this daemon. 1025 func (daemon *Daemon) initDiscovery(conf *config.Config) error { 1026 advertise, err := config.ParseClusterAdvertiseSettings(conf.ClusterStore, conf.ClusterAdvertise) 1027 if err != nil { 1028 if err == discovery.ErrDiscoveryDisabled { 1029 return nil 1030 } 1031 return err 1032 } 1033 1034 conf.ClusterAdvertise = advertise 1035 discoveryWatcher, err := discovery.Init(conf.ClusterStore, conf.ClusterAdvertise, conf.ClusterOpts) 1036 if err != nil { 1037 return fmt.Errorf("discovery initialization failed (%v)", err) 1038 } 1039 1040 daemon.discoveryWatcher = discoveryWatcher 1041 return nil 1042 } 1043 1044 func isBridgeNetworkDisabled(conf *config.Config) bool { 1045 return conf.BridgeConfig.Iface == config.DisableNetworkBridge 1046 } 1047 1048 func (daemon *Daemon) networkOptions(dconfig *config.Config, pg plugingetter.PluginGetter, activeSandboxes map[string]interface{}) ([]nwconfig.Option, error) { 1049 options := []nwconfig.Option{} 1050 if dconfig == nil { 1051 return options, nil 1052 } 1053 1054 options = append(options, nwconfig.OptionExperimental(dconfig.Experimental)) 1055 options = append(options, nwconfig.OptionDataDir(dconfig.Root)) 1056 options = append(options, nwconfig.OptionExecRoot(dconfig.GetExecRoot())) 1057 1058 dd := runconfig.DefaultDaemonNetworkMode() 1059 dn := runconfig.DefaultDaemonNetworkMode().NetworkName() 1060 options = append(options, nwconfig.OptionDefaultDriver(string(dd))) 1061 options = append(options, nwconfig.OptionDefaultNetwork(dn)) 1062 1063 if strings.TrimSpace(dconfig.ClusterStore) != "" { 1064 kv := strings.Split(dconfig.ClusterStore, "://") 1065 if len(kv) != 2 { 1066 return nil, errors.New("kv store daemon config must be of the form KV-PROVIDER://KV-URL") 1067 } 1068 options = append(options, nwconfig.OptionKVProvider(kv[0])) 1069 options = append(options, nwconfig.OptionKVProviderURL(kv[1])) 1070 } 1071 if len(dconfig.ClusterOpts) > 0 { 1072 options = append(options, nwconfig.OptionKVOpts(dconfig.ClusterOpts)) 1073 } 1074 1075 if daemon.discoveryWatcher != nil { 1076 options = append(options, nwconfig.OptionDiscoveryWatcher(daemon.discoveryWatcher)) 1077 } 1078 1079 if dconfig.ClusterAdvertise != "" { 1080 options = append(options, nwconfig.OptionDiscoveryAddress(dconfig.ClusterAdvertise)) 1081 } 1082 1083 options = append(options, nwconfig.OptionLabels(dconfig.Labels)) 1084 options = append(options, driverOptions(dconfig)...) 1085 1086 if daemon.configStore != nil && daemon.configStore.LiveRestoreEnabled && len(activeSandboxes) != 0 { 1087 options = append(options, nwconfig.OptionActiveSandboxes(activeSandboxes)) 1088 } 1089 1090 if pg != nil { 1091 options = append(options, nwconfig.OptionPluginGetter(pg)) 1092 } 1093 1094 return options, nil 1095 } 1096 1097 func copyBlkioEntry(entries []*containerd.BlkioStatsEntry) []types.BlkioStatEntry { 1098 out := make([]types.BlkioStatEntry, len(entries)) 1099 for i, re := range entries { 1100 out[i] = types.BlkioStatEntry{ 1101 Major: re.Major, 1102 Minor: re.Minor, 1103 Op: re.Op, 1104 Value: re.Value, 1105 } 1106 } 1107 return out 1108 } 1109 1110 // GetCluster returns the cluster 1111 func (daemon *Daemon) GetCluster() Cluster { 1112 return daemon.cluster 1113 } 1114 1115 // SetCluster sets the cluster 1116 func (daemon *Daemon) SetCluster(cluster Cluster) { 1117 daemon.cluster = cluster 1118 } 1119 1120 func (daemon *Daemon) pluginShutdown() { 1121 manager := daemon.pluginManager 1122 // Check for a valid manager object. In error conditions, daemon init can fail 1123 // and shutdown called, before plugin manager is initialized. 1124 if manager != nil { 1125 manager.Shutdown() 1126 } 1127 } 1128 1129 // PluginManager returns current pluginManager associated with the daemon 1130 func (daemon *Daemon) PluginManager() *plugin.Manager { // set up before daemon to avoid this method 1131 return daemon.pluginManager 1132 } 1133 1134 // PluginGetter returns current pluginStore associated with the daemon 1135 func (daemon *Daemon) PluginGetter() *plugin.Store { 1136 return daemon.PluginStore 1137 } 1138 1139 // CreateDaemonRoot creates the root for the daemon 1140 func CreateDaemonRoot(config *config.Config) error { 1141 // get the canonical path to the Docker root directory 1142 var realRoot string 1143 if _, err := os.Stat(config.Root); err != nil && os.IsNotExist(err) { 1144 realRoot = config.Root 1145 } else { 1146 realRoot, err = fileutils.ReadSymlinkedDirectory(config.Root) 1147 if err != nil { 1148 return fmt.Errorf("Unable to get the full path to root (%s): %s", config.Root, err) 1149 } 1150 } 1151 1152 idMappings, err := setupRemappedRoot(config) 1153 if err != nil { 1154 return err 1155 } 1156 return setupDaemonRoot(config, realRoot, idMappings.RootPair()) 1157 }