github.com/cloudfoundry-attic/garden-linux@v0.333.2-candidate/main.go (about) 1 package main 2 3 import ( 4 "flag" 5 "fmt" 6 "net" 7 "os" 8 "os/signal" 9 "path" 10 "path/filepath" 11 "runtime" 12 "strings" 13 "syscall" 14 "time" 15 16 "github.com/Sirupsen/logrus" 17 "github.com/blang/semver" 18 "github.com/cloudfoundry/gunk/command_runner" 19 "github.com/docker/docker/daemon/graphdriver" 20 "github.com/docker/docker/graph" 21 _ "github.com/docker/docker/pkg/chrootarchive" // allow reexec of docker-applyLayer 22 "github.com/eapache/go-resiliency/retrier" 23 24 "github.com/cloudfoundry-incubator/cf-debug-server" 25 "github.com/cloudfoundry-incubator/cf-lager" 26 "github.com/cloudfoundry-incubator/garden-linux/container_repository" 27 "github.com/cloudfoundry-incubator/garden-linux/linux_backend" 28 "github.com/cloudfoundry-incubator/garden-linux/linux_container" 29 "github.com/cloudfoundry-incubator/garden-linux/linux_container/bandwidth_manager" 30 "github.com/cloudfoundry-incubator/garden-linux/linux_container/cgroups_manager" 31 "github.com/cloudfoundry-incubator/garden-linux/linux_container/iptables_manager" 32 "github.com/cloudfoundry-incubator/garden-linux/metrics" 33 "github.com/cloudfoundry-incubator/garden-linux/network" 34 "github.com/cloudfoundry-incubator/garden-linux/network/bridgemgr" 35 "github.com/cloudfoundry-incubator/garden-linux/network/devices" 36 "github.com/cloudfoundry-incubator/garden-linux/network/iptables" 37 "github.com/cloudfoundry-incubator/garden-linux/network/subnets" 38 "github.com/cloudfoundry-incubator/garden-linux/pkg/vars" 39 "github.com/cloudfoundry-incubator/garden-linux/port_pool" 40 "github.com/cloudfoundry-incubator/garden-linux/process_tracker" 41 "github.com/cloudfoundry-incubator/garden-linux/resource_pool" 42 "github.com/cloudfoundry-incubator/garden-linux/sysconfig" 43 "github.com/cloudfoundry-incubator/garden-linux/sysinfo" 44 "github.com/cloudfoundry-incubator/garden-linux/system" 45 "github.com/cloudfoundry-incubator/garden-shed/distclient" 46 quotaed_aufs "github.com/cloudfoundry-incubator/garden-shed/docker_drivers/aufs" 47 "github.com/cloudfoundry-incubator/garden-shed/layercake" 48 "github.com/cloudfoundry-incubator/garden-shed/quota_manager" 49 "github.com/cloudfoundry-incubator/garden-shed/repository_fetcher" 50 "github.com/cloudfoundry-incubator/garden-shed/rootfs_provider" 51 "github.com/cloudfoundry-incubator/garden/server" 52 "github.com/cloudfoundry/dropsonde" 53 "github.com/cloudfoundry/gunk/command_runner/linux_command_runner" 54 _ "github.com/docker/docker/daemon/graphdriver/aufs" 55 _ "github.com/docker/docker/daemon/graphdriver/overlay" 56 _ "github.com/docker/docker/daemon/graphdriver/vfs" 57 _ "github.com/docker/docker/pkg/chrootarchive" // allow reexec of docker-applyLayer 58 "github.com/docker/docker/pkg/reexec" 59 "github.com/pivotal-golang/clock" 60 "github.com/pivotal-golang/lager" 61 "github.com/pivotal-golang/localip" 62 ) 63 64 const ( 65 DefaultNetworkPool = "10.254.0.0/22" 66 DefaultMTUSize = 1500 67 CurrentContainerVersion = "1.0.0" 68 ) 69 70 var listenNetwork = flag.String( 71 "listenNetwork", 72 "unix", 73 "how to listen on the address (unix, tcp, etc.)", 74 ) 75 76 var listenAddr = flag.String( 77 "listenAddr", 78 "/tmp/garden.sock", 79 "address to listen on", 80 ) 81 82 var snapshotsPath = flag.String( 83 "snapshots", 84 "", 85 "directory in which to store container state to persist through restarts", 86 ) 87 88 var binPath = flag.String( 89 "bin", 90 "", 91 "directory containing backend-specific scripts (i.e. ./create.sh)", 92 ) 93 94 var stateDirPath = flag.String( 95 "stateDir", 96 "", 97 "directory containing state files", 98 ) 99 100 var depotPath = flag.String( 101 "depot", 102 "", 103 "directory in which to store containers", 104 ) 105 106 var rootFSPath = flag.String( 107 "rootfs", 108 "", 109 "directory of the rootfs for the containers", 110 ) 111 112 var enableGraphCleanup = flag.Bool( 113 "enableGraphCleanup", 114 false, 115 "enables graph garbage collection", 116 ) 117 118 var containerGraceTime = flag.Duration( 119 "containerGraceTime", 120 0, 121 "time after which to destroy idle containers", 122 ) 123 124 var portPoolStart = flag.Uint( 125 "portPoolStart", 126 60000, 127 "start of ephemeral port range used for mapped container ports", 128 ) 129 130 var portPoolSize = flag.Uint( 131 "portPoolSize", 132 5000, 133 "size of port pool used for mapped container ports", 134 ) 135 136 var networkPool = flag.String("networkPool", 137 DefaultNetworkPool, 138 "Pool of dynamically allocated container subnets") 139 140 var denyNetworks = flag.String( 141 "denyNetworks", 142 "", 143 "CIDR blocks representing IPs to blacklist", 144 ) 145 146 var allowNetworks = flag.String( 147 "allowNetworks", 148 "", 149 "CIDR blocks representing IPs to whitelist", 150 ) 151 152 var graphRoot = flag.String( 153 "graph", 154 "/var/lib/garden-docker-graph", 155 "docker image graph", 156 ) 157 158 var dockerRegistry = flag.String( 159 "registry", 160 "registry-1.docker.io", 161 "docker registry API endpoint", 162 ) 163 164 var tag = flag.String( 165 "tag", 166 "", 167 "server-wide identifier used for 'global' configuration, must be less than 3 character long", 168 ) 169 170 var dropsondeOrigin = flag.String( 171 "dropsondeOrigin", 172 "garden-linux", 173 "Origin identifier for dropsonde-emitted metrics.", 174 ) 175 176 var dropsondeDestination = flag.String( 177 "dropsondeDestination", 178 "localhost:3457", 179 "Destination for dropsonde-emitted metrics.", 180 ) 181 182 var metricsEmissionInterval = flag.Duration( 183 "metricsEmissionInterval", 184 time.Minute, 185 "Interval in which to emit metrics to the metron agent", 186 ) 187 188 var allowHostAccess = flag.Bool( 189 "allowHostAccess", 190 false, 191 "allow network access to host", 192 ) 193 194 var iptablesLogMethod = flag.String( 195 "iptablesLogMethod", 196 "kernel", 197 "type of iptable logging to use, one of 'kernel' or 'nflog' (default: kernel)", 198 ) 199 200 var mtu = flag.Int( 201 "mtu", 202 DefaultMTUSize, 203 "MTU size for container network interfaces", 204 ) 205 206 var externalIP = flag.String( 207 "externalIP", 208 "", 209 "IP address to use to reach container's mapped ports", 210 ) 211 212 var maxContainers = flag.Uint( 213 "maxContainers", 214 0, 215 "Maximum number of containers that can be created", 216 ) 217 218 var graphDriverName = flag.String( 219 "graphDriver", 220 "auto", 221 "Docker graph driver to use. Only aufs is officially supported, but others may work.", 222 ) 223 224 func main() { 225 if reexec.Init() { 226 return 227 } 228 229 var insecureRegistries vars.StringList 230 flag.Var( 231 &insecureRegistries, 232 "insecureDockerRegistry", 233 "Docker registry to allow connecting to even if not secure. (Can be specified multiple times to allow insecure connection to multiple repositories)", 234 ) 235 236 var persistentImages vars.StringList 237 flag.Var( 238 &persistentImages, 239 "persistentImage", 240 "Image which should never be garbage collected. (Can be specified multiple times)", 241 ) 242 243 cf_debug_server.AddFlags(flag.CommandLine) 244 cf_lager.AddFlags(flag.CommandLine) 245 flag.Parse() 246 247 runtime.GOMAXPROCS(runtime.NumCPU()) 248 249 logger, reconfigurableSink := cf_lager.New("garden-linux") 250 initializeDropsonde(logger) 251 252 if *binPath == "" { 253 missing("-bin") 254 } 255 256 if *stateDirPath == "" { 257 missing("-stateDir") 258 } 259 260 if *depotPath == "" { 261 missing("-depot") 262 } 263 264 if len(*tag) > 2 { 265 println("-tag parameter must be less than 3 characters long") 266 println() 267 flag.Usage() 268 return 269 } 270 271 _, dynamicRange, err := net.ParseCIDR(*networkPool) 272 if err != nil { 273 logger.Fatal("failed-to-parse-network-pool", err) 274 } 275 276 subnetPool, err := subnets.NewSubnets(dynamicRange) 277 if err != nil { 278 logger.Fatal("failed-to-create-subnet-pool", err) 279 } 280 281 portPoolState, err := port_pool.LoadState(path.Join(*stateDirPath, "port_pool.json")) 282 if err != nil { 283 logger.Error("failed-to-parse-pool-state", err) 284 } 285 286 // TODO: use /proc/sys/net/ipv4/ip_local_port_range by default (end + 1) 287 portPool, err := port_pool.New(uint32(*portPoolStart), uint32(*portPoolSize), portPoolState) 288 if err != nil { 289 logger.Fatal("invalid pool range", err) 290 } 291 292 useKernelLogging := true 293 switch *iptablesLogMethod { 294 case "nflog": 295 useKernelLogging = false 296 case "kernel": 297 /* noop */ 298 default: 299 println("-iptablesLogMethod value not recognized") 300 println() 301 flag.Usage() 302 return 303 } 304 305 config := sysconfig.NewConfig(*tag, *allowHostAccess) 306 307 runner := sysconfig.NewRunner(config, linux_command_runner.New()) 308 309 if err := os.MkdirAll(*graphRoot, 0755); err != nil { 310 logger.Fatal("failed-to-create-graph-directory", err) 311 } 312 313 dockerGraphDriver, err := selectGraphDriver(logger, *graphDriverName, *graphRoot) 314 if err != nil { 315 logger.Fatal("failed-to-construct-graph-driver", err) 316 } 317 318 backingStoresPath := filepath.Join(*graphRoot, "backing_stores") 319 if err := os.MkdirAll(backingStoresPath, 0660); err != nil { 320 logger.Fatal("failed-to-mkdir-backing-stores", err) 321 } 322 323 quotaedGraphDriver := "aed_aufs.QuotaedDriver{ 324 GraphDriver: dockerGraphDriver, 325 Unmount: quotaed_aufs.Unmount, 326 BackingStoreMgr: "aed_aufs.BackingStore{ 327 RootPath: backingStoresPath, 328 Logger: logger.Session("backing-store-mgr"), 329 }, 330 LoopMounter: "aed_aufs.Loop{ 331 Retrier: retrier.New(retrier.ConstantBackoff(200, 500*time.Millisecond), nil), 332 Logger: logger.Session("loop-mounter"), 333 }, 334 Retrier: retrier.New(retrier.ConstantBackoff(200, 500*time.Millisecond), nil), 335 RootPath: *graphRoot, 336 Logger: logger.Session("quotaed-driver"), 337 } 338 339 metricsProvider := metrics.NewMetrics(logger, backingStoresPath, *depotPath) 340 341 if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" { 342 metrics.StartDebugServer(dbgAddr, reconfigurableSink, metricsProvider) 343 } 344 345 dockerGraph, err := graph.NewGraph(*graphRoot, quotaedGraphDriver) 346 if err != nil { 347 logger.Fatal("failed-to-construct-graph", err) 348 } 349 350 var cake layercake.Cake = &layercake.Docker{ 351 Graph: dockerGraph, 352 Driver: quotaedGraphDriver, 353 } 354 355 if cake.DriverName() == "aufs" { 356 cake = &layercake.AufsCake{ 357 Cake: cake, 358 Runner: runner, 359 GraphRoot: *graphRoot, 360 } 361 } 362 363 repo := container_repository.New() 364 retainer := layercake.NewRetainer() 365 366 repoFetcher := &repository_fetcher.CompositeFetcher{ 367 LocalFetcher: &repository_fetcher.Local{ 368 Cake: cake, 369 DefaultRootFSPath: *rootFSPath, 370 IDProvider: repository_fetcher.LayerIDProvider{}, 371 }, 372 RemoteFetcher: repository_fetcher.NewRemote( 373 logger, 374 *dockerRegistry, 375 cake, 376 distclient.NewDialer(insecureRegistries.List), 377 repository_fetcher.VerifyFunc(repository_fetcher.Verify), 378 ), 379 } 380 381 maxId := uint32(sysinfo.Min(sysinfo.MustGetMaxValidUID(), sysinfo.MustGetMaxValidGID())) 382 mappingList := rootfs_provider.MappingList{ 383 { 384 ContainerID: 0, 385 HostID: maxId, 386 Size: 1, 387 }, 388 { 389 ContainerID: 1, 390 HostID: 1, 391 Size: maxId - 1, 392 }, 393 } 394 395 rootFSNamespacer := &rootfs_provider.UidNamespacer{ 396 Logger: logger, 397 Translator: rootfs_provider.NewUidTranslator( 398 mappingList, // uid 399 mappingList, // gid 400 ), 401 } 402 403 cleaner := layercake.NewOvenCleaner( 404 retainer, 405 *enableGraphCleanup, 406 ) 407 408 layerCreator := rootfs_provider.NewLayerCreator(cake, rootfs_provider.SimpleVolumeCreator{}, rootFSNamespacer) 409 cakeOrdinator := rootfs_provider.NewCakeOrdinator(cake, repoFetcher, layerCreator, cleaner) 410 411 imageRetainer := &repository_fetcher.ImageRetainer{ 412 GraphRetainer: retainer, 413 DirectoryRootfsIDProvider: repository_fetcher.LayerIDProvider{}, 414 DockerImageIDFetcher: repoFetcher, 415 416 NamespaceCacheKey: rootFSNamespacer.CacheKey(), 417 Logger: logger, 418 } 419 420 // spawn off in a go function to avoid blocking startup 421 // worst case is if an image is immediately created and deleted faster than 422 // we can retain it we'll garbage collect it when we shouldn't. This 423 // is an OK trade-off for not having garden startup block on dockerhub. 424 go imageRetainer.Retain(persistentImages.List) 425 426 rootfsCleaner := &linux_backend.RootFSCleaner{ 427 FilePaths: []string{ 428 "/tmp", "/proc", "/sys", "/dev", "/etc", "/etc/config", "/etc/hostname", 429 "/etc/hosts", "/etc/resolv.conf", 430 }, 431 } 432 433 if *externalIP == "" { 434 ip, err := localip.LocalIP() 435 if err != nil { 436 panic("couldn't determine local IP to use for -externalIP parameter. You can use the -externalIP flag to pass an external IP") 437 } 438 439 externalIP = &ip 440 } 441 442 parsedExternalIP := net.ParseIP(*externalIP) 443 if parsedExternalIP == nil { 444 panic(fmt.Sprintf("Value of -externalIP %s could not be converted to an IP", *externalIP)) 445 } 446 447 var quotaManager linux_container.QuotaManager = "a_manager.AUFSQuotaManager{ 448 BaseSizer: quota_manager.NewAUFSBaseSizer(cake), 449 DiffSizer: "a_manager.AUFSDiffSizer{quotaedGraphDriver}, 450 } 451 452 ipTablesMgr := createIPTablesManager(config, runner, logger) 453 injector := &provider{ 454 useKernelLogging: useKernelLogging, 455 chainPrefix: config.IPTables.Filter.InstancePrefix, 456 runner: runner, 457 log: logger, 458 portPool: portPool, 459 ipTablesMgr: ipTablesMgr, 460 sysconfig: config, 461 quotaManager: quotaManager, 462 } 463 464 currentContainerVersion, err := semver.Make(CurrentContainerVersion) 465 if err != nil { 466 logger.Fatal("failed-to-parse-container-version", err) 467 } 468 469 pool := resource_pool.New( 470 logger, 471 *binPath, 472 *depotPath, 473 config, 474 cakeOrdinator, 475 rootfsCleaner, 476 mappingList, 477 parsedExternalIP, 478 *mtu, 479 subnetPool, 480 bridgemgr.New("w"+config.Tag+"b-", &devices.Bridge{}, &devices.Link{}), 481 ipTablesMgr, 482 injector, 483 iptables.NewGlobalChain(config.IPTables.Filter.DefaultChain, runner, logger.Session("global-chain")), 484 portPool, 485 strings.Split(*denyNetworks, ","), 486 strings.Split(*allowNetworks, ","), 487 runner, 488 quotaManager, 489 currentContainerVersion, 490 system.MkdirChowner{}, 491 ) 492 493 systemInfo := sysinfo.NewProvider(*depotPath) 494 495 backend := linux_backend.New(logger, pool, repo, injector, systemInfo, layercake.GraphPath(*graphRoot), *snapshotsPath, int(*maxContainers)) 496 497 err = backend.Setup() 498 if err != nil { 499 logger.Fatal("failed-to-set-up-backend", err) 500 } 501 502 graceTime := *containerGraceTime 503 504 gardenServer := server.New(*listenNetwork, *listenAddr, graceTime, backend, logger) 505 506 err = gardenServer.Start() 507 if err != nil { 508 logger.Fatal("failed-to-start-server", err) 509 } 510 511 clock := clock.NewClock() 512 metronNotifier := metrics.NewPeriodicMetronNotifier(logger, metricsProvider, *metricsEmissionInterval, clock) 513 metronNotifier.Start() 514 515 signals := make(chan os.Signal, 1) 516 517 go func() { 518 <-signals 519 520 portPoolState = portPool.RefreshState() 521 port_pool.SaveState(path.Join(*stateDirPath, "port_pool.json"), portPoolState) 522 523 gardenServer.Stop() 524 metronNotifier.Stop() 525 526 os.Exit(0) 527 }() 528 529 signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM, syscall.SIGHUP) 530 531 logger.Info("started", lager.Data{ 532 "network": *listenNetwork, 533 "addr": *listenAddr, 534 }) 535 536 select {} 537 } 538 539 func missing(flagName string) { 540 println("missing " + flagName) 541 println() 542 flag.Usage() 543 os.Exit(1) 544 } 545 546 func initializeDropsonde(logger lager.Logger) { 547 err := dropsonde.Initialize(*dropsondeDestination, *dropsondeOrigin) 548 if err != nil { 549 logger.Error("failed to initialize dropsonde", err) 550 } 551 } 552 553 func createIPTablesManager(sysconfig sysconfig.Config, runner command_runner.CommandRunner, log lager.Logger) linux_container.IPTablesManager { 554 filterChain := iptables_manager.NewFilterChain(&sysconfig.IPTables.Filter, runner, log.Session("iptables-manager-filter")) 555 natChain := iptables_manager.NewNATChain(&sysconfig.IPTables.NAT, runner, log.Session("iptables-manager-nat")) 556 return iptables_manager.New().AddChain(filterChain).AddChain(natChain) 557 } 558 559 type provider struct { 560 useKernelLogging bool 561 chainPrefix string 562 runner command_runner.CommandRunner 563 log lager.Logger 564 portPool *port_pool.PortPool 565 ipTablesMgr linux_container.IPTablesManager 566 quotaManager linux_container.QuotaManager 567 sysconfig sysconfig.Config 568 } 569 570 func (p *provider) ProvideFilter(containerId string) network.Filter { 571 return network.NewFilter(iptables.NewLoggingChain(p.chainPrefix+containerId, p.useKernelLogging, p.runner, p.log.Session(containerId).Session("filter"))) 572 } 573 574 func (p *provider) ProvideContainer(spec linux_backend.LinuxContainerSpec) linux_backend.Container { 575 cgroupReader := &cgroups_manager.LinuxCgroupReader{ 576 Path: p.sysconfig.CgroupNodeFilePath, 577 } 578 579 cgroupsManager := cgroups_manager.New(p.sysconfig.CgroupPath, spec.ID, cgroupReader) 580 581 oomWatcher := linux_container.NewOomNotifier( 582 p.runner, spec.ContainerPath, cgroupsManager, 583 ) 584 585 return linux_container.NewLinuxContainer( 586 spec, 587 p.portPool, 588 p.runner, 589 cgroupsManager, 590 p.quotaManager, 591 bandwidth_manager.New(spec.ContainerPath, spec.ID, p.runner), 592 process_tracker.New(p.log.Session("process-tracker"), spec.ContainerPath, p.runner), 593 p.ProvideFilter(spec.ID), 594 p.ipTablesMgr, 595 devices.Link{Name: p.sysconfig.NetworkInterfacePrefix + spec.ID + "-0"}, 596 oomWatcher, 597 p.log.Session("container", lager.Data{"handle": spec.Handle}), 598 ) 599 } 600 601 func selectGraphDriver(logger lager.Logger, name string, graphRoot string) (graphdriver.Driver, error) { 602 // silence docker graph debug logging; we'll do our own warning for non-aufs 603 // driver selection 604 logrus.SetLevel(logrus.WarnLevel) 605 606 var driver graphdriver.Driver 607 var err error 608 if name == "auto" { 609 driver, err = graphdriver.New(graphRoot, nil) 610 } else { 611 driver, err = graphdriver.GetDriver(name, graphRoot, nil) 612 } 613 if err != nil { 614 return nil, err 615 } 616 617 driverName := driver.String() 618 619 if driverName != "aufs" { 620 logger.Info("unsupported-graph-driver", lager.Data{"name": driverName}) 621 } 622 623 return driver, nil 624 }