github.com/cloud-green/juju@v0.0.0-20151002100041-a00291338d3d/worker/provisioner/lxc-broker.go (about) 1 // Copyright 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package provisioner 5 6 import ( 7 "bufio" 8 "bytes" 9 "fmt" 10 "net" 11 "os" 12 "strings" 13 "text/template" 14 15 "github.com/juju/errors" 16 "github.com/juju/loggo" 17 "github.com/juju/names" 18 "github.com/juju/utils/arch" 19 "github.com/juju/utils/exec" 20 21 "github.com/juju/juju/agent" 22 apiprovisioner "github.com/juju/juju/api/provisioner" 23 "github.com/juju/juju/apiserver/params" 24 "github.com/juju/juju/cloudconfig/instancecfg" 25 "github.com/juju/juju/container" 26 "github.com/juju/juju/container/lxc" 27 "github.com/juju/juju/environs" 28 "github.com/juju/juju/instance" 29 "github.com/juju/juju/network" 30 "github.com/juju/juju/storage/looputil" 31 "github.com/juju/juju/tools" 32 "github.com/juju/juju/version" 33 ) 34 35 var lxcLogger = loggo.GetLogger("juju.provisioner.lxc") 36 37 var _ environs.InstanceBroker = (*lxcBroker)(nil) 38 39 type APICalls interface { 40 ContainerConfig() (params.ContainerConfig, error) 41 PrepareContainerInterfaceInfo(names.MachineTag) ([]network.InterfaceInfo, error) 42 GetContainerInterfaceInfo(names.MachineTag) ([]network.InterfaceInfo, error) 43 } 44 45 var _ APICalls = (*apiprovisioner.State)(nil) 46 47 // Override for testing. 48 var NewLxcBroker = newLxcBroker 49 50 func newLxcBroker( 51 api APICalls, 52 agentConfig agent.Config, 53 managerConfig container.ManagerConfig, 54 imageURLGetter container.ImageURLGetter, 55 enableNAT bool, 56 defaultMTU int, 57 ) (environs.InstanceBroker, error) { 58 manager, err := lxc.NewContainerManager( 59 managerConfig, imageURLGetter, looputil.NewLoopDeviceManager(), 60 ) 61 if err != nil { 62 return nil, err 63 } 64 return &lxcBroker{ 65 manager: manager, 66 api: api, 67 agentConfig: agentConfig, 68 enableNAT: enableNAT, 69 defaultMTU: defaultMTU, 70 }, nil 71 } 72 73 type lxcBroker struct { 74 manager container.Manager 75 api APICalls 76 agentConfig agent.Config 77 enableNAT bool 78 defaultMTU int 79 } 80 81 // StartInstance is specified in the Broker interface. 82 func (broker *lxcBroker) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) { 83 if args.InstanceConfig.HasNetworks() { 84 return nil, errors.New("starting lxc containers with networks is not supported yet") 85 } 86 // TODO: refactor common code out of the container brokers. 87 machineId := args.InstanceConfig.MachineId 88 lxcLogger.Infof("starting lxc container for machineId: %s", machineId) 89 90 // Default to using the host network until we can configure. 91 bridgeDevice := broker.agentConfig.Value(agent.LxcBridge) 92 if bridgeDevice == "" { 93 bridgeDevice = lxc.DefaultLxcBridge 94 } 95 96 if !environs.AddressAllocationEnabled() { 97 logger.Debugf( 98 "address allocation feature flag not enabled; using DHCP for container %q", 99 machineId, 100 ) 101 } else { 102 logger.Debugf("trying to allocate static IP for container %q", machineId) 103 allocatedInfo, err := configureContainerNetwork( 104 machineId, 105 bridgeDevice, 106 broker.api, 107 args.NetworkInfo, 108 true, // allocate a new address. 109 broker.enableNAT, 110 ) 111 if err != nil { 112 // It's fine, just ignore it. The effect will be that the 113 // container won't have a static address configured. 114 logger.Infof("not allocating static IP for container %q: %v", machineId, err) 115 } else { 116 args.NetworkInfo = allocatedInfo 117 } 118 } 119 network := container.BridgeNetworkConfig(bridgeDevice, broker.defaultMTU, args.NetworkInfo) 120 121 // The provisioner worker will provide all tools it knows about 122 // (after applying explicitly specified constraints), which may 123 // include tools for architectures other than the host's. We 124 // must constrain to the host's architecture for LXC. 125 arch := arch.HostArch() 126 archTools, err := args.Tools.Match(tools.Filter{ 127 Arch: arch, 128 }) 129 if err == tools.ErrNoMatches { 130 return nil, errors.Errorf( 131 "need tools for arch %s, only found %s", 132 arch, 133 args.Tools.Arches(), 134 ) 135 } 136 137 series := archTools.OneSeries() 138 args.InstanceConfig.MachineContainerType = instance.LXC 139 args.InstanceConfig.Tools = archTools[0] 140 141 config, err := broker.api.ContainerConfig() 142 if err != nil { 143 lxcLogger.Errorf("failed to get container config: %v", err) 144 return nil, err 145 } 146 storageConfig := &container.StorageConfig{ 147 AllowMount: config.AllowLXCLoopMounts, 148 } 149 150 if err := instancecfg.PopulateInstanceConfig( 151 args.InstanceConfig, 152 config.ProviderType, 153 config.AuthorizedKeys, 154 config.SSLHostnameVerification, 155 config.Proxy, 156 config.AptProxy, 157 config.AptMirror, 158 config.PreferIPv6, 159 config.EnableOSRefreshUpdate, 160 config.EnableOSUpgrade, 161 ); err != nil { 162 lxcLogger.Errorf("failed to populate machine config: %v", err) 163 return nil, err 164 } 165 166 inst, hardware, err := broker.manager.CreateContainer(args.InstanceConfig, series, network, storageConfig) 167 if err != nil { 168 lxcLogger.Errorf("failed to start container: %v", err) 169 return nil, err 170 } 171 lxcLogger.Infof("started lxc container for machineId: %s, %s, %s", machineId, inst.Id(), hardware.String()) 172 return &environs.StartInstanceResult{ 173 Instance: inst, 174 Hardware: hardware, 175 NetworkInfo: network.Interfaces, 176 }, nil 177 } 178 179 // StopInstances shuts down the given instances. 180 func (broker *lxcBroker) StopInstances(ids ...instance.Id) error { 181 // TODO: potentially parallelise. 182 for _, id := range ids { 183 lxcLogger.Infof("stopping lxc container for instance: %s", id) 184 if err := broker.manager.DestroyContainer(id); err != nil { 185 lxcLogger.Errorf("container did not stop: %v", err) 186 return err 187 } 188 } 189 return nil 190 } 191 192 // AllInstances only returns running containers. 193 func (broker *lxcBroker) AllInstances() (result []instance.Instance, err error) { 194 return broker.manager.ListContainers() 195 } 196 197 type hostArchToolsFinder struct { 198 f ToolsFinder 199 } 200 201 // FindTools is defined on the ToolsFinder interface. 202 func (h hostArchToolsFinder) FindTools(v version.Number, series, _ string) (tools.List, error) { 203 // Override the arch constraint with the arch of the host. 204 return h.f.FindTools(v, series, arch.HostArch()) 205 } 206 207 // resolvConf is the full path to the resolv.conf file on the local 208 // system. Defined here so it can be overriden for testing. 209 var resolvConf = "/etc/resolv.conf" 210 211 // localDNSServers parses the /etc/resolv.conf file (if available) and 212 // extracts all nameservers addresses, and the default search domain 213 // and returns them. 214 func localDNSServers() ([]network.Address, string, error) { 215 file, err := os.Open(resolvConf) 216 if os.IsNotExist(err) { 217 return nil, "", nil 218 } else if err != nil { 219 return nil, "", errors.Annotatef(err, "cannot open %q", resolvConf) 220 } 221 defer file.Close() 222 223 var addresses []network.Address 224 var searchDomain string 225 scanner := bufio.NewScanner(file) 226 for scanner.Scan() { 227 line := strings.TrimSpace(scanner.Text()) 228 if strings.HasPrefix(line, "#") { 229 // Skip comments. 230 continue 231 } 232 if strings.HasPrefix(line, "nameserver") { 233 address := strings.TrimPrefix(line, "nameserver") 234 // Drop comments after the address, if any. 235 if strings.Contains(address, "#") { 236 address = address[:strings.Index(address, "#")] 237 } 238 address = strings.TrimSpace(address) 239 addresses = append(addresses, network.NewAddress(address)) 240 } 241 if strings.HasPrefix(line, "search") { 242 searchDomain = strings.TrimPrefix(line, "search") 243 // Drop comments after the domain, if any. 244 if strings.Contains(searchDomain, "#") { 245 searchDomain = searchDomain[:strings.Index(searchDomain, "#")] 246 } 247 searchDomain = strings.TrimSpace(searchDomain) 248 } 249 } 250 251 if err := scanner.Err(); err != nil { 252 return nil, "", errors.Annotatef(err, "cannot read DNS servers from %q", resolvConf) 253 } 254 return addresses, searchDomain, nil 255 } 256 257 // ipRouteAdd is the command template to add a static route for 258 // .ContainerIP using the .HostBridge device (usually lxcbr0 or virbr0). 259 var ipRouteAdd = mustParseTemplate("ipRouteAdd", ` 260 ip route add {{.ContainerIP}} dev {{.HostBridge}}`[1:]) 261 262 type IptablesRule struct { 263 Table string 264 Chain string 265 Rule string 266 } 267 268 var skipSNATRule = IptablesRule{ 269 // For EC2, to get internet access we need traffic to appear with 270 // source address matching the container's host. For internal 271 // traffic we want to keep the container IP because it is used 272 // by some services. This rule sits above the SNAT rule, which 273 // changes the source address of traffic to the container host IP 274 // address, skipping this modification if the traffic destination 275 // is inside the EC2 VPC. 276 "nat", 277 "POSTROUTING", 278 "-d {{.SubnetCIDR}} -o {{.HostIF}} -j RETURN", 279 } 280 281 var iptablesRules = map[string]IptablesRule{ 282 // iptablesCheckSNAT is the command template to verify if a SNAT 283 // rule already exists for the host NIC named .HostIF (usually 284 // eth0) and source address .HostIP (usually eth0's address). We 285 // need to check whether the rule exists because we only want to 286 // add it once. Exit code 0 means the rule exists, 1 means it 287 // doesn't. 288 "iptablesSNAT": { 289 "nat", 290 "POSTROUTING", 291 "-o {{.HostIF}} -j SNAT --to-source {{.HostIP}}", 292 }, "iptablesForwardOut": { 293 // Ensure that we have ACCEPT rules that apply to the containers that 294 // we are creating so any DROP rules added by libvirt while setting 295 // up virbr0 further down the chain don't disrupt wanted traffic. 296 "filter", 297 "FORWARD", 298 "-d {{.ContainerCIDR}} -o {{.HostBridge}} -j ACCEPT", 299 }, "iptablesForwardIn": { 300 "filter", 301 "FORWARD", 302 "-s {{.ContainerCIDR}} -i {{.HostBridge}} -j ACCEPT", 303 }} 304 305 // mustParseTemplate works like template.Parse, but panics on error. 306 func mustParseTemplate(name, source string) *template.Template { 307 templ, err := template.New(name).Parse(source) 308 if err != nil { 309 panic(err.Error()) 310 } 311 return templ 312 } 313 314 // mustExecTemplate works like template.Parse followed by template.Execute, 315 // but panics on error. 316 func mustExecTemplate(name, tmpl string, data interface{}) string { 317 t := mustParseTemplate(name, tmpl) 318 var buf bytes.Buffer 319 if err := t.Execute(&buf, data); err != nil { 320 panic(err.Error()) 321 } 322 return buf.String() 323 } 324 325 // runTemplateCommand executes the given template with the given data, 326 // which generates a command to execute. If exitNonZeroOK is true, no 327 // error is returned if the exit code is not 0, otherwise an error is 328 // returned. 329 func runTemplateCommand(t *template.Template, exitNonZeroOK bool, data interface{}) ( 330 exitCode int, err error, 331 ) { 332 // Clone the template to ensure the original won't be changed. 333 cloned, err := t.Clone() 334 if err != nil { 335 return -1, errors.Annotatef(err, "cannot clone command template %q", t.Name()) 336 } 337 var buf bytes.Buffer 338 if err := cloned.Execute(&buf, data); err != nil { 339 return -1, errors.Annotatef(err, "cannot execute command template %q", t.Name()) 340 } 341 command := buf.String() 342 logger.Debugf("running command %q", command) 343 result, err := exec.RunCommands(exec.RunParams{Commands: command}) 344 if err != nil { 345 return -1, errors.Annotatef(err, "cannot run command %q", command) 346 } 347 exitCode = result.Code 348 stdout := string(result.Stdout) 349 stderr := string(result.Stderr) 350 logger.Debugf( 351 "command %q returned code=%d, stdout=%q, stderr=%q", 352 command, exitCode, stdout, stderr, 353 ) 354 if exitCode != 0 { 355 if exitNonZeroOK { 356 return exitCode, nil 357 } 358 return exitCode, errors.Errorf( 359 "command %q failed with exit code %d", 360 command, exitCode, 361 ) 362 } 363 return 0, nil 364 } 365 366 // setupRoutesAndIPTables sets up on the host machine the needed 367 // iptables rules and static routes for an addressable container. 368 var setupRoutesAndIPTables = func( 369 primaryNIC string, 370 primaryAddr network.Address, 371 bridgeName string, 372 ifaceInfo []network.InterfaceInfo, 373 enableNAT bool, 374 ) error { 375 376 if primaryNIC == "" || primaryAddr.Value == "" || bridgeName == "" || len(ifaceInfo) == 0 { 377 return errors.Errorf("primaryNIC, primaryAddr, bridgeName, and ifaceInfo must be all set") 378 } 379 380 for _, iface := range ifaceInfo { 381 containerIP := iface.Address.Value 382 if containerIP == "" { 383 return errors.Errorf("container IP %q must be set", containerIP) 384 } 385 data := struct { 386 HostIF string 387 HostIP string 388 HostBridge string 389 ContainerIP string 390 ContainerCIDR string 391 SubnetCIDR string 392 }{primaryNIC, primaryAddr.Value, bridgeName, containerIP, iface.CIDR, iface.CIDR} 393 394 var addRuleIfDoesNotExist = func(name string, rule IptablesRule) error { 395 check := mustExecTemplate("rule", "iptables -t {{.Table}} -C {{.Chain}} {{.Rule}}", rule) 396 t := mustParseTemplate(name+"Check", check) 397 398 code, err := runTemplateCommand(t, true, data) 399 if err != nil { 400 return errors.Trace(err) 401 } 402 switch code { 403 case 0: 404 // Rule does exist. Do nothing 405 case 1: 406 // Rule does not exist, add it. We insert the rule at the top of the list so it precedes any 407 // REJECT rules. 408 action := mustExecTemplate("action", "iptables -t {{.Table}} -I {{.Chain}} 1 {{.Rule}}", rule) 409 t = mustParseTemplate(name+"Add", action) 410 _, err = runTemplateCommand(t, false, data) 411 if err != nil { 412 return errors.Trace(err) 413 } 414 default: 415 // Unexpected code - better report it. 416 return errors.Errorf("iptables failed with unexpected exit code %d", code) 417 } 418 return nil 419 } 420 421 for name, rule := range iptablesRules { 422 if !enableNAT && name == "iptablesSNAT" { 423 // Do not add the SNAT rule if we shouldn't enable 424 // NAT. 425 continue 426 } 427 if err := addRuleIfDoesNotExist(name, rule); err != nil { 428 return err 429 } 430 } 431 432 // TODO(dooferlad): subnets should be a list of subnets in the EC2 VPC and 433 // should be empty for MAAS. See bug http://pad.lv/1443942 434 if enableNAT { 435 // Only add the following hack to allow AWS egress traffic 436 // for hosted containers to work. 437 subnets := []string{data.HostIP + "/16"} 438 for _, subnet := range subnets { 439 data.SubnetCIDR = subnet 440 if err := addRuleIfDoesNotExist("skipSNAT", skipSNATRule); err != nil { 441 return err 442 } 443 } 444 } 445 446 code, err := runTemplateCommand(ipRouteAdd, false, data) 447 // Ignore errors if the exit code was 2, which signals that the route was not added 448 // because it already exists. 449 if code != 2 && err != nil { 450 return errors.Trace(err) 451 } 452 if code == 2 { 453 logger.Tracef("route already exists - not added") 454 } else { 455 logger.Tracef("route added: container uses host network interface") 456 } 457 } 458 logger.Infof("successfully configured iptables and routes for container interfaces") 459 460 return nil 461 } 462 463 var ( 464 netInterfaces = net.Interfaces 465 interfaceAddrs = (*net.Interface).Addrs 466 ) 467 468 // discoverPrimaryNIC returns the name of the first network interface 469 // on the machine which is up and has address, along with the first 470 // address it has. 471 func discoverPrimaryNIC() (string, network.Address, error) { 472 interfaces, err := netInterfaces() 473 if err != nil { 474 return "", network.Address{}, errors.Annotatef(err, "cannot get network interfaces") 475 } 476 logger.Tracef("trying to discover primary network interface") 477 for _, iface := range interfaces { 478 if iface.Flags&net.FlagLoopback != 0 { 479 // Skip the loopback. 480 logger.Tracef("not using loopback interface %q", iface.Name) 481 continue 482 } 483 if iface.Flags&net.FlagUp != 0 { 484 // Possibly the primary, but ensure it has an address as 485 // well. 486 logger.Tracef("verifying interface %q has addresses", iface.Name) 487 addrs, err := interfaceAddrs(&iface) 488 if err != nil { 489 return "", network.Address{}, errors.Annotatef(err, "cannot get %q addresses", iface.Name) 490 } 491 if len(addrs) > 0 { 492 // We found it. 493 // Check if it's an IP or a CIDR. 494 addr := addrs[0].String() 495 ip := net.ParseIP(addr) 496 if ip == nil { 497 // Try a CIDR. 498 ip, _, err = net.ParseCIDR(addr) 499 if err != nil { 500 return "", network.Address{}, errors.Annotatef(err, "cannot parse address %q", addr) 501 } 502 } 503 addr = ip.String() 504 505 logger.Tracef("primary network interface is %q, address %q", iface.Name, addr) 506 return iface.Name, network.NewAddress(addr), nil 507 } 508 } 509 } 510 return "", network.Address{}, errors.Errorf("cannot detect the primary network interface") 511 } 512 513 // configureContainerNetworking tries to allocate a static IP address 514 // for the given containerId using the provisioner API, when 515 // allocateAddress is true. Otherwise it configures the container with 516 // an already allocated address, when allocateAddress is false (e.g. 517 // after a host reboot). If the API call fails, it's not critical - 518 // just a warning, and it won't cause StartInstance to fail. 519 func configureContainerNetwork( 520 containerId, bridgeDevice string, 521 apiFacade APICalls, 522 ifaceInfo []network.InterfaceInfo, 523 allocateAddress bool, 524 enableNAT bool, 525 ) (finalIfaceInfo []network.InterfaceInfo, err error) { 526 defer func() { 527 if err != nil { 528 logger.Warningf( 529 "failed configuring a static IP for container %q: %v", 530 containerId, err, 531 ) 532 } 533 }() 534 535 if len(ifaceInfo) != 0 { 536 // When we already have interface info, don't overwrite it. 537 return nil, nil 538 } 539 540 var primaryNIC string 541 var primaryAddr network.Address 542 primaryNIC, primaryAddr, err = discoverPrimaryNIC() 543 if err != nil { 544 return nil, errors.Trace(err) 545 } 546 547 if allocateAddress { 548 logger.Debugf("trying to allocate a static IP for container %q", containerId) 549 finalIfaceInfo, err = apiFacade.PrepareContainerInterfaceInfo(names.NewMachineTag(containerId)) 550 } else { 551 logger.Debugf("getting allocated static IP for container %q", containerId) 552 finalIfaceInfo, err = apiFacade.GetContainerInterfaceInfo(names.NewMachineTag(containerId)) 553 } 554 if err != nil { 555 return nil, errors.Trace(err) 556 } 557 logger.Debugf("container interface info result %#v", finalIfaceInfo) 558 559 // Populate ConfigType and DNSServers as needed. 560 var dnsServers []network.Address 561 var searchDomain string 562 dnsServers, searchDomain, err = localDNSServers() 563 if err != nil { 564 return nil, errors.Trace(err) 565 } 566 // Generate the final configuration for each container interface. 567 for i, _ := range finalIfaceInfo { 568 // Always start at the first device index and generate the 569 // interface name based on that. We need to do this otherwise 570 // the container will inherit the host's device index and 571 // interface name. 572 finalIfaceInfo[i].DeviceIndex = i 573 finalIfaceInfo[i].InterfaceName = fmt.Sprintf("eth%d", i) 574 finalIfaceInfo[i].ConfigType = network.ConfigStatic 575 finalIfaceInfo[i].DNSServers = dnsServers 576 finalIfaceInfo[i].DNSSearch = searchDomain 577 finalIfaceInfo[i].GatewayAddress = primaryAddr 578 if finalIfaceInfo[i].NetworkName == "" { 579 finalIfaceInfo[i].NetworkName = network.DefaultPrivate 580 } 581 if finalIfaceInfo[i].ProviderId == "" { 582 finalIfaceInfo[i].ProviderId = network.DefaultProviderId 583 } 584 } 585 err = setupRoutesAndIPTables( 586 primaryNIC, 587 primaryAddr, 588 bridgeDevice, 589 finalIfaceInfo, 590 enableNAT, 591 ) 592 if err != nil { 593 return nil, errors.Trace(err) 594 } 595 return finalIfaceInfo, nil 596 } 597 598 // MaintainInstance checks that the container's host has the required iptables and routing 599 // rules to make the container visible to both the host and other machines on the same subnet. 600 func (broker *lxcBroker) MaintainInstance(args environs.StartInstanceParams) error { 601 machineId := args.InstanceConfig.MachineId 602 if !environs.AddressAllocationEnabled() { 603 lxcLogger.Debugf("address allocation disabled: Not running maintenance for lxc container with machineId: %s", 604 machineId) 605 return nil 606 } 607 608 lxcLogger.Debugf("running maintenance for lxc container with machineId: %s", machineId) 609 610 // Default to using the host network until we can configure. 611 bridgeDevice := broker.agentConfig.Value(agent.LxcBridge) 612 if bridgeDevice == "" { 613 bridgeDevice = lxc.DefaultLxcBridge 614 } 615 _, err := configureContainerNetwork( 616 machineId, 617 bridgeDevice, 618 broker.api, 619 args.NetworkInfo, 620 false, // don't allocate a new address. 621 broker.enableNAT, 622 ) 623 return err 624 }