github.com/cloud-green/juju@v0.0.0-20151002100041-a00291338d3d/worker/provisioner/provisioner_task.go (about) 1 // Copyright 2012, 2013 Canonical Ltd. 2 // Licensed under the AGPLv3, see LICENCE file for details. 3 4 package provisioner 5 6 import ( 7 "fmt" 8 "regexp" 9 "time" 10 11 "github.com/juju/errors" 12 "github.com/juju/names" 13 "github.com/juju/utils" 14 "github.com/juju/utils/set" 15 "launchpad.net/tomb" 16 17 apiprovisioner "github.com/juju/juju/api/provisioner" 18 apiwatcher "github.com/juju/juju/api/watcher" 19 "github.com/juju/juju/apiserver/params" 20 "github.com/juju/juju/cloudconfig/instancecfg" 21 "github.com/juju/juju/constraints" 22 "github.com/juju/juju/environmentserver/authentication" 23 "github.com/juju/juju/environs" 24 "github.com/juju/juju/environs/config" 25 "github.com/juju/juju/instance" 26 "github.com/juju/juju/network" 27 "github.com/juju/juju/state/watcher" 28 "github.com/juju/juju/storage" 29 coretools "github.com/juju/juju/tools" 30 "github.com/juju/juju/version" 31 "github.com/juju/juju/worker" 32 ) 33 34 type ProvisionerTask interface { 35 worker.Worker 36 Stop() error 37 Dying() <-chan struct{} 38 Err() error 39 40 // SetHarvestMode sets a flag to indicate how the provisioner task 41 // should harvest machines. See config.HarvestMode for 42 // documentation of behavior. 43 SetHarvestMode(mode config.HarvestMode) 44 } 45 46 type MachineGetter interface { 47 Machine(names.MachineTag) (*apiprovisioner.Machine, error) 48 MachinesWithTransientErrors() ([]*apiprovisioner.Machine, []params.StatusResult, error) 49 } 50 51 // ToolsFinder is an interface used for finding tools to run on 52 // provisioned instances. 53 type ToolsFinder interface { 54 // FindTools returns a list of tools matching the specified 55 // version, series, and architecture. If arch is empty, the 56 // implementation is expected to use a well documented default. 57 FindTools(version version.Number, series string, arch string) (coretools.List, error) 58 } 59 60 var _ MachineGetter = (*apiprovisioner.State)(nil) 61 var _ ToolsFinder = (*apiprovisioner.State)(nil) 62 63 func NewProvisionerTask( 64 machineTag names.MachineTag, 65 harvestMode config.HarvestMode, 66 machineGetter MachineGetter, 67 toolsFinder ToolsFinder, 68 machineWatcher apiwatcher.StringsWatcher, 69 retryWatcher apiwatcher.NotifyWatcher, 70 broker environs.InstanceBroker, 71 auth authentication.AuthenticationProvider, 72 imageStream string, 73 secureServerConnection bool, 74 ) ProvisionerTask { 75 task := &provisionerTask{ 76 machineTag: machineTag, 77 machineGetter: machineGetter, 78 toolsFinder: toolsFinder, 79 machineWatcher: machineWatcher, 80 retryWatcher: retryWatcher, 81 broker: broker, 82 auth: auth, 83 harvestMode: harvestMode, 84 harvestModeChan: make(chan config.HarvestMode, 1), 85 machines: make(map[string]*apiprovisioner.Machine), 86 imageStream: imageStream, 87 secureServerConnection: secureServerConnection, 88 } 89 go func() { 90 defer task.tomb.Done() 91 task.tomb.Kill(task.loop()) 92 }() 93 return task 94 } 95 96 type provisionerTask struct { 97 machineTag names.MachineTag 98 machineGetter MachineGetter 99 toolsFinder ToolsFinder 100 machineWatcher apiwatcher.StringsWatcher 101 retryWatcher apiwatcher.NotifyWatcher 102 broker environs.InstanceBroker 103 tomb tomb.Tomb 104 auth authentication.AuthenticationProvider 105 imageStream string 106 secureServerConnection bool 107 harvestMode config.HarvestMode 108 harvestModeChan chan config.HarvestMode 109 // instance id -> instance 110 instances map[instance.Id]instance.Instance 111 // machine id -> machine 112 machines map[string]*apiprovisioner.Machine 113 } 114 115 // Kill implements worker.Worker.Kill. 116 func (task *provisionerTask) Kill() { 117 task.tomb.Kill(nil) 118 } 119 120 // Wait implements worker.Worker.Wait. 121 func (task *provisionerTask) Wait() error { 122 return task.tomb.Wait() 123 } 124 125 func (task *provisionerTask) Stop() error { 126 task.Kill() 127 return task.Wait() 128 } 129 130 func (task *provisionerTask) Dying() <-chan struct{} { 131 return task.tomb.Dying() 132 } 133 134 func (task *provisionerTask) Err() error { 135 return task.tomb.Err() 136 } 137 138 func (task *provisionerTask) loop() error { 139 logger.Infof("Starting up provisioner task %s", task.machineTag) 140 defer watcher.Stop(task.machineWatcher, &task.tomb) 141 142 // Don't allow the harvesting mode to change until we have read at 143 // least one set of changes, which will populate the task.machines 144 // map. Otherwise we will potentially see all legitimate instances 145 // as unknown. 146 var harvestModeChan chan config.HarvestMode 147 148 // Not all provisioners have a retry channel. 149 var retryChan <-chan struct{} 150 if task.retryWatcher != nil { 151 retryChan = task.retryWatcher.Changes() 152 } 153 154 // When the watcher is started, it will have the initial changes be all 155 // the machines that are relevant. Also, since this is available straight 156 // away, we know there will be some changes right off the bat. 157 for { 158 select { 159 case <-task.tomb.Dying(): 160 logger.Infof("Shutting down provisioner task %s", task.machineTag) 161 return tomb.ErrDying 162 case ids, ok := <-task.machineWatcher.Changes(): 163 if !ok { 164 return watcher.EnsureErr(task.machineWatcher) 165 } 166 if err := task.processMachines(ids); err != nil { 167 return errors.Annotate(err, "failed to process updated machines") 168 } 169 // We've seen a set of changes. Enable modification of 170 // harvesting mode. 171 harvestModeChan = task.harvestModeChan 172 case harvestMode := <-harvestModeChan: 173 if harvestMode == task.harvestMode { 174 break 175 } 176 177 logger.Infof("harvesting mode changed to %s", harvestMode) 178 task.harvestMode = harvestMode 179 180 if harvestMode.HarvestUnknown() { 181 182 logger.Infof("harvesting unknown machines") 183 if err := task.processMachines(nil); err != nil { 184 return errors.Annotate(err, "failed to process machines after safe mode disabled") 185 } 186 } 187 case <-retryChan: 188 if err := task.processMachinesWithTransientErrors(); err != nil { 189 return errors.Annotate(err, "failed to process machines with transient errors") 190 } 191 } 192 } 193 } 194 195 // SetHarvestMode implements ProvisionerTask.SetHarvestMode(). 196 func (task *provisionerTask) SetHarvestMode(mode config.HarvestMode) { 197 select { 198 case task.harvestModeChan <- mode: 199 case <-task.Dying(): 200 } 201 } 202 203 func (task *provisionerTask) processMachinesWithTransientErrors() error { 204 machines, statusResults, err := task.machineGetter.MachinesWithTransientErrors() 205 if err != nil { 206 return nil 207 } 208 logger.Tracef("processMachinesWithTransientErrors(%v)", statusResults) 209 var pending []*apiprovisioner.Machine 210 for i, status := range statusResults { 211 if status.Error != nil { 212 logger.Errorf("cannot retry provisioning of machine %q: %v", status.Id, status.Error) 213 continue 214 } 215 machine := machines[i] 216 if err := machine.SetStatus(params.StatusPending, "", nil); err != nil { 217 logger.Errorf("cannot reset status of machine %q: %v", status.Id, err) 218 continue 219 } 220 task.machines[machine.Tag().String()] = machine 221 pending = append(pending, machine) 222 } 223 return task.startMachines(pending) 224 } 225 226 func (task *provisionerTask) processMachines(ids []string) error { 227 logger.Tracef("processMachines(%v)", ids) 228 229 // Populate the tasks maps of current instances and machines. 230 if err := task.populateMachineMaps(ids); err != nil { 231 return err 232 } 233 234 // Find machines without an instance id or that are dead 235 pending, dead, maintain, err := task.pendingOrDeadOrMaintain(ids) 236 if err != nil { 237 return err 238 } 239 240 // Stop all machines that are dead 241 stopping := task.instancesForMachines(dead) 242 243 // Find running instances that have no machines associated 244 unknown, err := task.findUnknownInstances(stopping) 245 if err != nil { 246 return err 247 } 248 if !task.harvestMode.HarvestUnknown() { 249 logger.Infof( 250 "%s is set to %s; unknown instances not stopped %v", 251 config.ProvisionerHarvestModeKey, 252 task.harvestMode.String(), 253 instanceIds(unknown), 254 ) 255 unknown = nil 256 } 257 if task.harvestMode.HarvestNone() || !task.harvestMode.HarvestDestroyed() { 258 logger.Infof( 259 `%s is set to "%s"; will not harvest %s`, 260 config.ProvisionerHarvestModeKey, 261 task.harvestMode.String(), 262 instanceIds(stopping), 263 ) 264 stopping = nil 265 } 266 267 if len(stopping) > 0 { 268 logger.Infof("stopping known instances %v", stopping) 269 } 270 if len(unknown) > 0 { 271 logger.Infof("stopping unknown instances %v", instanceIds(unknown)) 272 } 273 // It's important that we stop unknown instances before starting 274 // pending ones, because if we start an instance and then fail to 275 // set its InstanceId on the machine we don't want to start a new 276 // instance for the same machine ID. 277 if err := task.stopInstances(append(stopping, unknown...)); err != nil { 278 return err 279 } 280 281 // Remove any dead machines from state. 282 for _, machine := range dead { 283 logger.Infof("removing dead machine %q", machine) 284 if err := machine.Remove(); err != nil { 285 logger.Errorf("failed to remove dead machine %q", machine) 286 } 287 delete(task.machines, machine.Id()) 288 } 289 290 // Any machines that require maintenance get pinged 291 task.maintainMachines(maintain) 292 293 // Start an instance for the pending ones 294 return task.startMachines(pending) 295 } 296 297 func instanceIds(instances []instance.Instance) []string { 298 ids := make([]string, 0, len(instances)) 299 for _, inst := range instances { 300 ids = append(ids, string(inst.Id())) 301 } 302 return ids 303 } 304 305 // populateMachineMaps updates task.instances. Also updates 306 // task.machines map if a list of IDs is given. 307 func (task *provisionerTask) populateMachineMaps(ids []string) error { 308 task.instances = make(map[instance.Id]instance.Instance) 309 310 instances, err := task.broker.AllInstances() 311 if err != nil { 312 return errors.Annotate(err, "failed to get all instances from broker") 313 } 314 for _, i := range instances { 315 task.instances[i.Id()] = i 316 } 317 318 // Update the machines map with new data for each of the machines in the 319 // change list. 320 // TODO(thumper): update for API server later to get all machines in one go. 321 for _, id := range ids { 322 machineTag := names.NewMachineTag(id) 323 machine, err := task.machineGetter.Machine(machineTag) 324 switch { 325 case params.IsCodeNotFoundOrCodeUnauthorized(err): 326 logger.Debugf("machine %q not found in state", id) 327 delete(task.machines, id) 328 case err == nil: 329 task.machines[id] = machine 330 default: 331 return errors.Annotatef(err, "failed to get machine %v", id) 332 } 333 } 334 return nil 335 } 336 337 // pendingOrDead looks up machines with ids and returns those that do not 338 // have an instance id assigned yet, and also those that are dead. 339 func (task *provisionerTask) pendingOrDeadOrMaintain(ids []string) (pending, dead, maintain []*apiprovisioner.Machine, err error) { 340 for _, id := range ids { 341 machine, found := task.machines[id] 342 if !found { 343 logger.Infof("machine %q not found", id) 344 continue 345 } 346 var classification MachineClassification 347 classification, err = classifyMachine(machine) 348 if err != nil { 349 return // return the error 350 } 351 switch classification { 352 case Pending: 353 pending = append(pending, machine) 354 case Dead: 355 dead = append(dead, machine) 356 case Maintain: 357 maintain = append(maintain, machine) 358 } 359 } 360 logger.Tracef("pending machines: %v", pending) 361 logger.Tracef("dead machines: %v", dead) 362 return 363 } 364 365 type ClassifiableMachine interface { 366 Life() params.Life 367 InstanceId() (instance.Id, error) 368 EnsureDead() error 369 Status() (params.Status, string, error) 370 Id() string 371 } 372 373 type MachineClassification string 374 375 const ( 376 None MachineClassification = "none" 377 Pending MachineClassification = "Pending" 378 Dead MachineClassification = "Dead" 379 Maintain MachineClassification = "Maintain" 380 ) 381 382 func classifyMachine(machine ClassifiableMachine) ( 383 MachineClassification, error) { 384 switch machine.Life() { 385 case params.Dying: 386 if _, err := machine.InstanceId(); err == nil { 387 return None, nil 388 } else if !params.IsCodeNotProvisioned(err) { 389 return None, errors.Annotatef(err, "failed to load dying machine id:%s, details:%v", machine.Id(), machine) 390 } 391 logger.Infof("killing dying, unprovisioned machine %q", machine) 392 if err := machine.EnsureDead(); err != nil { 393 return None, errors.Annotatef(err, "failed to ensure machine dead id:%s, details:%v", machine.Id(), machine) 394 } 395 fallthrough 396 case params.Dead: 397 return Dead, nil 398 } 399 if instId, err := machine.InstanceId(); err != nil { 400 if !params.IsCodeNotProvisioned(err) { 401 return None, errors.Annotatef(err, "failed to load machine id:%s, details:%v", machine.Id(), machine) 402 } 403 status, _, err := machine.Status() 404 if err != nil { 405 logger.Infof("cannot get machine id:%s, details:%v, err:%v", machine.Id(), machine, err) 406 return None, nil 407 } 408 if status == params.StatusPending { 409 logger.Infof("found machine pending provisioning id:%s, details:%v", machine.Id(), machine) 410 return Pending, nil 411 } 412 } else { 413 logger.Infof("machine %s already started as instance %q", machine.Id(), instId) 414 if err != nil { 415 logger.Infof("Error fetching provisioning info") 416 } else { 417 isLxc := regexp.MustCompile(`\d+/lxc/\d+`) 418 isKvm := regexp.MustCompile(`\d+/kvm/\d+`) 419 if isLxc.MatchString(machine.Id()) || isKvm.MatchString(machine.Id()) { 420 return Maintain, nil 421 } 422 } 423 } 424 return None, nil 425 } 426 427 // findUnknownInstances finds instances which are not associated with a machine. 428 func (task *provisionerTask) findUnknownInstances(stopping []instance.Instance) ([]instance.Instance, error) { 429 // Make a copy of the instances we know about. 430 instances := make(map[instance.Id]instance.Instance) 431 for k, v := range task.instances { 432 instances[k] = v 433 } 434 435 for _, m := range task.machines { 436 instId, err := m.InstanceId() 437 switch { 438 case err == nil: 439 delete(instances, instId) 440 case params.IsCodeNotProvisioned(err): 441 case params.IsCodeNotFoundOrCodeUnauthorized(err): 442 default: 443 return nil, err 444 } 445 } 446 // Now remove all those instances that we are stopping already as we 447 // know about those and don't want to include them in the unknown list. 448 for _, inst := range stopping { 449 delete(instances, inst.Id()) 450 } 451 var unknown []instance.Instance 452 for _, inst := range instances { 453 unknown = append(unknown, inst) 454 } 455 return unknown, nil 456 } 457 458 // instancesForMachines returns a list of instance.Instance that represent 459 // the list of machines running in the provider. Missing machines are 460 // omitted from the list. 461 func (task *provisionerTask) instancesForMachines(machines []*apiprovisioner.Machine) []instance.Instance { 462 var instances []instance.Instance 463 for _, machine := range machines { 464 instId, err := machine.InstanceId() 465 if err == nil { 466 instance, found := task.instances[instId] 467 // If the instance is not found we can't stop it. 468 if found { 469 instances = append(instances, instance) 470 } 471 } 472 } 473 return instances 474 } 475 476 func (task *provisionerTask) stopInstances(instances []instance.Instance) error { 477 // Although calling StopInstance with an empty slice should produce no change in the 478 // provider, environs like dummy do not consider this a noop. 479 if len(instances) == 0 { 480 return nil 481 } 482 ids := make([]instance.Id, len(instances)) 483 for i, inst := range instances { 484 ids[i] = inst.Id() 485 } 486 if err := task.broker.StopInstances(ids...); err != nil { 487 return errors.Annotate(err, "broker failed to stop instances") 488 } 489 return nil 490 } 491 492 func (task *provisionerTask) constructInstanceConfig( 493 machine *apiprovisioner.Machine, 494 auth authentication.AuthenticationProvider, 495 pInfo *params.ProvisioningInfo, 496 ) (*instancecfg.InstanceConfig, error) { 497 498 stateInfo, apiInfo, err := auth.SetupAuthentication(machine) 499 if err != nil { 500 return nil, errors.Annotate(err, "failed to setup authentication") 501 } 502 503 // Generated a nonce for the new instance, with the format: "machine-#:UUID". 504 // The first part is a badge, specifying the tag of the machine the provisioner 505 // is running on, while the second part is a random UUID. 506 uuid, err := utils.NewUUID() 507 if err != nil { 508 return nil, errors.Annotate(err, "failed to generate a nonce for machine "+machine.Id()) 509 } 510 511 nonce := fmt.Sprintf("%s:%s", task.machineTag, uuid) 512 return instancecfg.NewInstanceConfig( 513 machine.Id(), 514 nonce, 515 task.imageStream, 516 pInfo.Series, 517 task.secureServerConnection, 518 nil, 519 stateInfo, 520 apiInfo, 521 ) 522 } 523 524 func constructStartInstanceParams( 525 machine *apiprovisioner.Machine, 526 instanceConfig *instancecfg.InstanceConfig, 527 provisioningInfo *params.ProvisioningInfo, 528 possibleTools coretools.List, 529 ) (environs.StartInstanceParams, error) { 530 531 volumes := make([]storage.VolumeParams, len(provisioningInfo.Volumes)) 532 for i, v := range provisioningInfo.Volumes { 533 volumeTag, err := names.ParseVolumeTag(v.VolumeTag) 534 if err != nil { 535 return environs.StartInstanceParams{}, errors.Trace(err) 536 } 537 if v.Attachment == nil { 538 return environs.StartInstanceParams{}, errors.Errorf("volume params missing attachment") 539 } 540 machineTag, err := names.ParseMachineTag(v.Attachment.MachineTag) 541 if err != nil { 542 return environs.StartInstanceParams{}, errors.Trace(err) 543 } 544 if machineTag != machine.Tag() { 545 return environs.StartInstanceParams{}, errors.Errorf("volume attachment params has invalid machine tag") 546 } 547 if v.Attachment.InstanceId != "" { 548 return environs.StartInstanceParams{}, errors.Errorf("volume attachment params specifies instance ID") 549 } 550 volumes[i] = storage.VolumeParams{ 551 volumeTag, 552 v.Size, 553 storage.ProviderType(v.Provider), 554 v.Attributes, 555 v.Tags, 556 &storage.VolumeAttachmentParams{ 557 AttachmentParams: storage.AttachmentParams{ 558 Machine: machineTag, 559 ReadOnly: v.Attachment.ReadOnly, 560 }, 561 Volume: volumeTag, 562 }, 563 } 564 } 565 var subnetsToZones map[network.Id][]string 566 if provisioningInfo.SubnetsToZones != nil { 567 // Convert subnet provider ids from string to network.Id. 568 subnetsToZones = make(map[network.Id][]string, len(provisioningInfo.SubnetsToZones)) 569 for providerId, zones := range provisioningInfo.SubnetsToZones { 570 subnetsToZones[network.Id(providerId)] = zones 571 } 572 } 573 574 return environs.StartInstanceParams{ 575 Constraints: provisioningInfo.Constraints, 576 Tools: possibleTools, 577 InstanceConfig: instanceConfig, 578 Placement: provisioningInfo.Placement, 579 DistributionGroup: machine.DistributionGroup, 580 Volumes: volumes, 581 SubnetsToZones: subnetsToZones, 582 }, nil 583 } 584 585 func (task *provisionerTask) maintainMachines(machines []*apiprovisioner.Machine) error { 586 for _, m := range machines { 587 logger.Infof("maintainMachines: %v", m) 588 startInstanceParams := environs.StartInstanceParams{} 589 startInstanceParams.InstanceConfig = &instancecfg.InstanceConfig{} 590 startInstanceParams.InstanceConfig.MachineId = m.Id() 591 if err := task.broker.MaintainInstance(startInstanceParams); err != nil { 592 return errors.Annotatef(err, "cannot maintain machine %v", m) 593 } 594 } 595 return nil 596 } 597 598 func (task *provisionerTask) startMachines(machines []*apiprovisioner.Machine) error { 599 for _, m := range machines { 600 601 pInfo, err := task.blockUntilProvisioned(m.ProvisioningInfo) 602 if err != nil { 603 return task.setErrorStatus("fetching provisioning info for machine %q: %v", m, err) 604 } 605 606 instanceCfg, err := task.constructInstanceConfig(m, task.auth, pInfo) 607 if err != nil { 608 return task.setErrorStatus("creating instance config for machine %q: %v", m, err) 609 } 610 611 assocProvInfoAndMachCfg(pInfo, instanceCfg) 612 613 var arch string 614 if pInfo.Constraints.Arch != nil { 615 arch = *pInfo.Constraints.Arch 616 } 617 618 possibleTools, err := task.toolsFinder.FindTools( 619 version.Current.Number, 620 pInfo.Series, 621 arch, 622 ) 623 if err != nil { 624 return task.setErrorStatus("cannot find tools for machine %q: %v", m, err) 625 } 626 627 startInstanceParams, err := constructStartInstanceParams( 628 m, 629 instanceCfg, 630 pInfo, 631 possibleTools, 632 ) 633 if err != nil { 634 return task.setErrorStatus("cannot construct params for machine %q: %v", m, err) 635 } 636 637 if err := task.startMachine(m, pInfo, startInstanceParams); err != nil { 638 return errors.Annotatef(err, "cannot start machine %v", m) 639 } 640 } 641 return nil 642 } 643 644 func (task *provisionerTask) setErrorStatus(message string, machine *apiprovisioner.Machine, err error) error { 645 logger.Errorf(message, machine, err) 646 if err1 := machine.SetStatus(params.StatusError, err.Error(), nil); err1 != nil { 647 // Something is wrong with this machine, better report it back. 648 return errors.Annotatef(err1, "cannot set error status for machine %q", machine) 649 } 650 return nil 651 } 652 653 func (task *provisionerTask) prepareNetworkAndInterfaces(networkInfo []network.InterfaceInfo) ( 654 networks []params.Network, ifaces []params.NetworkInterface, err error) { 655 if len(networkInfo) == 0 { 656 return nil, nil, nil 657 } 658 visitedNetworks := set.NewStrings() 659 for _, info := range networkInfo { 660 if !names.IsValidNetwork(info.NetworkName) { 661 return nil, nil, errors.Errorf("invalid network name %q", info.NetworkName) 662 } 663 networkTag := names.NewNetworkTag(info.NetworkName).String() 664 if !visitedNetworks.Contains(networkTag) { 665 networks = append(networks, params.Network{ 666 Tag: networkTag, 667 ProviderId: string(info.ProviderId), 668 CIDR: info.CIDR, 669 VLANTag: info.VLANTag, 670 }) 671 visitedNetworks.Add(networkTag) 672 } 673 ifaces = append(ifaces, params.NetworkInterface{ 674 InterfaceName: info.ActualInterfaceName(), 675 MACAddress: info.MACAddress, 676 NetworkTag: networkTag, 677 IsVirtual: info.IsVirtual(), 678 Disabled: info.Disabled, 679 }) 680 } 681 return networks, ifaces, nil 682 } 683 684 func (task *provisionerTask) startMachine( 685 machine *apiprovisioner.Machine, 686 provisioningInfo *params.ProvisioningInfo, 687 startInstanceParams environs.StartInstanceParams, 688 ) error { 689 690 result, err := task.broker.StartInstance(startInstanceParams) 691 if err != nil { 692 // If this is a retryable error, we retry once 693 if instance.IsRetryableCreationError(errors.Cause(err)) { 694 logger.Infof("retryable error received on start instance - retrying instance creation") 695 result, err = task.broker.StartInstance(startInstanceParams) 696 if err != nil { 697 return task.setErrorStatus("cannot start instance for machine after a retry %q: %v", machine, err) 698 } 699 } else { 700 // Set the state to error, so the machine will be skipped next 701 // time until the error is resolved, but don't return an 702 // error; just keep going with the other machines. 703 return task.setErrorStatus("cannot start instance for machine %q: %v", machine, err) 704 } 705 } 706 707 inst := result.Instance 708 hardware := result.Hardware 709 nonce := startInstanceParams.InstanceConfig.MachineNonce 710 networks, ifaces, err := task.prepareNetworkAndInterfaces(result.NetworkInfo) 711 if err != nil { 712 return task.setErrorStatus("cannot prepare network for machine %q: %v", machine, err) 713 } 714 volumes := volumesToApiserver(result.Volumes) 715 volumeAttachments := volumeAttachmentsToApiserver(result.VolumeAttachments) 716 717 // TODO(dimitern) In a newer Provisioner API version, change 718 // SetInstanceInfo or add a new method that takes and saves in 719 // state all the information available on a network.InterfaceInfo 720 // for each interface, so we can later manage interfaces 721 // dynamically at run-time. 722 err = machine.SetInstanceInfo(inst.Id(), nonce, hardware, networks, ifaces, volumes, volumeAttachments) 723 if err != nil && params.IsCodeNotImplemented(err) { 724 return fmt.Errorf("cannot provision instance %v for machine %q with networks: not implemented", inst.Id(), machine) 725 } else if err == nil { 726 logger.Infof( 727 "started machine %s as instance %s with hardware %q, networks %v, interfaces %v, volumes %v, volume attachments %v, subnets to zones %v", 728 machine, inst.Id(), hardware, 729 networks, ifaces, 730 volumes, volumeAttachments, 731 startInstanceParams.SubnetsToZones, 732 ) 733 return nil 734 } 735 // We need to stop the instance right away here, set error status and go on. 736 task.setErrorStatus("cannot register instance for machine %v: %v", machine, err) 737 if err := task.broker.StopInstances(inst.Id()); err != nil { 738 // We cannot even stop the instance, log the error and quit. 739 return errors.Annotatef(err, "cannot stop instance %q for machine %v", inst.Id(), machine) 740 } 741 return nil 742 } 743 744 type provisioningInfo struct { 745 Constraints constraints.Value 746 Series string 747 Placement string 748 InstanceConfig *instancecfg.InstanceConfig 749 SubnetsToZones map[string][]string 750 } 751 752 func assocProvInfoAndMachCfg( 753 provInfo *params.ProvisioningInfo, 754 instanceConfig *instancecfg.InstanceConfig, 755 ) *provisioningInfo { 756 757 instanceConfig.Networks = provInfo.Networks 758 instanceConfig.Tags = provInfo.Tags 759 760 if len(provInfo.Jobs) > 0 { 761 instanceConfig.Jobs = provInfo.Jobs 762 } 763 764 return &provisioningInfo{ 765 Constraints: provInfo.Constraints, 766 Series: provInfo.Series, 767 Placement: provInfo.Placement, 768 InstanceConfig: instanceConfig, 769 SubnetsToZones: provInfo.SubnetsToZones, 770 } 771 } 772 773 func volumesToApiserver(volumes []storage.Volume) []params.Volume { 774 result := make([]params.Volume, len(volumes)) 775 for i, v := range volumes { 776 result[i] = params.Volume{ 777 v.Tag.String(), 778 params.VolumeInfo{ 779 v.VolumeId, 780 v.HardwareId, 781 v.Size, 782 v.Persistent, 783 }, 784 } 785 } 786 return result 787 } 788 789 func volumeAttachmentsToApiserver(attachments []storage.VolumeAttachment) map[string]params.VolumeAttachmentInfo { 790 result := make(map[string]params.VolumeAttachmentInfo) 791 for _, a := range attachments { 792 result[a.Volume.String()] = params.VolumeAttachmentInfo{ 793 a.DeviceName, 794 a.DeviceLink, 795 a.BusAddress, 796 a.ReadOnly, 797 } 798 } 799 return result 800 } 801 802 // ProvisioningInfo is new in 1.20; wait for the API server to be 803 // upgraded so we don't spew errors on upgrade. 804 func (task *provisionerTask) blockUntilProvisioned( 805 provision func() (*params.ProvisioningInfo, error), 806 ) (*params.ProvisioningInfo, error) { 807 808 var pInfo *params.ProvisioningInfo 809 var err error 810 for { 811 if pInfo, err = provision(); err == nil { 812 break 813 } 814 if params.IsCodeNotImplemented(err) { 815 logger.Infof("waiting for state server to be upgraded") 816 select { 817 case <-task.tomb.Dying(): 818 return nil, tomb.ErrDying 819 case <-time.After(15 * time.Second): 820 continue 821 } 822 } 823 return nil, err 824 } 825 826 return pInfo, nil 827 }