github.com/leeprovoost/terraform@v0.6.10-0.20160119085442-96f3f76118e7/builtin/providers/aws/resource_aws_autoscaling_group.go (about)

     1  package aws
     2  
     3  import (
     4  	"fmt"
     5  	"log"
     6  	"strings"
     7  	"time"
     8  
     9  	"github.com/hashicorp/terraform/helper/resource"
    10  	"github.com/hashicorp/terraform/helper/schema"
    11  
    12  	"github.com/aws/aws-sdk-go/aws"
    13  	"github.com/aws/aws-sdk-go/aws/awserr"
    14  	"github.com/aws/aws-sdk-go/service/autoscaling"
    15  	"github.com/aws/aws-sdk-go/service/elb"
    16  )
    17  
    18  func resourceAwsAutoscalingGroup() *schema.Resource {
    19  	return &schema.Resource{
    20  		Create: resourceAwsAutoscalingGroupCreate,
    21  		Read:   resourceAwsAutoscalingGroupRead,
    22  		Update: resourceAwsAutoscalingGroupUpdate,
    23  		Delete: resourceAwsAutoscalingGroupDelete,
    24  
    25  		Schema: map[string]*schema.Schema{
    26  			"name": &schema.Schema{
    27  				Type:     schema.TypeString,
    28  				Optional: true,
    29  				Computed: true,
    30  				ForceNew: true,
    31  				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
    32  					// https://github.com/boto/botocore/blob/9f322b1/botocore/data/autoscaling/2011-01-01/service-2.json#L1862-L1873
    33  					value := v.(string)
    34  					if len(value) > 255 {
    35  						errors = append(errors, fmt.Errorf(
    36  							"%q cannot be longer than 255 characters", k))
    37  					}
    38  					return
    39  				},
    40  			},
    41  
    42  			"launch_configuration": &schema.Schema{
    43  				Type:     schema.TypeString,
    44  				Required: true,
    45  			},
    46  
    47  			"desired_capacity": &schema.Schema{
    48  				Type:     schema.TypeInt,
    49  				Optional: true,
    50  				Computed: true,
    51  			},
    52  
    53  			"min_elb_capacity": &schema.Schema{
    54  				Type:       schema.TypeInt,
    55  				Optional:   true,
    56  				Deprecated: "Please use 'wait_for_elb_capacity' instead.",
    57  			},
    58  
    59  			"min_size": &schema.Schema{
    60  				Type:     schema.TypeInt,
    61  				Required: true,
    62  			},
    63  
    64  			"max_size": &schema.Schema{
    65  				Type:     schema.TypeInt,
    66  				Required: true,
    67  			},
    68  
    69  			"default_cooldown": &schema.Schema{
    70  				Type:     schema.TypeInt,
    71  				Optional: true,
    72  				Computed: true,
    73  			},
    74  
    75  			"force_delete": &schema.Schema{
    76  				Type:     schema.TypeBool,
    77  				Optional: true,
    78  				Default:  false,
    79  			},
    80  
    81  			"health_check_grace_period": &schema.Schema{
    82  				Type:     schema.TypeInt,
    83  				Optional: true,
    84  				Computed: true,
    85  			},
    86  
    87  			"health_check_type": &schema.Schema{
    88  				Type:     schema.TypeString,
    89  				Optional: true,
    90  				Computed: true,
    91  			},
    92  
    93  			"availability_zones": &schema.Schema{
    94  				Type:     schema.TypeSet,
    95  				Optional: true,
    96  				Elem:     &schema.Schema{Type: schema.TypeString},
    97  				Set:      schema.HashString,
    98  			},
    99  
   100  			"placement_group": &schema.Schema{
   101  				Type:     schema.TypeString,
   102  				Optional: true,
   103  			},
   104  
   105  			"load_balancers": &schema.Schema{
   106  				Type:     schema.TypeSet,
   107  				Optional: true,
   108  				Elem:     &schema.Schema{Type: schema.TypeString},
   109  				Set:      schema.HashString,
   110  			},
   111  
   112  			"vpc_zone_identifier": &schema.Schema{
   113  				Type:     schema.TypeSet,
   114  				Optional: true,
   115  				Computed: true,
   116  				Elem:     &schema.Schema{Type: schema.TypeString},
   117  				Set:      schema.HashString,
   118  			},
   119  
   120  			"termination_policies": &schema.Schema{
   121  				Type:     schema.TypeList,
   122  				Optional: true,
   123  				Elem:     &schema.Schema{Type: schema.TypeString},
   124  			},
   125  
   126  			"wait_for_capacity_timeout": &schema.Schema{
   127  				Type:     schema.TypeString,
   128  				Optional: true,
   129  				Default:  "10m",
   130  				ValidateFunc: func(v interface{}, k string) (ws []string, errors []error) {
   131  					value := v.(string)
   132  					duration, err := time.ParseDuration(value)
   133  					if err != nil {
   134  						errors = append(errors, fmt.Errorf(
   135  							"%q cannot be parsed as a duration: %s", k, err))
   136  					}
   137  					if duration < 0 {
   138  						errors = append(errors, fmt.Errorf(
   139  							"%q must be greater than zero", k))
   140  					}
   141  					return
   142  				},
   143  			},
   144  
   145  			"wait_for_elb_capacity": &schema.Schema{
   146  				Type:     schema.TypeInt,
   147  				Optional: true,
   148  			},
   149  
   150  			"tag": autoscalingTagsSchema(),
   151  		},
   152  	}
   153  }
   154  
   155  func resourceAwsAutoscalingGroupCreate(d *schema.ResourceData, meta interface{}) error {
   156  	conn := meta.(*AWSClient).autoscalingconn
   157  
   158  	var autoScalingGroupOpts autoscaling.CreateAutoScalingGroupInput
   159  
   160  	var asgName string
   161  	if v, ok := d.GetOk("name"); ok {
   162  		asgName = v.(string)
   163  	} else {
   164  		asgName = resource.PrefixedUniqueId("tf-asg-")
   165  		d.Set("name", asgName)
   166  	}
   167  
   168  	autoScalingGroupOpts.AutoScalingGroupName = aws.String(asgName)
   169  	autoScalingGroupOpts.LaunchConfigurationName = aws.String(d.Get("launch_configuration").(string))
   170  	autoScalingGroupOpts.MinSize = aws.Int64(int64(d.Get("min_size").(int)))
   171  	autoScalingGroupOpts.MaxSize = aws.Int64(int64(d.Get("max_size").(int)))
   172  
   173  	// Availability Zones are optional if VPC Zone Identifer(s) are specified
   174  	if v, ok := d.GetOk("availability_zones"); ok && v.(*schema.Set).Len() > 0 {
   175  		autoScalingGroupOpts.AvailabilityZones = expandStringList(v.(*schema.Set).List())
   176  	}
   177  
   178  	if v, ok := d.GetOk("tag"); ok {
   179  		autoScalingGroupOpts.Tags = autoscalingTagsFromMap(
   180  			setToMapByKey(v.(*schema.Set), "key"), d.Get("name").(string))
   181  	}
   182  
   183  	if v, ok := d.GetOk("default_cooldown"); ok {
   184  		autoScalingGroupOpts.DefaultCooldown = aws.Int64(int64(v.(int)))
   185  	}
   186  
   187  	if v, ok := d.GetOk("health_check_type"); ok && v.(string) != "" {
   188  		autoScalingGroupOpts.HealthCheckType = aws.String(v.(string))
   189  	}
   190  
   191  	if v, ok := d.GetOk("desired_capacity"); ok {
   192  		autoScalingGroupOpts.DesiredCapacity = aws.Int64(int64(v.(int)))
   193  	}
   194  
   195  	if v, ok := d.GetOk("health_check_grace_period"); ok {
   196  		autoScalingGroupOpts.HealthCheckGracePeriod = aws.Int64(int64(v.(int)))
   197  	}
   198  
   199  	if v, ok := d.GetOk("placement_group"); ok {
   200  		autoScalingGroupOpts.PlacementGroup = aws.String(v.(string))
   201  	}
   202  
   203  	if v, ok := d.GetOk("load_balancers"); ok && v.(*schema.Set).Len() > 0 {
   204  		autoScalingGroupOpts.LoadBalancerNames = expandStringList(
   205  			v.(*schema.Set).List())
   206  	}
   207  
   208  	if v, ok := d.GetOk("vpc_zone_identifier"); ok && v.(*schema.Set).Len() > 0 {
   209  		autoScalingGroupOpts.VPCZoneIdentifier = expandVpcZoneIdentifiers(v.(*schema.Set).List())
   210  	}
   211  
   212  	if v, ok := d.GetOk("termination_policies"); ok && len(v.([]interface{})) > 0 {
   213  		autoScalingGroupOpts.TerminationPolicies = expandStringList(v.([]interface{}))
   214  	}
   215  
   216  	log.Printf("[DEBUG] AutoScaling Group create configuration: %#v", autoScalingGroupOpts)
   217  	_, err := conn.CreateAutoScalingGroup(&autoScalingGroupOpts)
   218  	if err != nil {
   219  		return fmt.Errorf("Error creating Autoscaling Group: %s", err)
   220  	}
   221  
   222  	d.SetId(d.Get("name").(string))
   223  	log.Printf("[INFO] AutoScaling Group ID: %s", d.Id())
   224  
   225  	if err := waitForASGCapacity(d, meta); err != nil {
   226  		return err
   227  	}
   228  
   229  	return resourceAwsAutoscalingGroupRead(d, meta)
   230  }
   231  
   232  func resourceAwsAutoscalingGroupRead(d *schema.ResourceData, meta interface{}) error {
   233  	g, err := getAwsAutoscalingGroup(d, meta)
   234  	if err != nil {
   235  		return err
   236  	}
   237  	if g == nil {
   238  		return nil
   239  	}
   240  
   241  	d.Set("availability_zones", g.AvailabilityZones)
   242  	d.Set("default_cooldown", g.DefaultCooldown)
   243  	d.Set("desired_capacity", g.DesiredCapacity)
   244  	d.Set("health_check_grace_period", g.HealthCheckGracePeriod)
   245  	d.Set("health_check_type", g.HealthCheckType)
   246  	d.Set("launch_configuration", g.LaunchConfigurationName)
   247  	d.Set("load_balancers", g.LoadBalancerNames)
   248  	d.Set("min_size", g.MinSize)
   249  	d.Set("max_size", g.MaxSize)
   250  	d.Set("placement_group", g.PlacementGroup)
   251  	d.Set("name", g.AutoScalingGroupName)
   252  	d.Set("tag", g.Tags)
   253  	d.Set("vpc_zone_identifier", strings.Split(*g.VPCZoneIdentifier, ","))
   254  	d.Set("termination_policies", g.TerminationPolicies)
   255  
   256  	return nil
   257  }
   258  
   259  func resourceAwsAutoscalingGroupUpdate(d *schema.ResourceData, meta interface{}) error {
   260  	conn := meta.(*AWSClient).autoscalingconn
   261  	shouldWaitForCapacity := false
   262  
   263  	opts := autoscaling.UpdateAutoScalingGroupInput{
   264  		AutoScalingGroupName: aws.String(d.Id()),
   265  	}
   266  
   267  	if d.HasChange("default_cooldown") {
   268  		opts.DefaultCooldown = aws.Int64(int64(d.Get("default_cooldown").(int)))
   269  	}
   270  
   271  	if d.HasChange("desired_capacity") {
   272  		opts.DesiredCapacity = aws.Int64(int64(d.Get("desired_capacity").(int)))
   273  		shouldWaitForCapacity = true
   274  	}
   275  
   276  	if d.HasChange("launch_configuration") {
   277  		opts.LaunchConfigurationName = aws.String(d.Get("launch_configuration").(string))
   278  	}
   279  
   280  	if d.HasChange("min_size") {
   281  		opts.MinSize = aws.Int64(int64(d.Get("min_size").(int)))
   282  		shouldWaitForCapacity = true
   283  	}
   284  
   285  	if d.HasChange("max_size") {
   286  		opts.MaxSize = aws.Int64(int64(d.Get("max_size").(int)))
   287  	}
   288  
   289  	if d.HasChange("health_check_grace_period") {
   290  		opts.HealthCheckGracePeriod = aws.Int64(int64(d.Get("health_check_grace_period").(int)))
   291  	}
   292  
   293  	if d.HasChange("health_check_type") {
   294  		opts.HealthCheckGracePeriod = aws.Int64(int64(d.Get("health_check_grace_period").(int)))
   295  		opts.HealthCheckType = aws.String(d.Get("health_check_type").(string))
   296  	}
   297  
   298  	if d.HasChange("vpc_zone_identifier") {
   299  		opts.VPCZoneIdentifier = expandVpcZoneIdentifiers(d.Get("vpc_zone_identifier").(*schema.Set).List())
   300  	}
   301  
   302  	if d.HasChange("availability_zones") {
   303  		if v, ok := d.GetOk("availability_zones"); ok && v.(*schema.Set).Len() > 0 {
   304  			opts.AvailabilityZones = expandStringList(d.Get("availability_zones").(*schema.Set).List())
   305  		}
   306  	}
   307  
   308  	if d.HasChange("placement_group") {
   309  		opts.PlacementGroup = aws.String(d.Get("placement_group").(string))
   310  	}
   311  
   312  	if d.HasChange("termination_policies") {
   313  		// If the termination policy is set to null, we need to explicitly set
   314  		// it back to "Default", or the API won't reset it for us.
   315  		// This means GetOk() will fail us on the zero check.
   316  		v := d.Get("termination_policies")
   317  		if len(v.([]interface{})) > 0 {
   318  			opts.TerminationPolicies = expandStringList(v.([]interface{}))
   319  		} else {
   320  			// Policies is a slice of string pointers, so build one.
   321  			// Maybe there's a better idiom for this?
   322  			log.Printf("[DEBUG] Explictly setting null termination policy to 'Default'")
   323  			pol := "Default"
   324  			s := make([]*string, 1, 1)
   325  			s[0] = &pol
   326  			opts.TerminationPolicies = s
   327  		}
   328  	}
   329  
   330  	if err := setAutoscalingTags(conn, d); err != nil {
   331  		return err
   332  	} else {
   333  		d.SetPartial("tag")
   334  	}
   335  
   336  	log.Printf("[DEBUG] AutoScaling Group update configuration: %#v", opts)
   337  	_, err := conn.UpdateAutoScalingGroup(&opts)
   338  	if err != nil {
   339  		d.Partial(true)
   340  		return fmt.Errorf("Error updating Autoscaling group: %s", err)
   341  	}
   342  
   343  	if d.HasChange("load_balancers") {
   344  
   345  		o, n := d.GetChange("load_balancers")
   346  		if o == nil {
   347  			o = new(schema.Set)
   348  		}
   349  		if n == nil {
   350  			n = new(schema.Set)
   351  		}
   352  
   353  		os := o.(*schema.Set)
   354  		ns := n.(*schema.Set)
   355  		remove := expandStringList(os.Difference(ns).List())
   356  		add := expandStringList(ns.Difference(os).List())
   357  
   358  		if len(remove) > 0 {
   359  			_, err := conn.DetachLoadBalancers(&autoscaling.DetachLoadBalancersInput{
   360  				AutoScalingGroupName: aws.String(d.Id()),
   361  				LoadBalancerNames:    remove,
   362  			})
   363  			if err != nil {
   364  				return fmt.Errorf("[WARN] Error updating Load Balancers for AutoScaling Group (%s), error: %s", d.Id(), err)
   365  			}
   366  		}
   367  
   368  		if len(add) > 0 {
   369  			_, err := conn.AttachLoadBalancers(&autoscaling.AttachLoadBalancersInput{
   370  				AutoScalingGroupName: aws.String(d.Id()),
   371  				LoadBalancerNames:    add,
   372  			})
   373  			if err != nil {
   374  				return fmt.Errorf("[WARN] Error updating Load Balancers for AutoScaling Group (%s), error: %s", d.Id(), err)
   375  			}
   376  		}
   377  	}
   378  
   379  	if shouldWaitForCapacity {
   380  		waitForASGCapacity(d, meta)
   381  	}
   382  
   383  	return resourceAwsAutoscalingGroupRead(d, meta)
   384  }
   385  
   386  func resourceAwsAutoscalingGroupDelete(d *schema.ResourceData, meta interface{}) error {
   387  	conn := meta.(*AWSClient).autoscalingconn
   388  
   389  	// Read the autoscaling group first. If it doesn't exist, we're done.
   390  	// We need the group in order to check if there are instances attached.
   391  	// If so, we need to remove those first.
   392  	g, err := getAwsAutoscalingGroup(d, meta)
   393  	if err != nil {
   394  		return err
   395  	}
   396  	if g == nil {
   397  		return nil
   398  	}
   399  	if len(g.Instances) > 0 || *g.DesiredCapacity > 0 {
   400  		if err := resourceAwsAutoscalingGroupDrain(d, meta); err != nil {
   401  			return err
   402  		}
   403  	}
   404  
   405  	log.Printf("[DEBUG] AutoScaling Group destroy: %v", d.Id())
   406  	deleteopts := autoscaling.DeleteAutoScalingGroupInput{
   407  		AutoScalingGroupName: aws.String(d.Id()),
   408  		ForceDelete:          aws.Bool(d.Get("force_delete").(bool)),
   409  	}
   410  
   411  	// We retry the delete operation to handle InUse/InProgress errors coming
   412  	// from scaling operations. We should be able to sneak in a delete in between
   413  	// scaling operations within 5m.
   414  	err = resource.Retry(5*time.Minute, func() error {
   415  		if _, err := conn.DeleteAutoScalingGroup(&deleteopts); err != nil {
   416  			if awserr, ok := err.(awserr.Error); ok {
   417  				switch awserr.Code() {
   418  				case "InvalidGroup.NotFound":
   419  					// Already gone? Sure!
   420  					return nil
   421  				case "ResourceInUse", "ScalingActivityInProgress":
   422  					// These are retryable
   423  					return awserr
   424  				}
   425  			}
   426  			// Didn't recognize the error, so shouldn't retry.
   427  			return resource.RetryError{Err: err}
   428  		}
   429  		// Successful delete
   430  		return nil
   431  	})
   432  	if err != nil {
   433  		return err
   434  	}
   435  
   436  	return resource.Retry(5*time.Minute, func() error {
   437  		if g, _ = getAwsAutoscalingGroup(d, meta); g != nil {
   438  			return fmt.Errorf("Auto Scaling Group still exists")
   439  		}
   440  		return nil
   441  	})
   442  }
   443  
   444  func getAwsAutoscalingGroup(
   445  	d *schema.ResourceData,
   446  	meta interface{}) (*autoscaling.Group, error) {
   447  	conn := meta.(*AWSClient).autoscalingconn
   448  
   449  	describeOpts := autoscaling.DescribeAutoScalingGroupsInput{
   450  		AutoScalingGroupNames: []*string{aws.String(d.Id())},
   451  	}
   452  
   453  	log.Printf("[DEBUG] AutoScaling Group describe configuration: %#v", describeOpts)
   454  	describeGroups, err := conn.DescribeAutoScalingGroups(&describeOpts)
   455  	if err != nil {
   456  		autoscalingerr, ok := err.(awserr.Error)
   457  		if ok && autoscalingerr.Code() == "InvalidGroup.NotFound" {
   458  			d.SetId("")
   459  			return nil, nil
   460  		}
   461  
   462  		return nil, fmt.Errorf("Error retrieving AutoScaling groups: %s", err)
   463  	}
   464  
   465  	// Search for the autoscaling group
   466  	for idx, asc := range describeGroups.AutoScalingGroups {
   467  		if *asc.AutoScalingGroupName == d.Id() {
   468  			return describeGroups.AutoScalingGroups[idx], nil
   469  		}
   470  	}
   471  
   472  	// ASG not found
   473  	d.SetId("")
   474  	return nil, nil
   475  }
   476  
   477  func resourceAwsAutoscalingGroupDrain(d *schema.ResourceData, meta interface{}) error {
   478  	conn := meta.(*AWSClient).autoscalingconn
   479  
   480  	if d.Get("force_delete").(bool) {
   481  		log.Printf("[DEBUG] Skipping ASG drain, force_delete was set.")
   482  		return nil
   483  	}
   484  
   485  	// First, set the capacity to zero so the group will drain
   486  	log.Printf("[DEBUG] Reducing autoscaling group capacity to zero")
   487  	opts := autoscaling.UpdateAutoScalingGroupInput{
   488  		AutoScalingGroupName: aws.String(d.Id()),
   489  		DesiredCapacity:      aws.Int64(0),
   490  		MinSize:              aws.Int64(0),
   491  		MaxSize:              aws.Int64(0),
   492  	}
   493  	if _, err := conn.UpdateAutoScalingGroup(&opts); err != nil {
   494  		return fmt.Errorf("Error setting capacity to zero to drain: %s", err)
   495  	}
   496  
   497  	// Next, wait for the autoscale group to drain
   498  	log.Printf("[DEBUG] Waiting for group to have zero instances")
   499  	return resource.Retry(10*time.Minute, func() error {
   500  		g, err := getAwsAutoscalingGroup(d, meta)
   501  		if err != nil {
   502  			return resource.RetryError{Err: err}
   503  		}
   504  		if g == nil {
   505  			return nil
   506  		}
   507  
   508  		if len(g.Instances) == 0 {
   509  			return nil
   510  		}
   511  
   512  		return fmt.Errorf("group still has %d instances", len(g.Instances))
   513  	})
   514  }
   515  
   516  // Waits for a minimum number of healthy instances to show up as healthy in the
   517  // ASG before continuing. Waits up to `waitForASGCapacityTimeout` for
   518  // "desired_capacity", or "min_size" if desired capacity is not specified.
   519  //
   520  // If "wait_for_elb_capacity" is specified, will also wait for that number of
   521  // instances to show up InService in all attached ELBs. See "Waiting for
   522  // Capacity" in docs for more discussion of the feature.
   523  func waitForASGCapacity(d *schema.ResourceData, meta interface{}) error {
   524  	wantASG := d.Get("min_size").(int)
   525  	if v := d.Get("desired_capacity").(int); v > 0 {
   526  		wantASG = v
   527  	}
   528  	wantELB := d.Get("wait_for_elb_capacity").(int)
   529  
   530  	// Covers deprecated field support
   531  	wantELB += d.Get("min_elb_capacity").(int)
   532  
   533  	wait, err := time.ParseDuration(d.Get("wait_for_capacity_timeout").(string))
   534  	if err != nil {
   535  		return err
   536  	}
   537  
   538  	if wait == 0 {
   539  		log.Printf("[DEBUG] Capacity timeout set to 0, skipping capacity waiting.")
   540  		return nil
   541  	}
   542  
   543  	log.Printf("[DEBUG] Waiting %s for capacity: %d ASG, %d ELB",
   544  		wait, wantASG, wantELB)
   545  
   546  	return resource.Retry(wait, func() error {
   547  		g, err := getAwsAutoscalingGroup(d, meta)
   548  		if err != nil {
   549  			return resource.RetryError{Err: err}
   550  		}
   551  		if g == nil {
   552  			return nil
   553  		}
   554  		lbis, err := getLBInstanceStates(g, meta)
   555  		if err != nil {
   556  			return resource.RetryError{Err: err}
   557  		}
   558  
   559  		haveASG := 0
   560  		haveELB := 0
   561  
   562  		for _, i := range g.Instances {
   563  			if i.HealthStatus == nil || i.InstanceId == nil || i.LifecycleState == nil {
   564  				continue
   565  			}
   566  
   567  			if !strings.EqualFold(*i.HealthStatus, "Healthy") {
   568  				continue
   569  			}
   570  
   571  			if !strings.EqualFold(*i.LifecycleState, "InService") {
   572  				continue
   573  			}
   574  
   575  			haveASG++
   576  
   577  			if wantELB > 0 {
   578  				inAllLbs := true
   579  				for _, states := range lbis {
   580  					state, ok := states[*i.InstanceId]
   581  					if !ok || !strings.EqualFold(state, "InService") {
   582  						inAllLbs = false
   583  					}
   584  				}
   585  				if inAllLbs {
   586  					haveELB++
   587  				}
   588  			}
   589  		}
   590  
   591  		log.Printf("[DEBUG] %q Capacity: %d/%d ASG, %d/%d ELB",
   592  			d.Id(), haveASG, wantASG, haveELB, wantELB)
   593  
   594  		if haveASG == wantASG && haveELB == wantELB {
   595  			return nil
   596  		}
   597  
   598  		return fmt.Errorf(
   599  			"Still waiting for %q instances. Current/Desired: %d/%d ASG, %d/%d ELB",
   600  			d.Id(), haveASG, wantASG, haveELB, wantELB)
   601  	})
   602  }
   603  
   604  // Returns a mapping of the instance states of all the ELBs attached to the
   605  // provided ASG.
   606  //
   607  // Nested like: lbName -> instanceId -> instanceState
   608  func getLBInstanceStates(g *autoscaling.Group, meta interface{}) (map[string]map[string]string, error) {
   609  	lbInstanceStates := make(map[string]map[string]string)
   610  	elbconn := meta.(*AWSClient).elbconn
   611  
   612  	for _, lbName := range g.LoadBalancerNames {
   613  		lbInstanceStates[*lbName] = make(map[string]string)
   614  		opts := &elb.DescribeInstanceHealthInput{LoadBalancerName: lbName}
   615  		r, err := elbconn.DescribeInstanceHealth(opts)
   616  		if err != nil {
   617  			return nil, err
   618  		}
   619  		for _, is := range r.InstanceStates {
   620  			if is.InstanceId == nil || is.State == nil {
   621  				continue
   622  			}
   623  			lbInstanceStates[*lbName][*is.InstanceId] = *is.State
   624  		}
   625  	}
   626  
   627  	return lbInstanceStates, nil
   628  }
   629  
   630  func expandVpcZoneIdentifiers(list []interface{}) *string {
   631  	strs := make([]string, len(list))
   632  	for _, s := range list {
   633  		strs = append(strs, s.(string))
   634  	}
   635  	return aws.String(strings.Join(strs, ","))
   636  }