github.com/cloud-green/juju@v0.0.0-20151002100041-a00291338d3d/provider/ec2/environ.go (about)

     1  // Copyright 2011-2014 Canonical Ltd.
     2  // Licensed under the AGPLv3, see LICENCE file for details.
     3  
     4  package ec2
     5  
     6  import (
     7  	"fmt"
     8  	"net"
     9  	"strings"
    10  	"sync"
    11  	"time"
    12  
    13  	"github.com/juju/errors"
    14  	"github.com/juju/names"
    15  	"github.com/juju/utils"
    16  	"github.com/juju/utils/arch"
    17  	"github.com/juju/utils/set"
    18  	"gopkg.in/amz.v3/aws"
    19  	"gopkg.in/amz.v3/ec2"
    20  	"gopkg.in/amz.v3/s3"
    21  
    22  	"github.com/juju/juju/cloudconfig/instancecfg"
    23  	"github.com/juju/juju/cloudconfig/providerinit"
    24  	"github.com/juju/juju/constraints"
    25  	"github.com/juju/juju/environs"
    26  	"github.com/juju/juju/environs/config"
    27  	"github.com/juju/juju/environs/imagemetadata"
    28  	"github.com/juju/juju/environs/instances"
    29  	"github.com/juju/juju/environs/simplestreams"
    30  	envstorage "github.com/juju/juju/environs/storage"
    31  	"github.com/juju/juju/environs/tags"
    32  	"github.com/juju/juju/instance"
    33  	"github.com/juju/juju/network"
    34  	"github.com/juju/juju/provider/common"
    35  	"github.com/juju/juju/state"
    36  	"github.com/juju/juju/state/multiwatcher"
    37  	"github.com/juju/juju/tools"
    38  )
    39  
    40  const (
    41  	none                        = "none"
    42  	invalidParameterValue       = "InvalidParameterValue"
    43  	privateAddressLimitExceeded = "PrivateIpAddressLimitExceeded"
    44  
    45  	// tagName is the AWS-specific tag key that populates resources'
    46  	// name columns in the console.
    47  	tagName = "Name"
    48  )
    49  
    50  // Use shortAttempt to poll for short-term events or for retrying API calls.
    51  var shortAttempt = utils.AttemptStrategy{
    52  	Total: 5 * time.Second,
    53  	Delay: 200 * time.Millisecond,
    54  }
    55  
    56  type environ struct {
    57  	common.SupportsUnitPlacementPolicy
    58  
    59  	name string
    60  
    61  	// archMutex gates access to supportedArchitectures
    62  	archMutex sync.Mutex
    63  	// supportedArchitectures caches the architectures
    64  	// for which images can be instantiated.
    65  	supportedArchitectures []string
    66  
    67  	// ecfgMutex protects the *Unlocked fields below.
    68  	ecfgMutex       sync.Mutex
    69  	ecfgUnlocked    *environConfig
    70  	ec2Unlocked     *ec2.EC2
    71  	s3Unlocked      *s3.S3
    72  	storageUnlocked envstorage.Storage
    73  
    74  	availabilityZonesMutex sync.Mutex
    75  	availabilityZones      []common.AvailabilityZone
    76  
    77  	// cachedDefaultVpc caches the id of the ec2 default vpc
    78  	cachedDefaultVpc *defaultVpc
    79  }
    80  
    81  // Ensure EC2 provider supports environs.NetworkingEnviron.
    82  var _ environs.NetworkingEnviron = (*environ)(nil)
    83  var _ simplestreams.HasRegion = (*environ)(nil)
    84  var _ state.Prechecker = (*environ)(nil)
    85  var _ state.InstanceDistributor = (*environ)(nil)
    86  
    87  type defaultVpc struct {
    88  	hasDefaultVpc bool
    89  	id            network.Id
    90  }
    91  
    92  // AssignPrivateIPAddress is a wrapper around ec2Inst.AssignPrivateIPAddresses.
    93  var AssignPrivateIPAddress = assignPrivateIPAddress
    94  
    95  // assignPrivateIPAddress should not be called directly so tests can patch it (use
    96  // AssignPrivateIPAddress).
    97  func assignPrivateIPAddress(ec2Inst *ec2.EC2, netId string, addr network.Address) error {
    98  	_, err := ec2Inst.AssignPrivateIPAddresses(netId, []string{addr.Value}, 0, false)
    99  	return err
   100  }
   101  
   102  func (e *environ) Config() *config.Config {
   103  	return e.ecfg().Config
   104  }
   105  
   106  func awsClients(cfg *config.Config) (*ec2.EC2, *s3.S3, *environConfig, error) {
   107  	ecfg, err := providerInstance.newConfig(cfg)
   108  	if err != nil {
   109  		return nil, nil, nil, err
   110  	}
   111  
   112  	auth := aws.Auth{ecfg.accessKey(), ecfg.secretKey()}
   113  	region := aws.Regions[ecfg.region()]
   114  	signer := aws.SignV4Factory(region.Name, "ec2")
   115  	return ec2.New(auth, region, signer), s3.New(auth, region), ecfg, nil
   116  }
   117  
   118  func (e *environ) SetConfig(cfg *config.Config) error {
   119  	ec2Client, s3Client, ecfg, err := awsClients(cfg)
   120  	if err != nil {
   121  		return err
   122  	}
   123  	e.ecfgMutex.Lock()
   124  	defer e.ecfgMutex.Unlock()
   125  	e.ecfgUnlocked = ecfg
   126  	e.ec2Unlocked = ec2Client
   127  	e.s3Unlocked = s3Client
   128  
   129  	bucket, err := e.s3Unlocked.Bucket(ecfg.controlBucket())
   130  	if err != nil {
   131  		return err
   132  	}
   133  
   134  	// create new storage instances, existing instances continue
   135  	// to reference their existing configuration.
   136  	e.storageUnlocked = &ec2storage{bucket: bucket}
   137  	return nil
   138  }
   139  
   140  func (e *environ) defaultVpc() (network.Id, bool, error) {
   141  	if e.cachedDefaultVpc != nil {
   142  		defaultVpc := e.cachedDefaultVpc
   143  		return defaultVpc.id, defaultVpc.hasDefaultVpc, nil
   144  	}
   145  	ec2 := e.ec2()
   146  	resp, err := ec2.AccountAttributes("default-vpc")
   147  	if err != nil {
   148  		return "", false, errors.Trace(err)
   149  	}
   150  
   151  	hasDefault := true
   152  	defaultVpcId := ""
   153  
   154  	if len(resp.Attributes) == 0 || len(resp.Attributes[0].Values) == 0 {
   155  		hasDefault = false
   156  		defaultVpcId = ""
   157  	} else {
   158  		defaultVpcId = resp.Attributes[0].Values[0]
   159  		if defaultVpcId == none {
   160  			hasDefault = false
   161  			defaultVpcId = ""
   162  		}
   163  	}
   164  	defaultVpc := &defaultVpc{
   165  		id:            network.Id(defaultVpcId),
   166  		hasDefaultVpc: hasDefault,
   167  	}
   168  	e.cachedDefaultVpc = defaultVpc
   169  	return defaultVpc.id, defaultVpc.hasDefaultVpc, nil
   170  }
   171  
   172  func (e *environ) ecfg() *environConfig {
   173  	e.ecfgMutex.Lock()
   174  	ecfg := e.ecfgUnlocked
   175  	e.ecfgMutex.Unlock()
   176  	return ecfg
   177  }
   178  
   179  func (e *environ) ec2() *ec2.EC2 {
   180  	e.ecfgMutex.Lock()
   181  	ec2 := e.ec2Unlocked
   182  	e.ecfgMutex.Unlock()
   183  	return ec2
   184  }
   185  
   186  func (e *environ) s3() *s3.S3 {
   187  	e.ecfgMutex.Lock()
   188  	s3 := e.s3Unlocked
   189  	e.ecfgMutex.Unlock()
   190  	return s3
   191  }
   192  
   193  func (e *environ) Name() string {
   194  	return e.name
   195  }
   196  
   197  func (e *environ) Storage() envstorage.Storage {
   198  	e.ecfgMutex.Lock()
   199  	stor := e.storageUnlocked
   200  	e.ecfgMutex.Unlock()
   201  	return stor
   202  }
   203  
   204  func (e *environ) Bootstrap(ctx environs.BootstrapContext, args environs.BootstrapParams) (arch, series string, _ environs.BootstrapFinalizer, _ error) {
   205  	return common.Bootstrap(ctx, e, args)
   206  }
   207  
   208  func (e *environ) StateServerInstances() ([]instance.Id, error) {
   209  	return common.ProviderStateInstances(e, e.Storage())
   210  }
   211  
   212  // SupportedArchitectures is specified on the EnvironCapability interface.
   213  func (e *environ) SupportedArchitectures() ([]string, error) {
   214  	e.archMutex.Lock()
   215  	defer e.archMutex.Unlock()
   216  	if e.supportedArchitectures != nil {
   217  		return e.supportedArchitectures, nil
   218  	}
   219  	// Create a filter to get all images from our region and for the correct stream.
   220  	cloudSpec, err := e.Region()
   221  	if err != nil {
   222  		return nil, err
   223  	}
   224  	imageConstraint := imagemetadata.NewImageConstraint(simplestreams.LookupParams{
   225  		CloudSpec: cloudSpec,
   226  		Stream:    e.Config().ImageStream(),
   227  	})
   228  	e.supportedArchitectures, err = common.SupportedArchitectures(e, imageConstraint)
   229  	return e.supportedArchitectures, err
   230  }
   231  
   232  // SupportsSpaces is specified on environs.Networking.
   233  func (e *environ) SupportsSpaces() (bool, error) {
   234  	return true, nil
   235  }
   236  
   237  // SupportsAddressAllocation is specified on environs.Networking.
   238  func (e *environ) SupportsAddressAllocation(_ network.Id) (bool, error) {
   239  	if !environs.AddressAllocationEnabled() {
   240  		return false, errors.NotSupportedf("address allocation")
   241  	}
   242  	_, hasDefaultVpc, err := e.defaultVpc()
   243  	if err != nil {
   244  		return false, errors.Trace(err)
   245  	}
   246  	return hasDefaultVpc, nil
   247  }
   248  
   249  var unsupportedConstraints = []string{
   250  	constraints.Tags,
   251  }
   252  
   253  // ConstraintsValidator is defined on the Environs interface.
   254  func (e *environ) ConstraintsValidator() (constraints.Validator, error) {
   255  	validator := constraints.NewValidator()
   256  	validator.RegisterConflicts(
   257  		[]string{constraints.InstanceType},
   258  		[]string{constraints.Mem, constraints.CpuCores, constraints.CpuPower})
   259  	validator.RegisterUnsupported(unsupportedConstraints)
   260  	supportedArches, err := e.SupportedArchitectures()
   261  	if err != nil {
   262  		return nil, err
   263  	}
   264  	validator.RegisterVocabulary(constraints.Arch, supportedArches)
   265  	instTypeNames := make([]string, len(allInstanceTypes))
   266  	for i, itype := range allInstanceTypes {
   267  		instTypeNames[i] = itype.Name
   268  	}
   269  	validator.RegisterVocabulary(constraints.InstanceType, instTypeNames)
   270  	return validator, nil
   271  }
   272  
   273  func archMatches(arches []string, arch *string) bool {
   274  	if arch == nil {
   275  		return true
   276  	}
   277  	for _, a := range arches {
   278  		if a == *arch {
   279  			return true
   280  		}
   281  	}
   282  	return false
   283  }
   284  
   285  var ec2AvailabilityZones = (*ec2.EC2).AvailabilityZones
   286  
   287  type ec2AvailabilityZone struct {
   288  	ec2.AvailabilityZoneInfo
   289  }
   290  
   291  func (z *ec2AvailabilityZone) Name() string {
   292  	return z.AvailabilityZoneInfo.Name
   293  }
   294  
   295  func (z *ec2AvailabilityZone) Available() bool {
   296  	return z.AvailabilityZoneInfo.State == "available"
   297  }
   298  
   299  // AvailabilityZones returns a slice of availability zones
   300  // for the configured region.
   301  func (e *environ) AvailabilityZones() ([]common.AvailabilityZone, error) {
   302  	e.availabilityZonesMutex.Lock()
   303  	defer e.availabilityZonesMutex.Unlock()
   304  	if e.availabilityZones == nil {
   305  		filter := ec2.NewFilter()
   306  		filter.Add("region-name", e.ecfg().region())
   307  		resp, err := ec2AvailabilityZones(e.ec2(), filter)
   308  		if err != nil {
   309  			return nil, err
   310  		}
   311  		logger.Debugf("availability zones: %+v", resp)
   312  		e.availabilityZones = make([]common.AvailabilityZone, len(resp.Zones))
   313  		for i, z := range resp.Zones {
   314  			e.availabilityZones[i] = &ec2AvailabilityZone{z}
   315  		}
   316  	}
   317  	return e.availabilityZones, nil
   318  }
   319  
   320  // InstanceAvailabilityZoneNames returns the availability zone names for each
   321  // of the specified instances.
   322  func (e *environ) InstanceAvailabilityZoneNames(ids []instance.Id) ([]string, error) {
   323  	instances, err := e.Instances(ids)
   324  	if err != nil && err != environs.ErrPartialInstances {
   325  		return nil, err
   326  	}
   327  	zones := make([]string, len(instances))
   328  	for i, inst := range instances {
   329  		if inst == nil {
   330  			continue
   331  		}
   332  		zones[i] = inst.(*ec2Instance).AvailZone
   333  	}
   334  	return zones, err
   335  }
   336  
   337  type ec2Placement struct {
   338  	availabilityZone ec2.AvailabilityZoneInfo
   339  }
   340  
   341  func (e *environ) parsePlacement(placement string) (*ec2Placement, error) {
   342  	pos := strings.IndexRune(placement, '=')
   343  	if pos == -1 {
   344  		return nil, fmt.Errorf("unknown placement directive: %v", placement)
   345  	}
   346  	switch key, value := placement[:pos], placement[pos+1:]; key {
   347  	case "zone":
   348  		availabilityZone := value
   349  		zones, err := e.AvailabilityZones()
   350  		if err != nil {
   351  			return nil, err
   352  		}
   353  		for _, z := range zones {
   354  			if z.Name() == availabilityZone {
   355  				return &ec2Placement{
   356  					z.(*ec2AvailabilityZone).AvailabilityZoneInfo,
   357  				}, nil
   358  			}
   359  		}
   360  		return nil, fmt.Errorf("invalid availability zone %q", availabilityZone)
   361  	}
   362  	return nil, fmt.Errorf("unknown placement directive: %v", placement)
   363  }
   364  
   365  // PrecheckInstance is defined on the state.Prechecker interface.
   366  func (e *environ) PrecheckInstance(series string, cons constraints.Value, placement string) error {
   367  	if placement != "" {
   368  		if _, err := e.parsePlacement(placement); err != nil {
   369  			return err
   370  		}
   371  	}
   372  	if !cons.HasInstanceType() {
   373  		return nil
   374  	}
   375  	// Constraint has an instance-type constraint so let's see if it is valid.
   376  	for _, itype := range allInstanceTypes {
   377  		if itype.Name != *cons.InstanceType {
   378  			continue
   379  		}
   380  		if archMatches(itype.Arches, cons.Arch) {
   381  			return nil
   382  		}
   383  	}
   384  	if cons.Arch == nil {
   385  		return fmt.Errorf("invalid AWS instance type %q specified", *cons.InstanceType)
   386  	}
   387  	return fmt.Errorf("invalid AWS instance type %q and arch %q specified", *cons.InstanceType, *cons.Arch)
   388  }
   389  
   390  // MetadataLookupParams returns parameters which are used to query simplestreams metadata.
   391  func (e *environ) MetadataLookupParams(region string) (*simplestreams.MetadataLookupParams, error) {
   392  	if region == "" {
   393  		region = e.ecfg().region()
   394  	}
   395  	cloudSpec, err := e.cloudSpec(region)
   396  	if err != nil {
   397  		return nil, err
   398  	}
   399  	return &simplestreams.MetadataLookupParams{
   400  		Series:        config.PreferredSeries(e.ecfg()),
   401  		Region:        cloudSpec.Region,
   402  		Endpoint:      cloudSpec.Endpoint,
   403  		Architectures: arch.AllSupportedArches,
   404  	}, nil
   405  }
   406  
   407  // Region is specified in the HasRegion interface.
   408  func (e *environ) Region() (simplestreams.CloudSpec, error) {
   409  	return e.cloudSpec(e.ecfg().region())
   410  }
   411  
   412  func (e *environ) cloudSpec(region string) (simplestreams.CloudSpec, error) {
   413  	ec2Region, ok := allRegions[region]
   414  	if !ok {
   415  		return simplestreams.CloudSpec{}, fmt.Errorf("unknown region %q", region)
   416  	}
   417  	return simplestreams.CloudSpec{
   418  		Region:   region,
   419  		Endpoint: ec2Region.EC2Endpoint,
   420  	}, nil
   421  }
   422  
   423  const (
   424  	ebsStorage = "ebs"
   425  	ssdStorage = "ssd"
   426  )
   427  
   428  // DistributeInstances implements the state.InstanceDistributor policy.
   429  func (e *environ) DistributeInstances(candidates, distributionGroup []instance.Id) ([]instance.Id, error) {
   430  	return common.DistributeInstances(e, candidates, distributionGroup)
   431  }
   432  
   433  var availabilityZoneAllocations = common.AvailabilityZoneAllocations
   434  
   435  // MaintainInstance is specified in the InstanceBroker interface.
   436  func (*environ) MaintainInstance(args environs.StartInstanceParams) error {
   437  	return nil
   438  }
   439  
   440  // resourceName returns the string to use for a resource's Name tag,
   441  // to help users identify Juju-managed resources in the AWS console.
   442  func resourceName(tag names.Tag, envName string) string {
   443  	return fmt.Sprintf("juju-%s-%s", envName, tag)
   444  }
   445  
   446  // StartInstance is specified in the InstanceBroker interface.
   447  func (e *environ) StartInstance(args environs.StartInstanceParams) (_ *environs.StartInstanceResult, resultErr error) {
   448  	var inst *ec2Instance
   449  	defer func() {
   450  		if resultErr == nil || inst == nil {
   451  			return
   452  		}
   453  		if err := e.StopInstances(inst.Id()); err != nil {
   454  			logger.Errorf("error stopping failed instance: %v", err)
   455  		}
   456  	}()
   457  
   458  	var availabilityZones []string
   459  	if args.Placement != "" {
   460  		placement, err := e.parsePlacement(args.Placement)
   461  		if err != nil {
   462  			return nil, err
   463  		}
   464  		if placement.availabilityZone.State != "available" {
   465  			return nil, errors.Errorf("availability zone %q is %s", placement.availabilityZone.Name, placement.availabilityZone.State)
   466  		}
   467  		availabilityZones = append(availabilityZones, placement.availabilityZone.Name)
   468  	}
   469  
   470  	// If no availability zone is specified, then automatically spread across
   471  	// the known zones for optimal spread across the instance distribution
   472  	// group.
   473  	if len(availabilityZones) == 0 {
   474  		var group []instance.Id
   475  		var err error
   476  		if args.DistributionGroup != nil {
   477  			group, err = args.DistributionGroup()
   478  			if err != nil {
   479  				return nil, err
   480  			}
   481  		}
   482  		zoneInstances, err := availabilityZoneAllocations(e, group)
   483  		if err != nil {
   484  			return nil, err
   485  		}
   486  		for _, z := range zoneInstances {
   487  			availabilityZones = append(availabilityZones, z.ZoneName)
   488  		}
   489  		if len(availabilityZones) == 0 {
   490  			return nil, errors.New("failed to determine availability zones")
   491  		}
   492  	}
   493  
   494  	// If spaces= constraints is also set, then filter availabilityZones to only
   495  	// contain zones matching the space's subnets' zones
   496  	if len(args.SubnetsToZones) > 0 {
   497  		// find all the available zones that match the subnets that match our
   498  		// space/subnet constraints
   499  		zonesSet := set.NewStrings(availabilityZones...)
   500  		subnetZones := set.NewStrings()
   501  
   502  		for _, zones := range args.SubnetsToZones {
   503  			for _, zone := range zones {
   504  				subnetZones.Add(zone)
   505  			}
   506  		}
   507  
   508  		if len(zonesSet.Intersection(subnetZones).SortedValues()) == 0 {
   509  			return nil, errors.Errorf(
   510  				"unable to resolve constraints: space and/or subnet unavailable in zones %v",
   511  				availabilityZones)
   512  		}
   513  
   514  		availabilityZones = zonesSet.Intersection(subnetZones).SortedValues()
   515  	}
   516  
   517  	if args.InstanceConfig.HasNetworks() {
   518  		return nil, errors.New("starting instances with networks is not supported yet")
   519  	}
   520  	arches := args.Tools.Arches()
   521  	sources, err := environs.ImageMetadataSources(e)
   522  	if err != nil {
   523  		return nil, err
   524  	}
   525  
   526  	series := args.Tools.OneSeries()
   527  	spec, err := findInstanceSpec(sources, e.Config().ImageStream(), &instances.InstanceConstraint{
   528  		Region:      e.ecfg().region(),
   529  		Series:      series,
   530  		Arches:      arches,
   531  		Constraints: args.Constraints,
   532  		Storage:     []string{ssdStorage, ebsStorage},
   533  	})
   534  	if err != nil {
   535  		return nil, err
   536  	}
   537  	tools, err := args.Tools.Match(tools.Filter{Arch: spec.Image.Arch})
   538  	if err != nil {
   539  		return nil, errors.Errorf("chosen architecture %v not present in %v", spec.Image.Arch, arches)
   540  	}
   541  
   542  	args.InstanceConfig.Tools = tools[0]
   543  	if err := instancecfg.FinishInstanceConfig(args.InstanceConfig, e.Config()); err != nil {
   544  		return nil, err
   545  	}
   546  
   547  	userData, err := providerinit.ComposeUserData(args.InstanceConfig, nil, AmazonRenderer{})
   548  	if err != nil {
   549  		return nil, errors.Annotate(err, "cannot make user data")
   550  	}
   551  	logger.Debugf("ec2 user data; %d bytes", len(userData))
   552  	cfg := e.Config()
   553  	groups, err := e.setUpGroups(args.InstanceConfig.MachineId, cfg.APIPort())
   554  	if err != nil {
   555  		return nil, errors.Annotate(err, "cannot set up groups")
   556  	}
   557  	var instResp *ec2.RunInstancesResp
   558  
   559  	blockDeviceMappings, err := getBlockDeviceMappings(args.Constraints)
   560  	if err != nil {
   561  		return nil, errors.Annotate(err, "cannot create block device mappings")
   562  	}
   563  	rootDiskSize := uint64(blockDeviceMappings[0].VolumeSize) * 1024
   564  
   565  	for _, availZone := range availabilityZones {
   566  		instResp, err = runInstances(e.ec2(), &ec2.RunInstances{
   567  			AvailZone: availZone,
   568  			// TODO: SubnetId: <a subnet in the AZ that conforms to our constraints>
   569  			ImageId:             spec.Image.Id,
   570  			MinCount:            1,
   571  			MaxCount:            1,
   572  			UserData:            userData,
   573  			InstanceType:        spec.InstanceType.Name,
   574  			SecurityGroups:      groups,
   575  			BlockDeviceMappings: blockDeviceMappings,
   576  		})
   577  		if isZoneConstrainedError(err) {
   578  			logger.Infof("%q is constrained, trying another availability zone", availZone)
   579  		} else {
   580  			break
   581  		}
   582  	}
   583  	if err != nil {
   584  		return nil, errors.Annotate(err, "cannot run instances")
   585  	}
   586  	if len(instResp.Instances) != 1 {
   587  		return nil, errors.Errorf("expected 1 started instance, got %d", len(instResp.Instances))
   588  	}
   589  
   590  	inst = &ec2Instance{
   591  		e:        e,
   592  		Instance: &instResp.Instances[0],
   593  	}
   594  	logger.Infof("started instance %q in %q", inst.Id(), inst.Instance.AvailZone)
   595  
   596  	// Tag instance, for accounting and identification.
   597  	instanceName := resourceName(
   598  		names.NewMachineTag(args.InstanceConfig.MachineId), e.Config().Name(),
   599  	)
   600  	args.InstanceConfig.Tags[tagName] = instanceName
   601  	if err := tagResources(e.ec2(), args.InstanceConfig.Tags, string(inst.Id())); err != nil {
   602  		return nil, errors.Annotate(err, "tagging instance")
   603  	}
   604  
   605  	// Tag the machine's root EBS volume, if it has one.
   606  	if inst.Instance.RootDeviceType == "ebs" {
   607  		uuid, _ := cfg.UUID()
   608  		tags := tags.ResourceTags(names.NewEnvironTag(uuid), cfg)
   609  		tags[tagName] = instanceName + "-root"
   610  		if err := tagRootDisk(e.ec2(), tags, inst.Instance); err != nil {
   611  			return nil, errors.Annotate(err, "tagging root disk")
   612  		}
   613  	}
   614  
   615  	if multiwatcher.AnyJobNeedsState(args.InstanceConfig.Jobs...) {
   616  		if err := common.AddStateInstance(e.Storage(), inst.Id()); err != nil {
   617  			return nil, errors.Annotate(err, "recording instance in provider-state")
   618  		}
   619  	}
   620  
   621  	hc := instance.HardwareCharacteristics{
   622  		Arch:     &spec.Image.Arch,
   623  		Mem:      &spec.InstanceType.Mem,
   624  		CpuCores: &spec.InstanceType.CpuCores,
   625  		CpuPower: spec.InstanceType.CpuPower,
   626  		RootDisk: &rootDiskSize,
   627  		// Tags currently not supported by EC2
   628  		AvailabilityZone: &inst.Instance.AvailZone,
   629  	}
   630  	return &environs.StartInstanceResult{
   631  		Instance: inst,
   632  		Hardware: &hc,
   633  	}, nil
   634  }
   635  
   636  // tagResources calls ec2.CreateTags, tagging each of the specified resources
   637  // with the given tags. tagResources will retry for a short period of time
   638  // if it receives a *.NotFound error response from EC2.
   639  func tagResources(e *ec2.EC2, tags map[string]string, resourceIds ...string) error {
   640  	if len(tags) == 0 {
   641  		return nil
   642  	}
   643  	ec2Tags := make([]ec2.Tag, 0, len(tags))
   644  	for k, v := range tags {
   645  		ec2Tags = append(ec2Tags, ec2.Tag{k, v})
   646  	}
   647  	var err error
   648  	for a := shortAttempt.Start(); a.Next(); {
   649  		_, err = e.CreateTags(resourceIds, ec2Tags)
   650  		if err == nil || !strings.HasSuffix(ec2ErrCode(err), ".NotFound") {
   651  			return err
   652  		}
   653  	}
   654  	return err
   655  }
   656  
   657  func tagRootDisk(e *ec2.EC2, tags map[string]string, inst *ec2.Instance) error {
   658  	if len(tags) == 0 {
   659  		return nil
   660  	}
   661  	findVolumeId := func(inst *ec2.Instance) string {
   662  		for _, m := range inst.BlockDeviceMappings {
   663  			if m.DeviceName != inst.RootDeviceName {
   664  				continue
   665  			}
   666  			return m.VolumeId
   667  		}
   668  		return ""
   669  	}
   670  	// Wait until the instance has an associated EBS volume in the
   671  	// block-device-mapping.
   672  	volumeId := findVolumeId(inst)
   673  	waitRootDiskAttempt := utils.AttemptStrategy{
   674  		Total: 5 * time.Minute,
   675  		Delay: 5 * time.Second,
   676  	}
   677  	for a := waitRootDiskAttempt.Start(); volumeId == "" && a.Next(); {
   678  		resp, err := e.Instances([]string{inst.InstanceId}, nil)
   679  		if err != nil {
   680  			return err
   681  		}
   682  		if len(resp.Reservations) > 0 && len(resp.Reservations[0].Instances) > 0 {
   683  			inst = &resp.Reservations[0].Instances[0]
   684  			volumeId = findVolumeId(inst)
   685  		}
   686  	}
   687  	if volumeId == "" {
   688  		return errors.New("timed out waiting for EBS volume to be associated")
   689  	}
   690  	return tagResources(e, tags, volumeId)
   691  }
   692  
   693  var runInstances = _runInstances
   694  
   695  // runInstances calls ec2.RunInstances for a fixed number of attempts until
   696  // RunInstances returns an error code that does not indicate an error that
   697  // may be caused by eventual consistency.
   698  func _runInstances(e *ec2.EC2, ri *ec2.RunInstances) (resp *ec2.RunInstancesResp, err error) {
   699  	for a := shortAttempt.Start(); a.Next(); {
   700  		resp, err = e.RunInstances(ri)
   701  		if err == nil || ec2ErrCode(err) != "InvalidGroup.NotFound" {
   702  			break
   703  		}
   704  	}
   705  	return resp, err
   706  }
   707  
   708  func (e *environ) StopInstances(ids ...instance.Id) error {
   709  	if err := e.terminateInstances(ids); err != nil {
   710  		return errors.Trace(err)
   711  	}
   712  	return common.RemoveStateInstances(e.Storage(), ids...)
   713  }
   714  
   715  // groupInfoByName returns information on the security group
   716  // with the given name including rules and other details.
   717  func (e *environ) groupInfoByName(groupName string) (ec2.SecurityGroupInfo, error) {
   718  	// Non-default VPC does not support name-based group lookups, can
   719  	// use a filter by group name instead when support is needed.
   720  	limitToGroups := []ec2.SecurityGroup{{Name: groupName}}
   721  	resp, err := e.ec2().SecurityGroups(limitToGroups, nil)
   722  	if err != nil {
   723  		return ec2.SecurityGroupInfo{}, err
   724  	}
   725  	if len(resp.Groups) != 1 {
   726  		return ec2.SecurityGroupInfo{}, fmt.Errorf("expected one security group named %q, got %v", groupName, resp.Groups)
   727  	}
   728  	return resp.Groups[0], nil
   729  }
   730  
   731  // groupByName returns the security group with the given name.
   732  func (e *environ) groupByName(groupName string) (ec2.SecurityGroup, error) {
   733  	groupInfo, err := e.groupInfoByName(groupName)
   734  	return groupInfo.SecurityGroup, err
   735  }
   736  
   737  // addGroupFilter sets a limit an instance filter so only those machines
   738  // with the juju environment wide security group associated will be listed.
   739  //
   740  // An EC2 API call is required to resolve the group name to an id, as VPC
   741  // enabled accounts do not support name based filtering.
   742  // TODO: Detect classic accounts and just filter by name for those.
   743  //
   744  // Callers must handle InvalidGroup.NotFound errors to mean the same as no
   745  // matching instances.
   746  func (e *environ) addGroupFilter(filter *ec2.Filter) error {
   747  	groupName := e.jujuGroupName()
   748  	group, err := e.groupByName(groupName)
   749  	if err != nil {
   750  		return err
   751  	}
   752  	// EC2 should support filtering with and without the 'instance.'
   753  	// prefix, but only the form with seems to work with default VPC.
   754  	filter.Add("instance.group-id", group.Id)
   755  	return nil
   756  }
   757  
   758  // gatherInstances tries to get information on each instance
   759  // id whose corresponding insts slot is nil.
   760  // It returns environs.ErrPartialInstances if the insts
   761  // slice has not been completely filled.
   762  func (e *environ) gatherInstances(ids []instance.Id, insts []instance.Instance) error {
   763  	var need []string
   764  	for i, inst := range insts {
   765  		if inst == nil {
   766  			need = append(need, string(ids[i]))
   767  		}
   768  	}
   769  	if len(need) == 0 {
   770  		return nil
   771  	}
   772  	filter := ec2.NewFilter()
   773  	filter.Add("instance-state-name", "pending", "running")
   774  	err := e.addGroupFilter(filter)
   775  	if err != nil {
   776  		if ec2ErrCode(err) == "InvalidGroup.NotFound" {
   777  			return environs.ErrPartialInstances
   778  		}
   779  		return err
   780  	}
   781  	filter.Add("instance-id", need...)
   782  	resp, err := e.ec2().Instances(nil, filter)
   783  	if err != nil {
   784  		return err
   785  	}
   786  	n := 0
   787  	// For each requested id, add it to the returned instances
   788  	// if we find it in the response.
   789  	for i, id := range ids {
   790  		if insts[i] != nil {
   791  			continue
   792  		}
   793  		for j := range resp.Reservations {
   794  			r := &resp.Reservations[j]
   795  			for k := range r.Instances {
   796  				if r.Instances[k].InstanceId == string(id) {
   797  					inst := r.Instances[k]
   798  					// TODO(wallyworld): lookup the details to fill in the instance type data
   799  					insts[i] = &ec2Instance{e: e, Instance: &inst}
   800  					n++
   801  				}
   802  			}
   803  		}
   804  	}
   805  	if n < len(ids) {
   806  		return environs.ErrPartialInstances
   807  	}
   808  	return nil
   809  }
   810  
   811  func (e *environ) Instances(ids []instance.Id) ([]instance.Instance, error) {
   812  	if len(ids) == 0 {
   813  		return nil, nil
   814  	}
   815  	insts := make([]instance.Instance, len(ids))
   816  	// Make a series of requests to cope with eventual consistency.
   817  	// Each request will attempt to add more instances to the requested
   818  	// set.
   819  	var err error
   820  	for a := shortAttempt.Start(); a.Next(); {
   821  		err = e.gatherInstances(ids, insts)
   822  		if err == nil || err != environs.ErrPartialInstances {
   823  			break
   824  		}
   825  	}
   826  	if err == environs.ErrPartialInstances {
   827  		for _, inst := range insts {
   828  			if inst != nil {
   829  				return insts, environs.ErrPartialInstances
   830  			}
   831  		}
   832  		return nil, environs.ErrNoInstances
   833  	}
   834  	if err != nil {
   835  		return nil, err
   836  	}
   837  	return insts, nil
   838  }
   839  
   840  func (e *environ) fetchNetworkInterfaceId(ec2Inst *ec2.EC2, instId instance.Id) (string, error) {
   841  	var err error
   842  	var instancesResp *ec2.InstancesResp
   843  	for a := shortAttempt.Start(); a.Next(); {
   844  		instancesResp, err = ec2Inst.Instances([]string{string(instId)}, nil)
   845  		if err == nil {
   846  			break
   847  		}
   848  		logger.Tracef("Instances(%q) returned: %v", instId, err)
   849  	}
   850  	if err != nil {
   851  		// either the instance doesn't exist or we couldn't get through to
   852  		// the ec2 api
   853  		return "", err
   854  	}
   855  
   856  	if len(instancesResp.Reservations) == 0 {
   857  		return "", errors.New("unexpected AWS response: reservation not found")
   858  	}
   859  	if len(instancesResp.Reservations[0].Instances) == 0 {
   860  		return "", errors.New("unexpected AWS response: instance not found")
   861  	}
   862  	if len(instancesResp.Reservations[0].Instances[0].NetworkInterfaces) == 0 {
   863  		return "", errors.New("unexpected AWS response: network interface not found")
   864  	}
   865  	networkInterfaceId := instancesResp.Reservations[0].Instances[0].NetworkInterfaces[0].Id
   866  	return networkInterfaceId, nil
   867  }
   868  
   869  // AllocateAddress requests an address to be allocated for the given
   870  // instance on the given subnet. Implements NetworkingEnviron.AllocateAddress.
   871  func (e *environ) AllocateAddress(instId instance.Id, _ network.Id, addr network.Address, _, _ string) (err error) {
   872  	if !environs.AddressAllocationEnabled() {
   873  		return errors.NotSupportedf("address allocation")
   874  	}
   875  
   876  	defer errors.DeferredAnnotatef(&err, "failed to allocate address %q for instance %q", addr, instId)
   877  
   878  	var nicId string
   879  	ec2Inst := e.ec2()
   880  	nicId, err = e.fetchNetworkInterfaceId(ec2Inst, instId)
   881  	if err != nil {
   882  		return errors.Trace(err)
   883  	}
   884  	for a := shortAttempt.Start(); a.Next(); {
   885  		err = AssignPrivateIPAddress(ec2Inst, nicId, addr)
   886  		logger.Tracef("AssignPrivateIPAddresses(%v, %v) returned: %v", nicId, addr, err)
   887  		if err == nil {
   888  			logger.Tracef("allocated address %v for instance %v, NIC %v", addr, instId, nicId)
   889  			break
   890  		}
   891  		if ec2Err, ok := err.(*ec2.Error); ok {
   892  			if ec2Err.Code == invalidParameterValue {
   893  				// Note: this Code is also used if we specify
   894  				// an IP address outside the subnet. Take care!
   895  				logger.Tracef("address %q not available for allocation", addr)
   896  				return environs.ErrIPAddressUnavailable
   897  			} else if ec2Err.Code == privateAddressLimitExceeded {
   898  				logger.Tracef("no more addresses available on the subnet")
   899  				return environs.ErrIPAddressesExhausted
   900  			}
   901  		}
   902  
   903  	}
   904  	return err
   905  }
   906  
   907  // ReleaseAddress releases a specific address previously allocated with
   908  // AllocateAddress. Implements NetworkingEnviron.ReleaseAddress.
   909  func (e *environ) ReleaseAddress(instId instance.Id, _ network.Id, addr network.Address, _ string) (err error) {
   910  	if !environs.AddressAllocationEnabled() {
   911  		return errors.NotSupportedf("address allocation")
   912  	}
   913  
   914  	defer errors.DeferredAnnotatef(&err, "failed to release address %q from instance %q", addr, instId)
   915  
   916  	// If the instance ID is unknown the address has already been released
   917  	// and we can ignore this request.
   918  	if instId == instance.UnknownId {
   919  		logger.Debugf("release address %q with an unknown instance ID is a no-op (ignoring)", addr.Value)
   920  		return nil
   921  	}
   922  
   923  	var nicId string
   924  	ec2Inst := e.ec2()
   925  	nicId, err = e.fetchNetworkInterfaceId(ec2Inst, instId)
   926  	if err != nil {
   927  		return errors.Trace(err)
   928  	}
   929  	for a := shortAttempt.Start(); a.Next(); {
   930  		_, err = ec2Inst.UnassignPrivateIPAddresses(nicId, []string{addr.Value})
   931  		logger.Tracef("UnassignPrivateIPAddresses(%q, %q) returned: %v", nicId, addr, err)
   932  		if err == nil {
   933  			logger.Tracef("released address %q from instance %q, NIC %q", addr, instId, nicId)
   934  			break
   935  		}
   936  	}
   937  	return err
   938  }
   939  
   940  // NetworkInterfaces implements NetworkingEnviron.NetworkInterfaces.
   941  func (e *environ) NetworkInterfaces(instId instance.Id) ([]network.InterfaceInfo, error) {
   942  	ec2Client := e.ec2()
   943  	var err error
   944  	var networkInterfacesResp *ec2.NetworkInterfacesResp
   945  	for a := shortAttempt.Start(); a.Next(); {
   946  		logger.Tracef("retrieving NICs for instance %q", instId)
   947  		filter := ec2.NewFilter()
   948  		filter.Add("attachment.instance-id", string(instId))
   949  		networkInterfacesResp, err = ec2Client.NetworkInterfaces(nil, filter)
   950  		logger.Tracef("instance %q NICs: %#v (err: %v)", instId, networkInterfacesResp, err)
   951  		if err != nil {
   952  			logger.Warningf("failed to get instance %q interfaces: %v (retrying)", instId, err)
   953  			continue
   954  		}
   955  		if len(networkInterfacesResp.Interfaces) == 0 {
   956  			logger.Tracef("instance %q has no NIC attachment yet, retrying...", instId)
   957  			continue
   958  		}
   959  		logger.Tracef("found instance %q NICS: %#v", instId, networkInterfacesResp.Interfaces)
   960  		break
   961  	}
   962  	if err != nil {
   963  		// either the instance doesn't exist or we couldn't get through to
   964  		// the ec2 api
   965  		return nil, errors.Annotatef(err, "cannot get instance %q network interfaces", instId)
   966  	}
   967  	ec2Interfaces := networkInterfacesResp.Interfaces
   968  	result := make([]network.InterfaceInfo, len(ec2Interfaces))
   969  	for i, iface := range ec2Interfaces {
   970  		resp, err := ec2Client.Subnets([]string{iface.SubnetId}, nil)
   971  		if err != nil {
   972  			return nil, errors.Annotatef(err, "failed to retrieve subnet %q info", iface.SubnetId)
   973  		}
   974  		if len(resp.Subnets) != 1 {
   975  			return nil, errors.Errorf("expected 1 subnet, got %d", len(resp.Subnets))
   976  		}
   977  		subnet := resp.Subnets[0]
   978  		cidr := subnet.CIDRBlock
   979  
   980  		result[i] = network.InterfaceInfo{
   981  			DeviceIndex:       iface.Attachment.DeviceIndex,
   982  			MACAddress:        iface.MACAddress,
   983  			CIDR:              cidr,
   984  			NetworkName:       "", // Not needed for now.
   985  			ProviderId:        network.Id(iface.Id),
   986  			ProviderSubnetId:  network.Id(iface.SubnetId),
   987  			AvailabilityZones: []string{subnet.AvailZone},
   988  			VLANTag:           0, // Not supported on EC2.
   989  			// Getting the interface name is not supported on EC2, so fake it.
   990  			InterfaceName: fmt.Sprintf("unsupported%d", iface.Attachment.DeviceIndex),
   991  			Disabled:      false,
   992  			NoAutoStart:   false,
   993  			ConfigType:    network.ConfigDHCP,
   994  			Address:       network.NewScopedAddress(iface.PrivateIPAddress, network.ScopeCloudLocal),
   995  		}
   996  	}
   997  	return result, nil
   998  }
   999  
  1000  func makeSubnetInfo(cidr string, subnetId network.Id, availZones []string) (network.SubnetInfo, error) {
  1001  	ip, ipnet, err := net.ParseCIDR(cidr)
  1002  	if err != nil {
  1003  		logger.Warningf("skipping subnet %q, invalid CIDR: %v", cidr, err)
  1004  		return network.SubnetInfo{}, err
  1005  	}
  1006  	// ec2 only uses IPv4 addresses for subnets
  1007  	start, err := network.IPv4ToDecimal(ip)
  1008  	if err != nil {
  1009  		logger.Warningf("skipping subnet %q, invalid IP: %v", cidr, err)
  1010  		return network.SubnetInfo{}, err
  1011  	}
  1012  	// First four addresses in a subnet are reserved, see
  1013  	// http://goo.gl/rrWTIo
  1014  	allocatableLow := network.DecimalToIPv4(start + 4)
  1015  
  1016  	ones, bits := ipnet.Mask.Size()
  1017  	zeros := bits - ones
  1018  	numIPs := uint32(1) << uint32(zeros)
  1019  	highIP := start + numIPs - 1
  1020  	// The last address in a subnet is also reserved (see same ref).
  1021  	allocatableHigh := network.DecimalToIPv4(highIP - 1)
  1022  
  1023  	info := network.SubnetInfo{
  1024  		CIDR:              cidr,
  1025  		ProviderId:        subnetId,
  1026  		VLANTag:           0, // Not supported on EC2
  1027  		AllocatableIPLow:  allocatableLow,
  1028  		AllocatableIPHigh: allocatableHigh,
  1029  		AvailabilityZones: availZones,
  1030  	}
  1031  	logger.Tracef("found subnet with info %#v", info)
  1032  	return info, nil
  1033  
  1034  }
  1035  
  1036  // Subnets returns basic information about the specified subnets known
  1037  // by the provider for the specified instance or list of ids. subnetIds can be
  1038  // empty, in which case all known are returned. Implements
  1039  // NetworkingEnviron.Subnets.
  1040  func (e *environ) Subnets(instId instance.Id, subnetIds []network.Id) ([]network.SubnetInfo, error) {
  1041  	var results []network.SubnetInfo
  1042  	subIdSet := make(map[string]bool)
  1043  	for _, subId := range subnetIds {
  1044  		subIdSet[string(subId)] = false
  1045  	}
  1046  
  1047  	if instId != instance.UnknownId {
  1048  		interfaces, err := e.NetworkInterfaces(instId)
  1049  		if err != nil {
  1050  			return results, errors.Trace(err)
  1051  		}
  1052  		if len(subnetIds) == 0 {
  1053  			for _, iface := range interfaces {
  1054  				subIdSet[string(iface.ProviderSubnetId)] = false
  1055  			}
  1056  		}
  1057  		for _, iface := range interfaces {
  1058  			_, ok := subIdSet[string(iface.ProviderSubnetId)]
  1059  			if !ok {
  1060  				logger.Tracef("subnet %q not in %v, skipping", iface.ProviderSubnetId, subnetIds)
  1061  				continue
  1062  			}
  1063  			subIdSet[string(iface.ProviderSubnetId)] = true
  1064  			info, err := makeSubnetInfo(iface.CIDR, iface.ProviderSubnetId, iface.AvailabilityZones)
  1065  			if err != nil {
  1066  				// Error will already have been logged.
  1067  				continue
  1068  			}
  1069  			results = append(results, info)
  1070  		}
  1071  	} else {
  1072  		ec2Inst := e.ec2()
  1073  		resp, err := ec2Inst.Subnets(nil, nil)
  1074  		if err != nil {
  1075  			return nil, errors.Annotatef(err, "failed to retrieve subnets")
  1076  		}
  1077  		if len(subnetIds) == 0 {
  1078  			for _, subnet := range resp.Subnets {
  1079  				subIdSet[subnet.Id] = false
  1080  			}
  1081  		}
  1082  
  1083  		for _, subnet := range resp.Subnets {
  1084  			_, ok := subIdSet[subnet.Id]
  1085  			if !ok {
  1086  				logger.Tracef("subnet %q not in %v, skipping", subnet.Id, subnetIds)
  1087  				continue
  1088  			}
  1089  			subIdSet[subnet.Id] = true
  1090  			cidr := subnet.CIDRBlock
  1091  			info, err := makeSubnetInfo(cidr, network.Id(subnet.Id), []string{subnet.AvailZone})
  1092  			if err != nil {
  1093  				// Error will already have been logged.
  1094  				continue
  1095  			}
  1096  			results = append(results, info)
  1097  
  1098  		}
  1099  	}
  1100  
  1101  	notFound := []string{}
  1102  	for subId, found := range subIdSet {
  1103  		if !found {
  1104  			notFound = append(notFound, subId)
  1105  		}
  1106  	}
  1107  	if len(notFound) != 0 {
  1108  		return nil, errors.Errorf("failed to find the following subnet ids: %v", notFound)
  1109  	}
  1110  
  1111  	return results, nil
  1112  }
  1113  
  1114  func (e *environ) AllInstances() ([]instance.Instance, error) {
  1115  	filter := ec2.NewFilter()
  1116  	filter.Add("instance-state-name", "pending", "running")
  1117  	err := e.addGroupFilter(filter)
  1118  	if err != nil {
  1119  		if ec2ErrCode(err) == "InvalidGroup.NotFound" {
  1120  			return nil, nil
  1121  		}
  1122  		return nil, err
  1123  	}
  1124  	resp, err := e.ec2().Instances(nil, filter)
  1125  	if err != nil {
  1126  		return nil, err
  1127  	}
  1128  	var insts []instance.Instance
  1129  	for _, r := range resp.Reservations {
  1130  		for i := range r.Instances {
  1131  			inst := r.Instances[i]
  1132  			// TODO(wallyworld): lookup the details to fill in the instance type data
  1133  			insts = append(insts, &ec2Instance{e: e, Instance: &inst})
  1134  		}
  1135  	}
  1136  	return insts, nil
  1137  }
  1138  
  1139  func (e *environ) Destroy() error {
  1140  	if err := common.Destroy(e); err != nil {
  1141  		return errors.Trace(err)
  1142  	}
  1143  	return e.Storage().RemoveAll()
  1144  }
  1145  
  1146  func portsToIPPerms(ports []network.PortRange) []ec2.IPPerm {
  1147  	ipPerms := make([]ec2.IPPerm, len(ports))
  1148  	for i, p := range ports {
  1149  		ipPerms[i] = ec2.IPPerm{
  1150  			Protocol:  p.Protocol,
  1151  			FromPort:  p.FromPort,
  1152  			ToPort:    p.ToPort,
  1153  			SourceIPs: []string{"0.0.0.0/0"},
  1154  		}
  1155  	}
  1156  	return ipPerms
  1157  }
  1158  
  1159  func (e *environ) openPortsInGroup(name string, ports []network.PortRange) error {
  1160  	if len(ports) == 0 {
  1161  		return nil
  1162  	}
  1163  	// Give permissions for anyone to access the given ports.
  1164  	g, err := e.groupByName(name)
  1165  	if err != nil {
  1166  		return err
  1167  	}
  1168  	ipPerms := portsToIPPerms(ports)
  1169  	_, err = e.ec2().AuthorizeSecurityGroup(g, ipPerms)
  1170  	if err != nil && ec2ErrCode(err) == "InvalidPermission.Duplicate" {
  1171  		if len(ports) == 1 {
  1172  			return nil
  1173  		}
  1174  		// If there's more than one port and we get a duplicate error,
  1175  		// then we go through authorizing each port individually,
  1176  		// otherwise the ports that were *not* duplicates will have
  1177  		// been ignored
  1178  		for i := range ipPerms {
  1179  			_, err := e.ec2().AuthorizeSecurityGroup(g, ipPerms[i:i+1])
  1180  			if err != nil && ec2ErrCode(err) != "InvalidPermission.Duplicate" {
  1181  				return fmt.Errorf("cannot open port %v: %v", ipPerms[i], err)
  1182  			}
  1183  		}
  1184  		return nil
  1185  	}
  1186  	if err != nil {
  1187  		return fmt.Errorf("cannot open ports: %v", err)
  1188  	}
  1189  	return nil
  1190  }
  1191  
  1192  func (e *environ) closePortsInGroup(name string, ports []network.PortRange) error {
  1193  	if len(ports) == 0 {
  1194  		return nil
  1195  	}
  1196  	// Revoke permissions for anyone to access the given ports.
  1197  	// Note that ec2 allows the revocation of permissions that aren't
  1198  	// granted, so this is naturally idempotent.
  1199  	g, err := e.groupByName(name)
  1200  	if err != nil {
  1201  		return err
  1202  	}
  1203  	_, err = e.ec2().RevokeSecurityGroup(g, portsToIPPerms(ports))
  1204  	if err != nil {
  1205  		return fmt.Errorf("cannot close ports: %v", err)
  1206  	}
  1207  	return nil
  1208  }
  1209  
  1210  func (e *environ) portsInGroup(name string) (ports []network.PortRange, err error) {
  1211  	group, err := e.groupInfoByName(name)
  1212  	if err != nil {
  1213  		return nil, err
  1214  	}
  1215  	for _, p := range group.IPPerms {
  1216  		if len(p.SourceIPs) != 1 {
  1217  			logger.Warningf("unexpected IP permission found: %v", p)
  1218  			continue
  1219  		}
  1220  		ports = append(ports, network.PortRange{
  1221  			Protocol: p.Protocol,
  1222  			FromPort: p.FromPort,
  1223  			ToPort:   p.ToPort,
  1224  		})
  1225  	}
  1226  	network.SortPortRanges(ports)
  1227  	return ports, nil
  1228  }
  1229  
  1230  func (e *environ) OpenPorts(ports []network.PortRange) error {
  1231  	if e.Config().FirewallMode() != config.FwGlobal {
  1232  		return fmt.Errorf("invalid firewall mode %q for opening ports on environment",
  1233  			e.Config().FirewallMode())
  1234  	}
  1235  	if err := e.openPortsInGroup(e.globalGroupName(), ports); err != nil {
  1236  		return err
  1237  	}
  1238  	logger.Infof("opened ports in global group: %v", ports)
  1239  	return nil
  1240  }
  1241  
  1242  func (e *environ) ClosePorts(ports []network.PortRange) error {
  1243  	if e.Config().FirewallMode() != config.FwGlobal {
  1244  		return fmt.Errorf("invalid firewall mode %q for closing ports on environment",
  1245  			e.Config().FirewallMode())
  1246  	}
  1247  	if err := e.closePortsInGroup(e.globalGroupName(), ports); err != nil {
  1248  		return err
  1249  	}
  1250  	logger.Infof("closed ports in global group: %v", ports)
  1251  	return nil
  1252  }
  1253  
  1254  func (e *environ) Ports() ([]network.PortRange, error) {
  1255  	if e.Config().FirewallMode() != config.FwGlobal {
  1256  		return nil, fmt.Errorf("invalid firewall mode %q for retrieving ports from environment",
  1257  			e.Config().FirewallMode())
  1258  	}
  1259  	return e.portsInGroup(e.globalGroupName())
  1260  }
  1261  
  1262  func (*environ) Provider() environs.EnvironProvider {
  1263  	return &providerInstance
  1264  }
  1265  
  1266  func (e *environ) terminateInstances(ids []instance.Id) error {
  1267  	if len(ids) == 0 {
  1268  		return nil
  1269  	}
  1270  	var err error
  1271  	ec2inst := e.ec2()
  1272  	strs := make([]string, len(ids))
  1273  	for i, id := range ids {
  1274  		strs[i] = string(id)
  1275  	}
  1276  	for a := shortAttempt.Start(); a.Next(); {
  1277  		_, err = ec2inst.TerminateInstances(strs)
  1278  		if err == nil || ec2ErrCode(err) != "InvalidInstanceID.NotFound" {
  1279  			return err
  1280  		}
  1281  	}
  1282  	if len(ids) == 1 {
  1283  		return err
  1284  	}
  1285  	// If we get a NotFound error, it means that no instances have been
  1286  	// terminated even if some exist, so try them one by one, ignoring
  1287  	// NotFound errors.
  1288  	var firstErr error
  1289  	for _, id := range ids {
  1290  		_, err = ec2inst.TerminateInstances([]string{string(id)})
  1291  		if ec2ErrCode(err) == "InvalidInstanceID.NotFound" {
  1292  			err = nil
  1293  		}
  1294  		if err != nil && firstErr == nil {
  1295  			firstErr = err
  1296  		}
  1297  	}
  1298  	return firstErr
  1299  }
  1300  
  1301  func (e *environ) globalGroupName() string {
  1302  	return fmt.Sprintf("%s-global", e.jujuGroupName())
  1303  }
  1304  
  1305  func (e *environ) machineGroupName(machineId string) string {
  1306  	return fmt.Sprintf("%s-%s", e.jujuGroupName(), machineId)
  1307  }
  1308  
  1309  func (e *environ) jujuGroupName() string {
  1310  	return "juju-" + e.name
  1311  }
  1312  
  1313  // setUpGroups creates the security groups for the new machine, and
  1314  // returns them.
  1315  //
  1316  // Instances are tagged with a group so they can be distinguished from
  1317  // other instances that might be running on the same EC2 account.  In
  1318  // addition, a specific machine security group is created for each
  1319  // machine, so that its firewall rules can be configured per machine.
  1320  func (e *environ) setUpGroups(machineId string, apiPort int) ([]ec2.SecurityGroup, error) {
  1321  	jujuGroup, err := e.ensureGroup(e.jujuGroupName(),
  1322  		[]ec2.IPPerm{
  1323  			{
  1324  				Protocol:  "tcp",
  1325  				FromPort:  22,
  1326  				ToPort:    22,
  1327  				SourceIPs: []string{"0.0.0.0/0"},
  1328  			},
  1329  			{
  1330  				Protocol:  "tcp",
  1331  				FromPort:  apiPort,
  1332  				ToPort:    apiPort,
  1333  				SourceIPs: []string{"0.0.0.0/0"},
  1334  			},
  1335  			{
  1336  				Protocol: "tcp",
  1337  				FromPort: 0,
  1338  				ToPort:   65535,
  1339  			},
  1340  			{
  1341  				Protocol: "udp",
  1342  				FromPort: 0,
  1343  				ToPort:   65535,
  1344  			},
  1345  			{
  1346  				Protocol: "icmp",
  1347  				FromPort: -1,
  1348  				ToPort:   -1,
  1349  			},
  1350  		})
  1351  	if err != nil {
  1352  		return nil, err
  1353  	}
  1354  	var machineGroup ec2.SecurityGroup
  1355  	switch e.Config().FirewallMode() {
  1356  	case config.FwInstance:
  1357  		machineGroup, err = e.ensureGroup(e.machineGroupName(machineId), nil)
  1358  	case config.FwGlobal:
  1359  		machineGroup, err = e.ensureGroup(e.globalGroupName(), nil)
  1360  	}
  1361  	if err != nil {
  1362  		return nil, err
  1363  	}
  1364  	return []ec2.SecurityGroup{jujuGroup, machineGroup}, nil
  1365  }
  1366  
  1367  // zeroGroup holds the zero security group.
  1368  var zeroGroup ec2.SecurityGroup
  1369  
  1370  // ensureGroup returns the security group with name and perms.
  1371  // If a group with name does not exist, one will be created.
  1372  // If it exists, its permissions are set to perms.
  1373  // Any entries in perms without SourceIPs will be granted for
  1374  // the named group only.
  1375  func (e *environ) ensureGroup(name string, perms []ec2.IPPerm) (g ec2.SecurityGroup, err error) {
  1376  	ec2inst := e.ec2()
  1377  	resp, err := ec2inst.CreateSecurityGroup("", name, "juju group")
  1378  	if err != nil && ec2ErrCode(err) != "InvalidGroup.Duplicate" {
  1379  		return zeroGroup, err
  1380  	}
  1381  
  1382  	var have permSet
  1383  	if err == nil {
  1384  		g = resp.SecurityGroup
  1385  	} else {
  1386  		resp, err := ec2inst.SecurityGroups(ec2.SecurityGroupNames(name), nil)
  1387  		if err != nil {
  1388  			return zeroGroup, err
  1389  		}
  1390  		info := resp.Groups[0]
  1391  		// It's possible that the old group has the wrong
  1392  		// description here, but if it does it's probably due
  1393  		// to something deliberately playing games with juju,
  1394  		// so we ignore it.
  1395  		g = info.SecurityGroup
  1396  		have = newPermSetForGroup(info.IPPerms, g)
  1397  	}
  1398  	want := newPermSetForGroup(perms, g)
  1399  	revoke := make(permSet)
  1400  	for p := range have {
  1401  		if !want[p] {
  1402  			revoke[p] = true
  1403  		}
  1404  	}
  1405  	if len(revoke) > 0 {
  1406  		_, err := ec2inst.RevokeSecurityGroup(g, revoke.ipPerms())
  1407  		if err != nil {
  1408  			return zeroGroup, fmt.Errorf("cannot revoke security group: %v", err)
  1409  		}
  1410  	}
  1411  
  1412  	add := make(permSet)
  1413  	for p := range want {
  1414  		if !have[p] {
  1415  			add[p] = true
  1416  		}
  1417  	}
  1418  	if len(add) > 0 {
  1419  		_, err := ec2inst.AuthorizeSecurityGroup(g, add.ipPerms())
  1420  		if err != nil {
  1421  			return zeroGroup, fmt.Errorf("cannot authorize securityGroup: %v", err)
  1422  		}
  1423  	}
  1424  	return g, nil
  1425  }
  1426  
  1427  // permKey represents a permission for a group or an ip address range
  1428  // to access the given range of ports. Only one of groupName or ipAddr
  1429  // should be non-empty.
  1430  type permKey struct {
  1431  	protocol string
  1432  	fromPort int
  1433  	toPort   int
  1434  	groupId  string
  1435  	ipAddr   string
  1436  }
  1437  
  1438  type permSet map[permKey]bool
  1439  
  1440  // newPermSetForGroup returns a set of all the permissions in the
  1441  // given slice of IPPerms. It ignores the name and owner
  1442  // id in source groups, and any entry with no source ips will
  1443  // be granted for the given group only.
  1444  func newPermSetForGroup(ps []ec2.IPPerm, group ec2.SecurityGroup) permSet {
  1445  	m := make(permSet)
  1446  	for _, p := range ps {
  1447  		k := permKey{
  1448  			protocol: p.Protocol,
  1449  			fromPort: p.FromPort,
  1450  			toPort:   p.ToPort,
  1451  		}
  1452  		if len(p.SourceIPs) > 0 {
  1453  			for _, ip := range p.SourceIPs {
  1454  				k.ipAddr = ip
  1455  				m[k] = true
  1456  			}
  1457  		} else {
  1458  			k.groupId = group.Id
  1459  			m[k] = true
  1460  		}
  1461  	}
  1462  	return m
  1463  }
  1464  
  1465  // ipPerms returns m as a slice of permissions usable
  1466  // with the ec2 package.
  1467  func (m permSet) ipPerms() (ps []ec2.IPPerm) {
  1468  	// We could compact the permissions, but it
  1469  	// hardly seems worth it.
  1470  	for p := range m {
  1471  		ipp := ec2.IPPerm{
  1472  			Protocol: p.protocol,
  1473  			FromPort: p.fromPort,
  1474  			ToPort:   p.toPort,
  1475  		}
  1476  		if p.ipAddr != "" {
  1477  			ipp.SourceIPs = []string{p.ipAddr}
  1478  		} else {
  1479  			ipp.SourceGroups = []ec2.UserSecurityGroup{{Id: p.groupId}}
  1480  		}
  1481  		ps = append(ps, ipp)
  1482  	}
  1483  	return
  1484  }
  1485  
  1486  // isZoneConstrainedError reports whether or not the error indicates
  1487  // RunInstances failed due to the specified availability zone being
  1488  // constrained for the instance type being provisioned, or is
  1489  // otherwise unusable for the specific request made.
  1490  func isZoneConstrainedError(err error) bool {
  1491  	switch err := err.(type) {
  1492  	case *ec2.Error:
  1493  		switch err.Code {
  1494  		case "Unsupported", "InsufficientInstanceCapacity":
  1495  			// A big hammer, but we've now seen several different error messages
  1496  			// for constrained zones, and who knows how many more there might
  1497  			// be. If the message contains "Availability Zone", it's a fair
  1498  			// bet that it's constrained or otherwise unusable.
  1499  			return strings.Contains(err.Message, "Availability Zone")
  1500  		case "InvalidInput":
  1501  			// If the region has a default VPC, then we will receive an error
  1502  			// if the AZ does not have a default subnet. Until we have proper
  1503  			// support for networks, we'll skip over these.
  1504  			return strings.HasPrefix(err.Message, "No default subnet for availability zone")
  1505  		case "VolumeTypeNotAvailableInZone":
  1506  			return true
  1507  		}
  1508  	}
  1509  	return false
  1510  }
  1511  
  1512  // If the err is of type *ec2.Error, ec2ErrCode returns
  1513  // its code, otherwise it returns the empty string.
  1514  func ec2ErrCode(err error) string {
  1515  	ec2err, _ := err.(*ec2.Error)
  1516  	if ec2err == nil {
  1517  		return ""
  1518  	}
  1519  	return ec2err.Code
  1520  }