github.com/in4it/ecs-deploy@v0.0.42-0.20240508120354-ed77ff16df25/provider/ecs/ecs.go (about)

     1  package ecs
     2  
     3  import (
     4  	"github.com/aws/aws-sdk-go/aws"
     5  	"github.com/aws/aws-sdk-go/aws/awserr"
     6  	"github.com/aws/aws-sdk-go/aws/request"
     7  	"github.com/aws/aws-sdk-go/aws/session"
     8  	"github.com/aws/aws-sdk-go/service/ec2"
     9  	"github.com/aws/aws-sdk-go/service/ecs"
    10  	"github.com/in4it/ecs-deploy/integrations"
    11  	"github.com/in4it/ecs-deploy/service"
    12  	"github.com/in4it/ecs-deploy/util"
    13  	"github.com/juju/loggo"
    14  
    15  	"context"
    16  	"crypto/rsa"
    17  	"crypto/x509"
    18  	"encoding/base64"
    19  	"encoding/pem"
    20  	"errors"
    21  	"fmt"
    22  	"math"
    23  	"strconv"
    24  	"strings"
    25  	"time"
    26  )
    27  
    28  // logging
    29  var ecsLogger = loggo.GetLogger("ecs")
    30  
    31  // ECS struct
    32  type ECS struct {
    33  	ClusterName    string
    34  	ServiceName    string
    35  	IamRoleArn     string
    36  	TaskDefinition *ecs.RegisterTaskDefinitionInput
    37  	TaskDefArn     *string
    38  	TargetGroupArn *string
    39  }
    40  
    41  type ECSIf interface {
    42  	GetInstanceResources(clusterName string) ([]FreeInstanceResource, []RegisteredInstanceResource, error)
    43  }
    44  
    45  // Task definition and Container definition
    46  type TaskDefinition struct {
    47  	Family               string                `json:"family"`
    48  	Revision             int64                 `json:"revision"`
    49  	ExecutionRoleArn     string                `json:"executionRole"`
    50  	ContainerDefinitions []ContainerDefinition `json:"containerDefinitions"`
    51  }
    52  type ContainerDefinition struct {
    53  	Name      string `json:"name"`
    54  	Essential bool   `json:"essential"`
    55  }
    56  
    57  // containerInstance
    58  type ContainerInstance struct {
    59  	ContainerInstanceArn string
    60  	Ec2InstanceId        string
    61  	AvailabilityZone     string
    62  	PendingTasksCount    int64
    63  	RegisteredAt         time.Time
    64  	RegisteredResources  []ContainerInstanceResource
    65  	RemainingResources   []ContainerInstanceResource
    66  	RunningTasksCount    int64
    67  	Status               string
    68  	Version              int64
    69  }
    70  type ContainerInstanceResource struct {
    71  	DoubleValue    float64  `json:"doubleValue"`
    72  	IntegerValue   int64    `json:"integerValue"`
    73  	Name           string   `json:"name"`
    74  	StringSetValue []string `json:"stringSetValue"`
    75  	Type           string   `json:"type"`
    76  }
    77  
    78  // free instance resource
    79  type FreeInstanceResource struct {
    80  	InstanceId       string
    81  	AvailabilityZone string
    82  	Status           string
    83  	FreeMemory       int64
    84  	FreeCpu          int64
    85  }
    86  
    87  // registered instance resource
    88  type RegisteredInstanceResource struct {
    89  	InstanceId       string
    90  	RegisteredMemory int64
    91  	RegisteredCpu    int64
    92  }
    93  
    94  // version info
    95  type EcsVersionInfo struct {
    96  	AgentHash     string `json:"agentHash"`
    97  	AgentVersion  string `json:"agentVersion"`
    98  	DockerVersion string `json:"dockerVersion"`
    99  }
   100  
   101  // task metadata
   102  type EcsTaskMetadata struct {
   103  	Cluster       string                         `json:"Cluster"`
   104  	TaskARN       string                         `json:"TaskARN"`
   105  	Family        string                         `json:"Family"`
   106  	Revision      string                         `json:"Revision"`
   107  	DesiredStatus string                         `json:"DesiredStatus"`
   108  	KnownStatus   string                         `json:"KnownStatus"`
   109  	Containers    []EcsTaskMetadataItemContainer `json:"Containers"`
   110  }
   111  type EcsTaskMetadataItemContainer struct {
   112  	DockerId      string                                 `json:"DockerId"`
   113  	DockerName    string                                 `json:"DockerName"`
   114  	Name          string                                 `json:"Name"`
   115  	Image         string                                 `json:"Image"`
   116  	ImageID       string                                 `json:"ImageID"`
   117  	Labels        map[string]string                      `json:"Labels"`
   118  	DesiredStatus string                                 `json:"DesiredStatus"`
   119  	KnownStatus   string                                 `json:"KnownStatus"`
   120  	CreatedAt     time.Time                              `json:"CreatedAt"`
   121  	StartedAt     time.Time                              `json:"CreatedAt"`
   122  	Limits        EcsTaskMetadataItemContainerLimits     `json:"Limits"`
   123  	Type          string                                 `json:"Type"`
   124  	Networks      []EcsTaskMetadataItemContainerNetworks `json:"Networks"`
   125  }
   126  type EcsTaskMetadataItemContainerLimits struct {
   127  	CPU    int64 `json:"CPU"`
   128  	Memory int64 `json:"Memory"`
   129  }
   130  type EcsTaskMetadataItemContainerNetworks struct {
   131  	NetworkMode   string   `json:"NetworkMode"`
   132  	Ipv4Addresses []string `json:"Ipv4Addresses"`
   133  }
   134  
   135  // create cluster
   136  func (e *ECS) CreateCluster(clusterName string) (*string, error) {
   137  	svc := ecs.New(session.New())
   138  	createClusterInput := &ecs.CreateClusterInput{
   139  		ClusterName: aws.String(clusterName),
   140  	}
   141  
   142  	result, err := svc.CreateCluster(createClusterInput)
   143  	if err != nil {
   144  		if aerr, ok := err.(awserr.Error); ok {
   145  			ecsLogger.Errorf("%v", aerr.Error())
   146  		} else {
   147  			ecsLogger.Errorf("%v", err.Error())
   148  		}
   149  		return nil, err
   150  	}
   151  	return result.Cluster.ClusterArn, nil
   152  }
   153  func (e *ECS) GetECSAMI() (string, error) {
   154  	var amiId string
   155  	svc := ec2.New(session.New())
   156  	input := &ec2.DescribeImagesInput{
   157  		Owners: []*string{aws.String("591542846629")}, // AWS
   158  		Filters: []*ec2.Filter{
   159  			{Name: aws.String("name"), Values: []*string{aws.String("amzn-ami-*-amazon-ecs-optimized")}},
   160  			{Name: aws.String("virtualization-type"), Values: []*string{aws.String("hvm")}},
   161  		},
   162  	}
   163  	result, err := svc.DescribeImages(input)
   164  	if err != nil {
   165  		if aerr, ok := err.(awserr.Error); ok {
   166  			ecsLogger.Errorf("%v", aerr.Error())
   167  		} else {
   168  			ecsLogger.Errorf("%v", err.Error())
   169  		}
   170  		return amiId, err
   171  	}
   172  	if len(result.Images) == 0 {
   173  		return amiId, errors.New("No ECS AMI found")
   174  	}
   175  	layout := "2006-01-02T15:04:05.000Z"
   176  	var lastTime time.Time
   177  	for _, v := range result.Images {
   178  		t, err := time.Parse(layout, *v.CreationDate)
   179  		if err != nil {
   180  			return amiId, err
   181  		}
   182  		if t.After(lastTime) {
   183  			lastTime = t
   184  			amiId = *v.ImageId
   185  		}
   186  	}
   187  	return amiId, nil
   188  }
   189  func (e *ECS) ImportKeyPair(keyName string, publicKey []byte) error {
   190  	svc := ec2.New(session.New())
   191  	input := &ec2.ImportKeyPairInput{
   192  		KeyName:           aws.String(keyName),
   193  		PublicKeyMaterial: publicKey,
   194  	}
   195  	_, err := svc.ImportKeyPair(input)
   196  	if err != nil {
   197  		if aerr, ok := err.(awserr.Error); ok {
   198  			ecsLogger.Errorf("%v", aerr.Error())
   199  		} else {
   200  			ecsLogger.Errorf("%v", err.Error())
   201  		}
   202  		return err
   203  	}
   204  	return nil
   205  }
   206  func (e *ECS) GetPubKeyFromPrivateKey(privateKey string) ([]byte, error) {
   207  	var pubASN1 []byte
   208  	var key *rsa.PrivateKey
   209  	block, _ := pem.Decode([]byte(privateKey))
   210  	if block == nil {
   211  		return pubASN1, errors.New("No private key found")
   212  	}
   213  	if block.Type != "RSA PRIVATE KEY" {
   214  		return pubASN1, errors.New("Key not a RSA PRIVATE KEY")
   215  	}
   216  	key, err := x509.ParsePKCS1PrivateKey([]byte(block.Bytes))
   217  	if err != nil {
   218  		return pubASN1, err
   219  	}
   220  	pubASN1, err = x509.MarshalPKIXPublicKey(&key.PublicKey)
   221  	if err != nil {
   222  		return pubASN1, err
   223  	}
   224  	return []byte(base64.StdEncoding.EncodeToString(pubASN1)), nil
   225  }
   226  func (e *ECS) DeleteKeyPair(keyName string) error {
   227  	svc := ec2.New(session.New())
   228  	input := &ec2.DeleteKeyPairInput{
   229  		KeyName: aws.String(keyName),
   230  	}
   231  	_, err := svc.DeleteKeyPair(input)
   232  	if err != nil {
   233  		if aerr, ok := err.(awserr.Error); ok {
   234  			ecsLogger.Errorf("%v", aerr.Error())
   235  		} else {
   236  			ecsLogger.Errorf("%v", err.Error())
   237  		}
   238  		return err
   239  	}
   240  	return nil
   241  }
   242  
   243  // delete cluster
   244  func (e *ECS) DeleteCluster(clusterName string) error {
   245  	svc := ecs.New(session.New())
   246  	deleteClusterInput := &ecs.DeleteClusterInput{
   247  		Cluster: aws.String(clusterName),
   248  	}
   249  
   250  	_, err := svc.DeleteCluster(deleteClusterInput)
   251  	if err != nil {
   252  		if aerr, ok := err.(awserr.Error); ok {
   253  			ecsLogger.Errorf("%v", aerr.Error())
   254  		} else {
   255  			ecsLogger.Errorf("%v", err.Error())
   256  		}
   257  		return err
   258  	}
   259  	return nil
   260  }
   261  
   262  // Creates ECS repository
   263  func (e *ECS) CreateTaskDefinitionInput(d service.Deploy, secrets map[string]string, accountId string) error {
   264  	e.TaskDefinition = &ecs.RegisterTaskDefinitionInput{
   265  		Family:      aws.String(e.ServiceName),
   266  		TaskRoleArn: aws.String(e.IamRoleArn),
   267  	}
   268  
   269  	// set network mode if set
   270  	if d.NetworkMode != "" {
   271  		e.TaskDefinition.SetNetworkMode(d.NetworkMode)
   272  	}
   273  
   274  	// placement constraints
   275  	if len(d.PlacementConstraints) > 0 {
   276  		var pcs []*ecs.TaskDefinitionPlacementConstraint
   277  		for _, pc := range d.PlacementConstraints {
   278  			tdpc := &ecs.TaskDefinitionPlacementConstraint{}
   279  			if pc.Expression != "" {
   280  				tdpc.SetExpression(pc.Expression)
   281  			}
   282  			if pc.Type != "" {
   283  				tdpc.SetType(pc.Type)
   284  			}
   285  			pcs = append(pcs, tdpc)
   286  		}
   287  		e.TaskDefinition.SetPlacementConstraints(pcs)
   288  	}
   289  
   290  	// volumes
   291  	if len(d.Volumes) > 0 {
   292  		var volumes []*ecs.Volume
   293  		for _, vol := range d.Volumes {
   294  			volume := &ecs.Volume{
   295  				Name: aws.String(vol.Name),
   296  			}
   297  			if len(vol.Host.SourcePath) > 0 {
   298  				volume.SetHost(&ecs.HostVolumeProperties{
   299  					SourcePath: aws.String(vol.Host.SourcePath),
   300  				})
   301  			}
   302  			if len(vol.DockerVolumeConfiguration.Scope) > 0 {
   303  				volumeConfig := &ecs.DockerVolumeConfiguration{
   304  					Autoprovision: aws.Bool(vol.DockerVolumeConfiguration.Autoprovision),
   305  					Driver:        aws.String(vol.DockerVolumeConfiguration.Driver),
   306  					Scope:         aws.String(vol.DockerVolumeConfiguration.Scope),
   307  				}
   308  				if len(vol.DockerVolumeConfiguration.DriverOpts) > 0 {
   309  					volumeConfig.SetDriverOpts(aws.StringMap(vol.DockerVolumeConfiguration.DriverOpts))
   310  				}
   311  				if len(vol.DockerVolumeConfiguration.Labels) > 0 {
   312  					volumeConfig.SetLabels(aws.StringMap(vol.DockerVolumeConfiguration.Labels))
   313  				}
   314  				volume.SetDockerVolumeConfiguration(volumeConfig)
   315  			}
   316  			volumes = append(volumes, volume)
   317  		}
   318  		e.TaskDefinition.SetVolumes(volumes)
   319  	}
   320  
   321  	// loop over containers
   322  	for _, container := range d.Containers {
   323  
   324  		// prepare image Uri
   325  		var imageUri string
   326  		if container.ContainerURI == "" {
   327  			if container.ContainerImage == "" {
   328  				imageUri = accountId + ".dkr.ecr." + util.GetEnv("AWS_REGION", "") + ".amazonaws.com" + "/" + container.ContainerName
   329  			} else {
   330  				imageUri = accountId + ".dkr.ecr." + util.GetEnv("AWS_REGION", "") + ".amazonaws.com" + "/" + container.ContainerImage
   331  			}
   332  			if container.ContainerTag != "" {
   333  				imageUri += ":" + container.ContainerTag
   334  			}
   335  		} else {
   336  			imageUri = container.ContainerURI
   337  		}
   338  
   339  		// prepare container definition
   340  		containerDefinition := &ecs.ContainerDefinition{
   341  			Name:         aws.String(container.ContainerName),
   342  			Image:        aws.String(imageUri),
   343  			DockerLabels: aws.StringMap(container.DockerLabels),
   344  		}
   345  
   346  		if len(container.HealthCheck.Command) > 0 {
   347  			healthCheck := &ecs.HealthCheck{
   348  				Command: container.HealthCheck.Command,
   349  			}
   350  			if container.HealthCheck.Interval > 0 {
   351  				healthCheck.SetInterval(container.HealthCheck.Interval)
   352  			}
   353  			if container.HealthCheck.Retries > 0 {
   354  				healthCheck.SetRetries(container.HealthCheck.Retries)
   355  			}
   356  			if container.HealthCheck.StartPeriod > 0 {
   357  				healthCheck.SetStartPeriod(container.HealthCheck.StartPeriod)
   358  			}
   359  			if container.HealthCheck.Timeout > 0 {
   360  				healthCheck.SetTimeout(container.HealthCheck.Timeout)
   361  			}
   362  			containerDefinition.SetHealthCheck(healthCheck)
   363  		}
   364  		// set containerPort if not empty
   365  		if container.ContainerPort > 0 {
   366  			if len(container.PortMappings) > 0 {
   367  				var portMapping []*ecs.PortMapping
   368  				for _, v := range container.PortMappings {
   369  					protocol := "tcp"
   370  					if v.Protocol != "" {
   371  						protocol = v.Protocol
   372  					}
   373  					if v.HostPort > 0 {
   374  						portMapping = append(portMapping, &ecs.PortMapping{
   375  							ContainerPort: aws.Int64(v.ContainerPort),
   376  							HostPort:      aws.Int64(v.HostPort),
   377  							Protocol:      aws.String(protocol),
   378  						})
   379  					} else {
   380  						portMapping = append(portMapping, &ecs.PortMapping{
   381  							ContainerPort: aws.Int64(v.ContainerPort),
   382  							Protocol:      aws.String(protocol),
   383  						})
   384  					}
   385  				}
   386  				containerDefinition.SetPortMappings(portMapping)
   387  			} else {
   388  				containerDefinition.SetPortMappings([]*ecs.PortMapping{
   389  					{
   390  						ContainerPort: aws.Int64(container.ContainerPort),
   391  					},
   392  				})
   393  			}
   394  		}
   395  		// set containerCommand if not empty
   396  		if len(container.ContainerCommand) > 0 {
   397  			containerDefinition.SetCommand(container.ContainerCommand)
   398  		}
   399  		// set containerEntryPoint if not empty
   400  		if len(container.ContainerEntryPoint) > 0 {
   401  			containerDefinition.SetEntryPoint(container.ContainerEntryPoint)
   402  		}
   403  		// set cloudwacht logs if enabled
   404  		if util.GetEnv("CLOUDWATCH_LOGS_ENABLED", "no") == "yes" {
   405  			var logPrefix string
   406  			if util.GetEnv("CLOUDWATCH_LOGS_PREFIX", "") != "" {
   407  				logPrefix = util.GetEnv("CLOUDWATCH_LOGS_PREFIX", "") + "-" + util.GetEnv("AWS_ACCOUNT_ENV", "")
   408  			}
   409  			containerDefinition.SetLogConfiguration(&ecs.LogConfiguration{
   410  				LogDriver: aws.String("awslogs"),
   411  				Options: map[string]*string{
   412  					"awslogs-group":         aws.String(logPrefix),
   413  					"awslogs-region":        aws.String(util.GetEnv("AWS_REGION", "")),
   414  					"awslogs-stream-prefix": aws.String(container.ContainerName),
   415  				},
   416  			})
   417  		}
   418  		// override logconfiguration if set in deploy config
   419  		if container.LogConfiguration.LogDriver != "" {
   420  			containerDefinition.SetLogConfiguration(&ecs.LogConfiguration{
   421  				LogDriver: aws.String(container.LogConfiguration.LogDriver),
   422  			})
   423  			options := map[string]*string{}
   424  			if container.LogConfiguration.Options.MaxFile != "" {
   425  				options["max-file"] = aws.String(container.LogConfiguration.Options.MaxFile)
   426  			}
   427  			if container.LogConfiguration.Options.MaxSize != "" {
   428  				options["max-size"] = aws.String(container.LogConfiguration.Options.MaxSize)
   429  			}
   430  			containerDefinition.LogConfiguration.SetOptions(options)
   431  		}
   432  		if container.Memory > 0 {
   433  			containerDefinition.Memory = aws.Int64(container.Memory)
   434  		}
   435  		if container.MemoryReservation > 0 {
   436  			containerDefinition.MemoryReservation = aws.Int64(container.MemoryReservation)
   437  		}
   438  		if container.CPU > 0 {
   439  			containerDefinition.Cpu = aws.Int64(container.CPU)
   440  		} else {
   441  			if container.CPU == 0 && util.GetEnv("DEFAULT_CONTAINER_CPU_LIMIT", "") != "" {
   442  				defaultCpuLimit, err := strconv.ParseInt(util.GetEnv("DEFAULT_CONTAINER_CPU_LIMIT", ""), 10, 64)
   443  				if err != nil {
   444  					return err
   445  				}
   446  				containerDefinition.Cpu = aws.Int64(defaultCpuLimit)
   447  			}
   448  		}
   449  
   450  		if container.Essential {
   451  			containerDefinition.Essential = aws.Bool(container.Essential)
   452  		}
   453  
   454  		// environment variables
   455  		var environment []*ecs.KeyValuePair
   456  		if len(container.Environment) > 0 {
   457  			for _, v := range container.Environment {
   458  				environment = append(environment, &ecs.KeyValuePair{Name: aws.String(v.Name), Value: aws.String(v.Value)})
   459  			}
   460  		}
   461  		if util.GetEnv("PARAMSTORE_ENABLED", "no") == "yes" {
   462  			namespace := d.EnvNamespace
   463  			if namespace == "" {
   464  				namespace = e.ServiceName
   465  			}
   466  			environment = append(environment, &ecs.KeyValuePair{Name: aws.String("AWS_REGION"), Value: aws.String(util.GetEnv("AWS_REGION", ""))})
   467  			environment = append(environment, &ecs.KeyValuePair{Name: aws.String("AWS_ENV_PATH"), Value: aws.String("/" + util.GetEnv("PARAMSTORE_PREFIX", "") + "-" + util.GetEnv("AWS_ACCOUNT_ENV", "") + "/" + namespace + "/")})
   468  		}
   469  
   470  		if len(environment) > 0 {
   471  			containerDefinition.SetEnvironment(environment)
   472  		}
   473  
   474  		// ulimits
   475  		if len(container.Ulimits) > 0 {
   476  			var us []*ecs.Ulimit
   477  			for _, u := range container.Ulimits {
   478  				us = append(us, &ecs.Ulimit{
   479  					Name:      aws.String(u.Name),
   480  					SoftLimit: aws.Int64(u.SoftLimit),
   481  					HardLimit: aws.Int64(u.HardLimit),
   482  				})
   483  			}
   484  			containerDefinition.SetUlimits(us)
   485  		}
   486  
   487  		// MountPoints
   488  		if len(container.MountPoints) > 0 {
   489  			var mps []*ecs.MountPoint
   490  			for _, mp := range container.MountPoints {
   491  				mps = append(mps, &ecs.MountPoint{
   492  					ContainerPath: aws.String(mp.ContainerPath),
   493  					SourceVolume:  aws.String(mp.SourceVolume),
   494  					ReadOnly:      aws.Bool(mp.ReadOnly),
   495  				})
   496  			}
   497  			containerDefinition.SetMountPoints(mps)
   498  		}
   499  
   500  		// Links
   501  		if len(container.Links) > 0 {
   502  			containerDefinition.SetLinks(container.Links)
   503  		}
   504  
   505  		// inject parameter store entries as secrets
   506  		if util.GetEnv("PARAMSTORE_INJECT", "no") == "yes" {
   507  			ecsSecrets := []*ecs.Secret{}
   508  			for k, v := range secrets {
   509  				ecsSecrets = append(ecsSecrets, &ecs.Secret{
   510  					Name:      aws.String(k),
   511  					ValueFrom: aws.String(v),
   512  				})
   513  			}
   514  			containerDefinition.SetSecrets(ecsSecrets)
   515  		}
   516  
   517  		e.TaskDefinition.ContainerDefinitions = append(e.TaskDefinition.ContainerDefinitions, containerDefinition)
   518  	}
   519  
   520  	// add execution role
   521  	if util.GetEnv("PARAMSTORE_INJECT", "no") == "yes" {
   522  		iam := IAM{}
   523  		iamExecutionRoleName := util.GetEnv("AWS_ECS_EXECUTION_ROLE", "ecs-"+d.Cluster+"-task-execution-role")
   524  		iamExecutionRoleArn, err := iam.RoleExists(iamExecutionRoleName)
   525  		if err != nil {
   526  			return err
   527  		}
   528  		if iamExecutionRoleArn == nil {
   529  			return fmt.Errorf("Execution role %s not found and PARAMSTORE_INJECT enabled", iamExecutionRoleName)
   530  		}
   531  		e.TaskDefinition.SetExecutionRoleArn(aws.StringValue(iamExecutionRoleArn))
   532  	}
   533  
   534  	// app mesh
   535  	if d.AppMesh.Name != "" && d.NetworkMode == "awsvpc" {
   536  		a := AppMesh{}
   537  		virtualNodeName := d.ServiceName
   538  		virtualNodeDNS := strings.ToLower(d.ServiceName + "." + d.ServiceRegistry)
   539  		virtualServiceName := strings.ToLower(d.ServiceName + "." + d.ServiceRegistry)
   540  
   541  		// list virtual nodes and services
   542  		virtualNodes, err := a.listVirtualNodes(d.AppMesh.Name)
   543  		if err != nil {
   544  			return err
   545  		}
   546  		virtualServices, err := a.listVirtualServices(d.AppMesh.Name)
   547  		if err != nil {
   548  			return err
   549  		}
   550  		// get healthcheck object
   551  		healthCheck, err := e.prepareAppMeshHealthcheck(d.HealthCheck, d.ServicePort, d.ServiceProtocol)
   552  		if err != nil {
   553  			return err
   554  		}
   555  
   556  		// create virtual node if it doesn't exist yet
   557  		if _, ok := virtualNodes[virtualNodeName]; !ok {
   558  			if err := a.createVirtualNode(virtualNodeName, virtualNodeDNS, d.AppMesh.Name, d.ServicePort, healthCheck, d.AppMesh.Backends); err != nil {
   559  				return err
   560  			}
   561  		} else {
   562  			// update
   563  			if err := a.updateVirtualNode(virtualNodeName, virtualNodeDNS, d.AppMesh.Name, d.ServicePort, healthCheck, d.AppMesh.Backends); err != nil {
   564  				return err
   565  			}
   566  		}
   567  
   568  		// retry policy
   569  		if d.AppMesh.RetryPolicy.MaxRetries > 0 {
   570  			virtualRouters, err := a.listVirtualRouters(d.AppMesh.Name)
   571  			if err != nil {
   572  				return err
   573  			}
   574  			virtualRouterName := "retries_" + d.ServiceName
   575  			if _, ok := virtualRouters[virtualRouterName]; !ok {
   576  				if err := a.createVirtualRouter(virtualRouterName, d.AppMesh.Name, d.ServicePort); err != nil {
   577  					return err
   578  				}
   579  				if err := a.createRoute("retries", virtualRouterName, virtualNodeName, virtualNodeDNS, d.AppMesh); err != nil {
   580  					return err
   581  				}
   582  			}
   583  			// create virtual service if it doesn't exist yet, or update existing with new virtualRouter
   584  			if _, ok := virtualServices[virtualServiceName]; !ok {
   585  				if err := a.createVirtualServiceWithVirtualRouter(virtualServiceName, virtualRouterName, d.AppMesh.Name); err != nil {
   586  					return err
   587  				}
   588  			} else {
   589  				if err := a.updateVirtualServiceWithVirtualRouter(virtualServiceName, virtualRouterName, d.AppMesh.Name); err != nil {
   590  					return err
   591  				}
   592  			}
   593  		} else {
   594  			// create virtual service if it doesn't exist yet
   595  			if _, ok := virtualServices[virtualServiceName]; !ok {
   596  				if err := a.createVirtualServiceWithVirtualNode(virtualServiceName, virtualNodeName, d.AppMesh.Name); err != nil {
   597  					return err
   598  				}
   599  			}
   600  		}
   601  
   602  		proxyConfiguration := &ecs.ProxyConfiguration{
   603  			Type:          aws.String("APPMESH"),
   604  			ContainerName: aws.String("envoy"),
   605  			Properties: []*ecs.KeyValuePair{
   606  				{
   607  					Name:  aws.String("IgnoredUID"),
   608  					Value: aws.String("1337"),
   609  				},
   610  				{
   611  					Name:  aws.String("ProxyIngressPort"),
   612  					Value: aws.String("15000"),
   613  				},
   614  				{
   615  					Name:  aws.String("ProxyEgressPort"),
   616  					Value: aws.String("15001"),
   617  				},
   618  				{
   619  					Name:  aws.String("AppPorts"),
   620  					Value: aws.String(strconv.FormatInt(d.ServicePort, 10)),
   621  				},
   622  				{
   623  					Name:  aws.String("EgressIgnoredIPs"),
   624  					Value: aws.String("169.254.170.2,169.254.169.254"),
   625  				},
   626  			},
   627  		}
   628  		if len(d.AppMesh.Configuration.EgressIgnoredPorts) != 0 {
   629  			proxyConfiguration.Properties = append(proxyConfiguration.Properties, &ecs.KeyValuePair{
   630  				Name:  aws.String("EgressIgnoredPorts"),
   631  				Value: aws.String(strings.Join(d.AppMesh.Configuration.EgressIgnoredPorts, ",")),
   632  			})
   633  		}
   634  		e.TaskDefinition.SetProxyConfiguration(proxyConfiguration)
   635  		for k := range e.TaskDefinition.ContainerDefinitions {
   636  			e.TaskDefinition.ContainerDefinitions[k].SetDependsOn([]*ecs.ContainerDependency{
   637  				{
   638  					Condition:     aws.String("HEALTHY"),
   639  					ContainerName: aws.String("envoy"),
   640  				},
   641  			})
   642  		}
   643  		e.TaskDefinition.ContainerDefinitions = append(e.TaskDefinition.ContainerDefinitions, &ecs.ContainerDefinition{
   644  			Name:              aws.String("envoy"),
   645  			Image:             aws.String(util.GetEnv("APPMESH_IMAGE", "111345817488.dkr.ecr."+util.GetEnv("AWS_REGION", "us-west-2")+".amazonaws.com/aws-appmesh-envoy:v1.11.1.1-prod")),
   646  			Essential:         aws.Bool(true),
   647  			MemoryReservation: aws.Int64(256),
   648  			Environment: []*ecs.KeyValuePair{
   649  				{
   650  					Name:  aws.String("APPMESH_VIRTUAL_NODE_NAME"),
   651  					Value: aws.String("mesh/" + d.AppMesh.Name + "/virtualNode/" + virtualNodeName),
   652  				},
   653  			},
   654  			HealthCheck: &ecs.HealthCheck{
   655  				Command:     aws.StringSlice([]string{"CMD-SHELL", "curl -s http://localhost:9901/server_info | grep state | grep -q LIVE"}),
   656  				StartPeriod: aws.Int64(10),
   657  				Interval:    aws.Int64(5),
   658  				Timeout:     aws.Int64(2),
   659  				Retries:     aws.Int64(3),
   660  			},
   661  			User: aws.String("1337"),
   662  		})
   663  	}
   664  
   665  	return nil
   666  }
   667  
   668  func (e *ECS) prepareAppMeshHealthcheck(healthCheck service.DeployHealthCheck, servicePort int64, serviceProtocol string) (AppMeshHealthCheck, error) {
   669  	var healthCheckPort int64
   670  	var err error
   671  	if healthCheck.HealthyThreshold == 0 {
   672  		healthCheck.HealthyThreshold = 3
   673  	}
   674  	if healthCheck.Interval == 0 {
   675  		healthCheck.Interval = 60
   676  	}
   677  	if healthCheck.Path == "" && strings.ToLower(healthCheck.Protocol) == "http" {
   678  		return AppMeshHealthCheck{}, fmt.Errorf("Healthcheck path must be set when enabling AppMesh and protocol is http")
   679  	}
   680  	if healthCheck.Port == "" {
   681  		healthCheckPort = servicePort
   682  	} else {
   683  		healthCheckPort, err = strconv.ParseInt(healthCheck.Port, 10, 64)
   684  		if err == nil {
   685  			return AppMeshHealthCheck{}, err
   686  		}
   687  	}
   688  	if healthCheck.Protocol == "" {
   689  		if serviceProtocol != "" {
   690  			healthCheck.Protocol = strings.ToLower(serviceProtocol)
   691  		} else {
   692  			healthCheck.Protocol = "http"
   693  		}
   694  	}
   695  	if healthCheck.Timeout == 0 {
   696  		healthCheck.Timeout = 30
   697  	}
   698  	if healthCheck.UnhealthyThreshold == 0 {
   699  		healthCheck.UnhealthyThreshold = 3
   700  	}
   701  	return AppMeshHealthCheck{
   702  		HealthyThreshold:   healthCheck.HealthyThreshold,
   703  		IntervalMillis:     healthCheck.Interval * 1000,
   704  		Path:               healthCheck.Path,
   705  		Port:               healthCheckPort,
   706  		Protocol:           strings.ToLower(healthCheck.Protocol),
   707  		TimeoutMillis:      healthCheck.Timeout * 1000,
   708  		UnhealthyThreshold: healthCheck.UnhealthyThreshold,
   709  	}, nil
   710  }
   711  
   712  func (e *ECS) CreateTaskDefinition(d service.Deploy, secrets map[string]string) (*string, error) {
   713  	var err error
   714  
   715  	svc := ecs.New(session.New())
   716  
   717  	// get account id
   718  	iam := IAM{}
   719  	err = iam.GetAccountId()
   720  	if err != nil {
   721  		return nil, errors.New("Could not get accountId during createTaskDefinition")
   722  	}
   723  
   724  	err = e.CreateTaskDefinitionInput(d, secrets, iam.AccountId)
   725  	if err != nil {
   726  		return nil, err
   727  	}
   728  
   729  	// going to register
   730  	ecsLogger.Debugf("Going to register: %+v", e.TaskDefinition)
   731  
   732  	result, err := svc.RegisterTaskDefinition(e.TaskDefinition)
   733  	if err != nil {
   734  		if aerr, ok := err.(awserr.Error); ok {
   735  			switch aerr.Code() {
   736  			case ecs.ErrCodeServerException:
   737  				ecsLogger.Errorf(ecs.ErrCodeServerException+": %v", aerr.Error())
   738  			case ecs.ErrCodeClientException:
   739  				ecsLogger.Errorf(ecs.ErrCodeClientException+": %v", aerr.Error())
   740  			case ecs.ErrCodeInvalidParameterException:
   741  				ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException+": %v", aerr.Error())
   742  			default:
   743  				ecsLogger.Errorf(aerr.Error())
   744  			}
   745  		}
   746  		// return error
   747  		return nil, errors.New("Could not register task definition")
   748  	} else {
   749  		return result.TaskDefinition.TaskDefinitionArn, nil
   750  	}
   751  }
   752  
   753  // check whether service exists
   754  func (e *ECS) ServiceExists(serviceName string) (bool, error) {
   755  	svc := ecs.New(session.New())
   756  	input := &ecs.DescribeServicesInput{
   757  		Cluster: aws.String(e.ClusterName),
   758  		Services: []*string{
   759  			aws.String(serviceName),
   760  		},
   761  	}
   762  
   763  	result, err := svc.DescribeServices(input)
   764  	if err != nil {
   765  		if aerr, ok := err.(awserr.Error); ok {
   766  			switch aerr.Code() {
   767  			case ecs.ErrCodeServerException:
   768  				ecsLogger.Errorf(ecs.ErrCodeServerException, aerr.Error())
   769  			case ecs.ErrCodeClientException:
   770  				ecsLogger.Errorf(ecs.ErrCodeClientException, aerr.Error())
   771  			case ecs.ErrCodeInvalidParameterException:
   772  				ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException, aerr.Error())
   773  			case ecs.ErrCodeClusterNotFoundException:
   774  				ecsLogger.Errorf(ecs.ErrCodeClusterNotFoundException, aerr.Error())
   775  			default:
   776  				ecsLogger.Errorf(aerr.Error())
   777  			}
   778  		} else {
   779  			// Print the error, cast err to awserr.Error to get the Code and
   780  			// Message from an error.
   781  			ecsLogger.Errorf(err.Error())
   782  		}
   783  		return false, err
   784  	}
   785  	if len(result.Services) == 0 {
   786  		return false, nil
   787  	} else if len(result.Services) == 1 && *result.Services[0].Status == "INACTIVE" {
   788  		return false, nil
   789  	} else {
   790  		return true, nil
   791  	}
   792  }
   793  
   794  // Update ECS service
   795  func (e *ECS) UpdateService(serviceName string, taskDefArn *string, d service.Deploy) (*string, error) {
   796  	svc := ecs.New(session.New())
   797  	input := &ecs.UpdateServiceInput{
   798  		Cluster:        aws.String(e.ClusterName),
   799  		Service:        aws.String(serviceName),
   800  		TaskDefinition: aws.String(*taskDefArn),
   801  	}
   802  
   803  	// network configuration
   804  	if d.NetworkMode == "awsvpc" && len(d.NetworkConfiguration.Subnets) > 0 {
   805  		input.SetNetworkConfiguration(e.getNetworkConfiguration(d))
   806  	}
   807  
   808  	// set gracePeriodSeconds
   809  	if d.HealthCheck.GracePeriodSeconds > 0 {
   810  		input.SetHealthCheckGracePeriodSeconds(d.HealthCheck.GracePeriodSeconds)
   811  	}
   812  
   813  	ecsLogger.Debugf("Running UpdateService with input: %+v", input)
   814  
   815  	result, err := svc.UpdateService(input)
   816  	if err != nil {
   817  		if aerr, ok := err.(awserr.Error); ok {
   818  			switch aerr.Code() {
   819  			case ecs.ErrCodeServerException:
   820  				ecsLogger.Errorf(ecs.ErrCodeServerException+": %v", aerr.Error())
   821  			case ecs.ErrCodeClientException:
   822  				ecsLogger.Errorf(ecs.ErrCodeClientException+": %v", aerr.Error())
   823  			case ecs.ErrCodeInvalidParameterException:
   824  				ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException+": %v", aerr.Error())
   825  			case ecs.ErrCodeClusterNotFoundException:
   826  				ecsLogger.Errorf(ecs.ErrCodeClusterNotFoundException+": %v", aerr.Error())
   827  			case ecs.ErrCodeServiceNotFoundException:
   828  				ecsLogger.Infof(ecs.ErrCodeServiceNotFoundException+": %v", aerr.Error())
   829  				// return error code to create new service
   830  				return nil, errors.New("ServiceNotFoundException")
   831  			case ecs.ErrCodeServiceNotActiveException:
   832  				ecsLogger.Errorf(ecs.ErrCodeServiceNotActiveException+": %v", aerr.Error())
   833  			default:
   834  				ecsLogger.Errorf(aerr.Error())
   835  			}
   836  		} else {
   837  			// Print the error, cast err to awserr.Error to get the Code and
   838  			// Message from an error.
   839  			ecsLogger.Errorf(err.Error())
   840  		}
   841  		return nil, errors.New("Could not update service: " + serviceName)
   842  	}
   843  	return result.Service.ServiceName, nil
   844  }
   845  
   846  // delete ECS service
   847  func (e *ECS) DeleteService(clusterName, serviceName string) error {
   848  	// first set desiredCount to 0
   849  	svc := ecs.New(session.New())
   850  	input := &ecs.UpdateServiceInput{
   851  		Cluster:      aws.String(clusterName),
   852  		Service:      aws.String(serviceName),
   853  		DesiredCount: aws.Int64(0),
   854  	}
   855  
   856  	_, err := svc.UpdateService(input)
   857  	if err != nil {
   858  		if aerr, ok := err.(awserr.Error); ok {
   859  			ecsLogger.Errorf("%v", aerr.Error())
   860  		} else {
   861  			ecsLogger.Errorf("%v", err.Error())
   862  		}
   863  		return err
   864  	}
   865  	// delete service
   866  	input2 := &ecs.DeleteServiceInput{
   867  		Cluster: aws.String(clusterName),
   868  		Service: aws.String(serviceName),
   869  	}
   870  
   871  	_, err = svc.DeleteService(input2)
   872  	if err != nil {
   873  		if aerr, ok := err.(awserr.Error); ok {
   874  			ecsLogger.Errorf("%v", aerr.Error())
   875  		} else {
   876  			ecsLogger.Errorf("%v", err.Error())
   877  		}
   878  		return err
   879  	}
   880  	return nil
   881  }
   882  
   883  // create service
   884  func (e *ECS) CreateService(d service.Deploy) error {
   885  	svc := ecs.New(session.New())
   886  
   887  	// sanity checks
   888  	if len(d.Containers) == 0 {
   889  		return errors.New("No containers defined")
   890  	}
   891  
   892  	input := &ecs.CreateServiceInput{
   893  		Cluster:        aws.String(d.Cluster),
   894  		ServiceName:    aws.String(e.ServiceName),
   895  		TaskDefinition: aws.String(*e.TaskDefArn),
   896  	}
   897  
   898  	if d.SchedulingStrategy != "DAEMON" {
   899  		input.SetPlacementStrategy([]*ecs.PlacementStrategy{
   900  			{
   901  				Field: aws.String("attribute:ecs.availability-zone"),
   902  				Type:  aws.String("spread"),
   903  			},
   904  			{
   905  				Field: aws.String("memory"),
   906  				Type:  aws.String("binpack"),
   907  			},
   908  		},
   909  		)
   910  		input.SetDesiredCount(d.DesiredCount)
   911  	}
   912  
   913  	if d.SchedulingStrategy != "" {
   914  		input.SetSchedulingStrategy(d.SchedulingStrategy)
   915  	}
   916  
   917  	if strings.ToLower(d.ServiceProtocol) != "none" {
   918  		input.SetLoadBalancers([]*ecs.LoadBalancer{
   919  			{
   920  				ContainerName:  aws.String(e.ServiceName),
   921  				ContainerPort:  aws.Int64(d.ServicePort),
   922  				TargetGroupArn: aws.String(*e.TargetGroupArn),
   923  			},
   924  		})
   925  	}
   926  
   927  	// network configuration
   928  	if d.NetworkMode == "awsvpc" && len(d.NetworkConfiguration.Subnets) > 0 {
   929  		if strings.ToUpper(d.LaunchType) == "FARGATE" {
   930  			input.SetLaunchType("FARGATE")
   931  		}
   932  		input.SetNetworkConfiguration(e.getNetworkConfiguration(d))
   933  	} else {
   934  		// only set role if network mode is not awsvpc (it will be set automatically)
   935  		// only set role if serviceregistry is not defined
   936  		// only set the role if there's a loadbalancer necessary
   937  		if d.ServiceRegistry == "" && strings.ToLower(d.ServiceProtocol) != "none" {
   938  			input.SetRole(util.GetEnv("AWS_ECS_SERVICE_ROLE", "ecs-service-role"))
   939  		}
   940  	}
   941  
   942  	// check whether min/max is set
   943  	dc := &ecs.DeploymentConfiguration{}
   944  	if d.MinimumHealthyPercent > 0 {
   945  		dc.SetMinimumHealthyPercent(d.MinimumHealthyPercent)
   946  	}
   947  	if d.MaximumPercent > 0 {
   948  		dc.SetMaximumPercent(d.MaximumPercent)
   949  	}
   950  	if (ecs.DeploymentConfiguration{}) != *dc {
   951  		input.SetDeploymentConfiguration(dc)
   952  	}
   953  
   954  	// set gracePeriodSeconds
   955  	if d.HealthCheck.GracePeriodSeconds > 0 {
   956  		input.SetHealthCheckGracePeriodSeconds(d.HealthCheck.GracePeriodSeconds)
   957  	}
   958  
   959  	// set ServiceRegistry
   960  	if d.ServiceRegistry != "" && strings.ToLower(d.ServiceProtocol) != "none" {
   961  		sd := ServiceDiscovery{}
   962  		_, serviceDiscoveryNamespaceID, err := sd.getNamespaceArnAndId(d.ServiceRegistry)
   963  		if err != nil {
   964  			ecsLogger.Warningf("Could not apply ServiceRegistry Config: %s", err.Error())
   965  		} else {
   966  			serviceDiscoveryServiceArn, err := sd.getServiceArn(d.ServiceName, serviceDiscoveryNamespaceID)
   967  			if err != nil && strings.HasPrefix(err.Error(), "Service not found") {
   968  				// Service not found, create service in service registry
   969  				serviceDiscoveryServiceArn, err = sd.createService(d.ServiceName, serviceDiscoveryNamespaceID)
   970  			}
   971  			// check for error, else set service registry
   972  			if err != nil && !strings.HasPrefix(err.Error(), "Service not found") {
   973  				ecsLogger.Warningf("Could not get service from ServiceRegistry: %s", err.Error())
   974  			} else {
   975  				ecsLogger.Debugf("Applying ServiceRegistry for %s with Arn %s", e.ServiceName, serviceDiscoveryServiceArn)
   976  				input.SetServiceRegistries([]*ecs.ServiceRegistry{
   977  					{
   978  						ContainerName: aws.String(e.ServiceName),
   979  						ContainerPort: aws.Int64(d.ServicePort),
   980  						RegistryArn:   aws.String(serviceDiscoveryServiceArn),
   981  					},
   982  				})
   983  			}
   984  		}
   985  	}
   986  
   987  	// create service
   988  	_, err := svc.CreateService(input)
   989  	if err != nil {
   990  		if aerr, ok := err.(awserr.Error); ok {
   991  			switch aerr.Code() {
   992  			case ecs.ErrCodeServerException:
   993  				ecsLogger.Errorf(ecs.ErrCodeServerException+": %v", aerr.Error())
   994  			case ecs.ErrCodeClientException:
   995  				ecsLogger.Errorf(ecs.ErrCodeClientException+": %v", aerr.Error())
   996  			case ecs.ErrCodeInvalidParameterException:
   997  				ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException+": %v", aerr.Error())
   998  			case ecs.ErrCodeClusterNotFoundException:
   999  				ecsLogger.Errorf(ecs.ErrCodeClusterNotFoundException+": %v", aerr.Error())
  1000  			default:
  1001  				ecsLogger.Errorf(aerr.Error())
  1002  			}
  1003  		} else {
  1004  			ecsLogger.Errorf(err.Error())
  1005  		}
  1006  		return errors.New("Could not create service")
  1007  	}
  1008  	return nil
  1009  }
  1010  
  1011  // wait until service is inactive
  1012  func (e *ECS) WaitUntilServicesInactive(clusterName, serviceName string) error {
  1013  	svc := ecs.New(session.New())
  1014  	input := &ecs.DescribeServicesInput{
  1015  		Cluster:  aws.String(clusterName),
  1016  		Services: []*string{aws.String(serviceName)},
  1017  	}
  1018  
  1019  	ecsLogger.Debugf("Waiting for service %v on %v to become inactive", serviceName, clusterName)
  1020  
  1021  	err := svc.WaitUntilServicesInactive(input)
  1022  	if err != nil {
  1023  		if aerr, ok := err.(awserr.Error); ok {
  1024  			ecsLogger.Errorf(aerr.Error())
  1025  		} else {
  1026  			ecsLogger.Errorf(err.Error())
  1027  		}
  1028  		return err
  1029  	}
  1030  	return nil
  1031  }
  1032  
  1033  // wait until service is stable
  1034  func (e *ECS) WaitUntilServicesStable(clusterName, serviceName string, maxWaitMinutes int) error {
  1035  	svc := ecs.New(session.New())
  1036  	maxAttempts := maxWaitMinutes * 4
  1037  	input := &ecs.DescribeServicesInput{
  1038  		Cluster:  aws.String(clusterName),
  1039  		Services: []*string{aws.String(serviceName)},
  1040  	}
  1041  
  1042  	ecsLogger.Debugf("Waiting for service %v on %v to become stable", serviceName, clusterName)
  1043  
  1044  	err := svc.WaitUntilServicesStableWithContext(context.Background(), input, request.WithWaiterMaxAttempts(maxAttempts))
  1045  	if err != nil {
  1046  		if aerr, ok := err.(awserr.Error); ok {
  1047  			ecsLogger.Errorf(aerr.Error())
  1048  		} else {
  1049  			ecsLogger.Errorf(err.Error())
  1050  		}
  1051  		return err
  1052  	}
  1053  	return nil
  1054  }
  1055  
  1056  func (e *ECS) getMaxWaitMinutes(gracePeriodSeconds int64) int {
  1057  	// check whether service exists, otherwise wait might give error
  1058  	if maxWaitSecondsString := util.GetEnv("DEPLOY_MAX_WAIT_SECONDS", "900"); maxWaitSecondsString != "900" {
  1059  		maxWaitSeconds, err := strconv.Atoi(maxWaitSecondsString)
  1060  		if err != nil {
  1061  			return 15
  1062  		} else {
  1063  			return int(math.Ceil(float64(maxWaitSeconds) / 60))
  1064  		}
  1065  	} else {
  1066  		if gracePeriodSeconds > 0 {
  1067  			return (1 + int(math.Ceil(float64(gracePeriodSeconds)/60/10))) * 10
  1068  		}
  1069  	}
  1070  	return 15
  1071  }
  1072  
  1073  func (e *ECS) LaunchWaitUntilServicesStable(dd, ddLast *service.DynamoDeployment, notification integrations.Notification) error {
  1074  	var failed bool
  1075  
  1076  	s := service.NewService()
  1077  	err := e.WaitUntilServicesStable(dd.DeployData.Cluster, dd.ServiceName, e.getMaxWaitMinutes(dd.DeployData.HealthCheck.GracePeriodSeconds))
  1078  	if err != nil {
  1079  		ecsLogger.Debugf("waitUntilServiceStable didn't succeed: %v", err)
  1080  		failed = true
  1081  	}
  1082  	// check whether deployment has latest task definition
  1083  	runningService, err := e.DescribeService(dd.DeployData.Cluster, dd.ServiceName, false, true, true)
  1084  	if err != nil {
  1085  		return err
  1086  	}
  1087  	if len(runningService.Deployments) != 1 {
  1088  		reason := "Deployment failed: deployment was still running after 10 minutes"
  1089  		ecsLogger.Debugf(reason)
  1090  		err := s.SetDeploymentStatusWithReason(dd, "failed", reason)
  1091  		if err != nil {
  1092  			return err
  1093  		}
  1094  		err = notification.LogFailure(dd.ServiceName + ": " + reason)
  1095  		if err != nil {
  1096  			ecsLogger.Errorf("Could not send notification: %s", err)
  1097  		}
  1098  		err = e.Rollback(dd.DeployData.Cluster, dd.ServiceName)
  1099  		if err != nil {
  1100  			return err
  1101  		}
  1102  		return nil
  1103  	}
  1104  	if runningService.Deployments[0].TaskDefinition != *dd.TaskDefinitionArn {
  1105  		reason := "Deployment failed: Still running old task definition"
  1106  		ecsLogger.Debugf(reason)
  1107  		err := s.SetDeploymentStatusWithReason(dd, "failed", reason)
  1108  		if err != nil {
  1109  			return err
  1110  		}
  1111  		err = notification.LogFailure(dd.ServiceName + ": " + reason)
  1112  		if err != nil {
  1113  			ecsLogger.Errorf("Could not send notification: %s", err)
  1114  		}
  1115  		err = e.Rollback(dd.DeployData.Cluster, dd.ServiceName)
  1116  		if err != nil {
  1117  			return err
  1118  		}
  1119  		return nil
  1120  	}
  1121  	if len(runningService.Tasks) == 0 {
  1122  		reason := "Deployment failed: no tasks running"
  1123  		ecsLogger.Debugf(reason)
  1124  		err := s.SetDeploymentStatusWithReason(dd, "failed", reason)
  1125  		if err != nil {
  1126  			return err
  1127  		}
  1128  		err = notification.LogFailure(dd.ServiceName + ": " + reason)
  1129  		if err != nil {
  1130  			ecsLogger.Errorf("Could not send notification: %s", err)
  1131  		}
  1132  		err = e.Rollback(dd.DeployData.Cluster, dd.ServiceName)
  1133  		if err != nil {
  1134  			return err
  1135  		}
  1136  		return nil
  1137  	}
  1138  	if failed {
  1139  		reason := "Deployment timed out"
  1140  		s.SetDeploymentStatusWithReason(dd, "failed", reason)
  1141  		err = notification.LogFailure(dd.ServiceName + ": " + reason)
  1142  		if err != nil {
  1143  			ecsLogger.Errorf("Could not send notification: %s", err)
  1144  		}
  1145  		return nil
  1146  	}
  1147  	// set success
  1148  	s.SetDeploymentStatus(dd, "success")
  1149  	if ddLast != nil && ddLast.Status != "success" && ddLast.Status != "aborted" {
  1150  		err = notification.LogRecovery(dd.ServiceName + ": Deployed successfully")
  1151  		if err != nil {
  1152  			ecsLogger.Errorf("Could not send notification: %s", err)
  1153  		}
  1154  	}
  1155  	return nil
  1156  }
  1157  func (e *ECS) Rollback(clusterName, serviceName string) error {
  1158  	ecsLogger.Debugf("Starting rollback")
  1159  	s := service.NewService()
  1160  	s.ServiceName = serviceName
  1161  	dd, err := s.GetDeploys("secondToLast", 1)
  1162  	if err != nil {
  1163  		ecsLogger.Errorf("Error: %v", err.Error())
  1164  		return err
  1165  	}
  1166  	if len(dd) == 0 || dd[0].Status != "success" {
  1167  		ecsLogger.Debugf("Rollback: Previous deploy was not successful")
  1168  		dd, err := s.GetDeploys("byMonth", 10)
  1169  		if err != nil {
  1170  			return err
  1171  		}
  1172  		ecsLogger.Debugf("Rollback: checking last %d deploys", len(dd))
  1173  	}
  1174  	for _, v := range dd {
  1175  		ecsLogger.Debugf("Looping previous deployments: %v with status %v", *v.TaskDefinitionArn, v.Status)
  1176  		if v.Status == "success" {
  1177  			ecsLogger.Debugf("Rollback: rolling back to %v", *v.TaskDefinitionArn)
  1178  			e.UpdateService(v.ServiceName, v.TaskDefinitionArn, *v.DeployData)
  1179  			return nil
  1180  		}
  1181  	}
  1182  	ecsLogger.Debugf("Could not rollback, no stable version found")
  1183  	return errors.New("Could not rollback, no stable version found")
  1184  }
  1185  
  1186  // describe services
  1187  func (e *ECS) DescribeService(clusterName string, serviceName string, showEvents bool, showTasks bool, showStoppedTasks bool) (service.RunningService, error) {
  1188  	s, err := e.DescribeServices(clusterName, []*string{aws.String(serviceName)}, showEvents, showTasks, showStoppedTasks)
  1189  	if err == nil && len(s) == 1 {
  1190  		return s[0], nil
  1191  	} else {
  1192  		if err == nil {
  1193  			return service.RunningService{}, errors.New("describeService: No error, but array length != 1")
  1194  		} else {
  1195  			return service.RunningService{}, err
  1196  		}
  1197  	}
  1198  }
  1199  func (e *ECS) DescribeServices(clusterName string, serviceNames []*string, showEvents bool, showTasks bool, showStoppedTasks bool) ([]service.RunningService, error) {
  1200  	return e.DescribeServicesWithOptions(clusterName, serviceNames, showEvents, showTasks, showStoppedTasks, map[string]string{})
  1201  }
  1202  func (e *ECS) DescribeServicesWithOptions(clusterName string, serviceNames []*string, showEvents bool, showTasks bool, showStoppedTasks bool, options map[string]string) ([]service.RunningService, error) {
  1203  	var rss []service.RunningService
  1204  	svc := ecs.New(session.New())
  1205  
  1206  	// fetch per 10
  1207  	var y float64 = float64(len(serviceNames)) / 10
  1208  	for i := 0; i < int(math.Ceil(y)); i++ {
  1209  
  1210  		f := i * 10
  1211  		t := int(math.Min(float64(10+10*i), float64(len(serviceNames))))
  1212  
  1213  		input := &ecs.DescribeServicesInput{
  1214  			Cluster:  aws.String(clusterName),
  1215  			Services: serviceNames[f:t],
  1216  		}
  1217  
  1218  		result, err := svc.DescribeServices(input)
  1219  		if err != nil {
  1220  			if aerr, ok := err.(awserr.Error); ok {
  1221  				ecsLogger.Errorf(aerr.Error())
  1222  			} else {
  1223  				ecsLogger.Errorf(err.Error())
  1224  			}
  1225  			return rss, err
  1226  		}
  1227  		for _, ecsService := range result.Services {
  1228  			rs := service.RunningService{ServiceName: *ecsService.ServiceName, ClusterName: clusterName}
  1229  			rs.RunningCount = *ecsService.RunningCount
  1230  			rs.PendingCount = *ecsService.PendingCount
  1231  			rs.DesiredCount = *ecsService.DesiredCount
  1232  			rs.Status = *ecsService.Status
  1233  			for _, deployment := range ecsService.Deployments {
  1234  				var ds service.RunningServiceDeployment
  1235  				ds.Status = *deployment.Status
  1236  				ds.RunningCount = *deployment.RunningCount
  1237  				ds.PendingCount = *deployment.PendingCount
  1238  				ds.DesiredCount = *deployment.DesiredCount
  1239  				ds.CreatedAt = *deployment.CreatedAt
  1240  				ds.UpdatedAt = *deployment.UpdatedAt
  1241  				ds.TaskDefinition = *deployment.TaskDefinition
  1242  				rs.Deployments = append(rs.Deployments, ds)
  1243  			}
  1244  			if showEvents {
  1245  				for _, event := range ecsService.Events {
  1246  					event := service.RunningServiceEvent{
  1247  						Id:        *event.Id,
  1248  						CreatedAt: *event.CreatedAt,
  1249  						Message:   *event.Message,
  1250  					}
  1251  					rs.Events = append(rs.Events, event)
  1252  				}
  1253  			}
  1254  			if showTasks {
  1255  				taskArns, err := e.ListTasks(clusterName, *ecsService.ServiceName, "RUNNING", "service")
  1256  				if err != nil {
  1257  					return rss, err
  1258  				}
  1259  				if showStoppedTasks {
  1260  					taskArnsStopped, err := e.ListTasks(clusterName, *ecsService.ServiceName, "STOPPED", "service")
  1261  					if err != nil {
  1262  						return rss, err
  1263  					}
  1264  					taskArns = append(taskArns, taskArnsStopped...)
  1265  				}
  1266  				runningTasks, err := e.DescribeTasks(clusterName, taskArns)
  1267  				if err != nil {
  1268  					return rss, err
  1269  				}
  1270  				rs.Tasks = runningTasks
  1271  			}
  1272  			rss = append(rss, rs)
  1273  		}
  1274  		// check whether to sleep between calls
  1275  		for k, v := range options {
  1276  			if k == "sleep" {
  1277  				seconds, err := strconv.Atoi(v)
  1278  				if err != nil {
  1279  					return rss, fmt.Errorf("Couldn't convert sleep value to int (in options)")
  1280  				}
  1281  				time.Sleep(time.Duration(seconds) * time.Second)
  1282  			}
  1283  		}
  1284  	}
  1285  	return rss, nil
  1286  }
  1287  
  1288  // list tasks
  1289  func (e *ECS) ListTasks(clusterName, name, desiredStatus, filterBy string) ([]*string, error) {
  1290  	svc := ecs.New(session.New())
  1291  	var tasks []*string
  1292  
  1293  	input := &ecs.ListTasksInput{
  1294  		Cluster: aws.String(clusterName),
  1295  	}
  1296  	if filterBy == "service" {
  1297  		input.SetServiceName(name)
  1298  	} else if filterBy == "family" {
  1299  		input.SetFamily(name)
  1300  	} else {
  1301  		return tasks, errors.New("Invalid filterBy")
  1302  	}
  1303  	if desiredStatus == "STOPPED" {
  1304  		input.SetDesiredStatus(desiredStatus)
  1305  	}
  1306  
  1307  	pageNum := 0
  1308  	err := svc.ListTasksPages(input,
  1309  		func(page *ecs.ListTasksOutput, lastPage bool) bool {
  1310  			pageNum++
  1311  			tasks = append(tasks, page.TaskArns...)
  1312  			return pageNum <= 100
  1313  		})
  1314  
  1315  	if err != nil {
  1316  		if aerr, ok := err.(awserr.Error); ok {
  1317  			ecsLogger.Errorf(aerr.Error())
  1318  		} else {
  1319  			ecsLogger.Errorf(err.Error())
  1320  		}
  1321  	}
  1322  	return tasks, err
  1323  }
  1324  func (e *ECS) DescribeTasks(clusterName string, tasks []*string) ([]service.RunningTask, error) {
  1325  	var rts []service.RunningTask
  1326  	svc := ecs.New(session.New())
  1327  
  1328  	// fetch per 100
  1329  	var y float64 = float64(len(tasks)) / 100
  1330  	for i := 0; i < int(math.Ceil(y)); i++ {
  1331  
  1332  		f := i * 100
  1333  		t := int(math.Min(float64(100+100*i), float64(len(tasks))))
  1334  
  1335  		input := &ecs.DescribeTasksInput{
  1336  			Cluster: aws.String(clusterName),
  1337  			Tasks:   tasks[f:t],
  1338  		}
  1339  
  1340  		result, err := svc.DescribeTasks(input)
  1341  		if err != nil {
  1342  			if aerr, ok := err.(awserr.Error); ok {
  1343  				ecsLogger.Errorf(aerr.Error())
  1344  			} else {
  1345  				ecsLogger.Errorf(err.Error())
  1346  			}
  1347  			return rts, err
  1348  		}
  1349  		for _, task := range result.Tasks {
  1350  			rs := service.RunningTask{}
  1351  			rs.ContainerInstanceArn = *task.ContainerInstanceArn
  1352  			rs.Cpu = *task.Cpu
  1353  			rs.CreatedAt = *task.CreatedAt
  1354  			rs.DesiredStatus = *task.DesiredStatus
  1355  			if task.ExecutionStoppedAt != nil {
  1356  				rs.ExecutionStoppedAt = *task.ExecutionStoppedAt
  1357  			}
  1358  			if task.Group != nil {
  1359  				rs.Group = *task.Group
  1360  			}
  1361  			rs.LastStatus = *task.LastStatus
  1362  			rs.LaunchType = *task.LaunchType
  1363  			rs.Memory = *task.Memory
  1364  			if task.PullStartedAt != nil {
  1365  				rs.PullStartedAt = *task.PullStartedAt
  1366  			}
  1367  			if task.PullStoppedAt != nil {
  1368  				rs.PullStoppedAt = *task.PullStoppedAt
  1369  			}
  1370  			if task.StartedAt != nil {
  1371  				rs.StartedAt = *task.StartedAt
  1372  			}
  1373  			if task.StartedBy != nil {
  1374  				rs.StartedBy = *task.StartedBy
  1375  			}
  1376  			if task.StoppedAt != nil {
  1377  				rs.StoppedAt = *task.StoppedAt
  1378  			}
  1379  			if task.StoppedReason != nil {
  1380  				rs.StoppedReason = *task.StoppedReason
  1381  			}
  1382  			if task.StoppingAt != nil {
  1383  				rs.StoppingAt = *task.StoppingAt
  1384  			}
  1385  			rs.TaskArn = *task.TaskArn
  1386  			rs.TaskDefinitionArn = *task.TaskDefinitionArn
  1387  			rs.Version = *task.Version
  1388  			for _, container := range task.Containers {
  1389  				var tc service.RunningTaskContainer
  1390  				tc.ContainerArn = *container.ContainerArn
  1391  				if container.ExitCode != nil {
  1392  					tc.ExitCode = *container.ExitCode
  1393  				}
  1394  				if container.LastStatus != nil {
  1395  					tc.LastStatus = *container.LastStatus
  1396  				}
  1397  				tc.Name = *container.Name
  1398  				if container.Reason != nil {
  1399  					tc.Reason = *container.Reason
  1400  				}
  1401  				rs.Containers = append(rs.Containers, tc)
  1402  			}
  1403  			rts = append(rts, rs)
  1404  		}
  1405  	}
  1406  	return rts, nil
  1407  }
  1408  
  1409  func (e *ECS) ListContainerInstances(clusterName string) ([]string, error) {
  1410  	svc := ecs.New(session.New())
  1411  	input := &ecs.ListContainerInstancesInput{
  1412  		Cluster: aws.String(clusterName),
  1413  	}
  1414  	var instanceArns []*string
  1415  
  1416  	pageNum := 0
  1417  	err := svc.ListContainerInstancesPages(input,
  1418  		func(page *ecs.ListContainerInstancesOutput, lastPage bool) bool {
  1419  			pageNum++
  1420  			instanceArns = append(instanceArns, page.ContainerInstanceArns...)
  1421  			return pageNum <= 100
  1422  		})
  1423  
  1424  	if err != nil {
  1425  		if aerr, ok := err.(awserr.Error); ok {
  1426  			ecsLogger.Errorf("%v", aerr.Error())
  1427  		} else {
  1428  			ecsLogger.Errorf("%v", err.Error())
  1429  		}
  1430  		return aws.StringValueSlice(instanceArns), err
  1431  	}
  1432  	return aws.StringValueSlice(instanceArns), nil
  1433  }
  1434  
  1435  // describe container instances
  1436  func (e *ECS) DescribeContainerInstances(clusterName string, containerInstances []string) ([]ContainerInstance, error) {
  1437  	var cis []ContainerInstance
  1438  	svc := ecs.New(session.New())
  1439  	input := &ecs.DescribeContainerInstancesInput{
  1440  		Cluster:            aws.String(clusterName),
  1441  		ContainerInstances: aws.StringSlice(containerInstances),
  1442  	}
  1443  
  1444  	result, err := svc.DescribeContainerInstances(input)
  1445  	if err != nil {
  1446  		if aerr, ok := err.(awserr.Error); ok {
  1447  			ecsLogger.Errorf(aerr.Error())
  1448  		} else {
  1449  			ecsLogger.Errorf(err.Error())
  1450  		}
  1451  		return cis, err
  1452  	}
  1453  	if len(result.ContainerInstances) == 0 {
  1454  		return cis, errors.New("No container instances returned")
  1455  	}
  1456  	for _, ci := range result.ContainerInstances {
  1457  		var c ContainerInstance
  1458  		c.ContainerInstanceArn = aws.StringValue(ci.ContainerInstanceArn)
  1459  		c.Ec2InstanceId = aws.StringValue(ci.Ec2InstanceId)
  1460  		c.PendingTasksCount = aws.Int64Value(ci.PendingTasksCount)
  1461  		c.RegisteredAt = aws.TimeValue(ci.RegisteredAt)
  1462  		c.RunningTasksCount = aws.Int64Value(ci.RunningTasksCount)
  1463  		c.Status = aws.StringValue(ci.Status)
  1464  		c.Version = aws.Int64Value(ci.Version)
  1465  		for _, v := range ci.RegisteredResources {
  1466  			var vv ContainerInstanceResource
  1467  			switch aws.StringValue(v.Type) {
  1468  			case "INTEGER":
  1469  				vv.IntegerValue = aws.Int64Value(v.IntegerValue)
  1470  			case "DOUBLE":
  1471  				vv.DoubleValue = aws.Float64Value(v.DoubleValue)
  1472  			case "LONG":
  1473  				vv.IntegerValue = aws.Int64Value(v.IntegerValue)
  1474  			case "STRINGSET":
  1475  				vv.StringSetValue = aws.StringValueSlice(v.StringSetValue)
  1476  			}
  1477  			vv.Name = aws.StringValue(v.Name)
  1478  			vv.Type = aws.StringValue(v.Type)
  1479  			c.RegisteredResources = append(c.RegisteredResources, vv)
  1480  		}
  1481  		for _, v := range ci.RemainingResources {
  1482  			var vv ContainerInstanceResource
  1483  			switch aws.StringValue(v.Type) {
  1484  			case "INTEGER":
  1485  				vv.IntegerValue = aws.Int64Value(v.IntegerValue)
  1486  			case "DOUBLE":
  1487  				vv.DoubleValue = aws.Float64Value(v.DoubleValue)
  1488  			case "LONG":
  1489  				vv.IntegerValue = aws.Int64Value(v.IntegerValue)
  1490  			case "STRINGSET":
  1491  				vv.StringSetValue = aws.StringValueSlice(v.StringSetValue)
  1492  			}
  1493  			vv.Name = aws.StringValue(v.Name)
  1494  			vv.Type = aws.StringValue(v.Type)
  1495  			c.RemainingResources = append(c.RemainingResources, vv)
  1496  		}
  1497  		// get AZ
  1498  		for _, ciAttr := range ci.Attributes {
  1499  			if aws.StringValue(ciAttr.Name) == "ecs.availability-zone" {
  1500  				c.AvailabilityZone = aws.StringValue(ciAttr.Value)
  1501  			}
  1502  		}
  1503  		cis = append(cis, c)
  1504  	}
  1505  	return cis, nil
  1506  }
  1507  
  1508  // manual scale ECS service
  1509  func (e *ECS) ManualScaleService(clusterName, serviceName string, desiredCount int64) error {
  1510  	svc := ecs.New(session.New())
  1511  	input := &ecs.UpdateServiceInput{
  1512  		Cluster:      aws.String(clusterName),
  1513  		Service:      aws.String(serviceName),
  1514  		DesiredCount: aws.Int64(desiredCount),
  1515  	}
  1516  
  1517  	ecsLogger.Debugf("Manually scaling %v to a count of %d", serviceName, desiredCount)
  1518  
  1519  	_, err := svc.UpdateService(input)
  1520  	if err != nil {
  1521  		if aerr, ok := err.(awserr.Error); ok {
  1522  			ecsLogger.Errorf(aerr.Error())
  1523  		} else {
  1524  			ecsLogger.Errorf(err.Error())
  1525  		}
  1526  		return err
  1527  	}
  1528  	return nil
  1529  }
  1530  
  1531  func (e *ECS) getNetworkConfiguration(d service.Deploy) *ecs.NetworkConfiguration {
  1532  	var sns []*string
  1533  	var sgs []*string
  1534  	var aIp string
  1535  	nc := &ecs.NetworkConfiguration{AwsvpcConfiguration: &ecs.AwsVpcConfiguration{}}
  1536  	ec2 := EC2{}
  1537  	for i := range d.NetworkConfiguration.Subnets {
  1538  		if strings.HasPrefix(d.NetworkConfiguration.Subnets[i], "subnet-") {
  1539  			sns = append(sns, &d.NetworkConfiguration.Subnets[i])
  1540  		} else {
  1541  			subnetID, err := ec2.GetSubnetID(d.NetworkConfiguration.Subnets[i])
  1542  			if err != nil {
  1543  				ecsLogger.Errorf("Couldn't retrieve subnet name %s: %s", d.NetworkConfiguration.Subnets[i], err)
  1544  			} else {
  1545  				sns = append(sns, &subnetID)
  1546  			}
  1547  
  1548  		}
  1549  	}
  1550  	nc.AwsvpcConfiguration.SetSubnets(sns)
  1551  	for i := range d.NetworkConfiguration.SecurityGroups {
  1552  		if strings.HasPrefix(d.NetworkConfiguration.SecurityGroups[i], "sg-") {
  1553  			sgs = append(sgs, &d.NetworkConfiguration.SecurityGroups[i])
  1554  		} else {
  1555  			securityGroupID, err := ec2.GetSecurityGroupID(d.NetworkConfiguration.SecurityGroups[i])
  1556  			if err != nil {
  1557  				ecsLogger.Errorf("Couldn't retrieve subnet name %s: %s", d.NetworkConfiguration.Subnets[i], err)
  1558  			} else {
  1559  				sgs = append(sgs, &securityGroupID)
  1560  			}
  1561  		}
  1562  	}
  1563  	nc.AwsvpcConfiguration.SetSecurityGroups(sgs)
  1564  	if d.NetworkConfiguration.AssignPublicIp == "" {
  1565  		aIp = "DISABLED"
  1566  	} else {
  1567  		aIp = d.NetworkConfiguration.AssignPublicIp
  1568  	}
  1569  	nc.AwsvpcConfiguration.SetAssignPublicIp(aIp)
  1570  	return nc
  1571  }
  1572  
  1573  // run one-off task
  1574  func (e *ECS) RunTask(clusterName, taskDefinition string, runTask service.RunTask, d service.Deploy) (string, error) {
  1575  	var taskArn string
  1576  	svc := ecs.New(session.New())
  1577  	input := &ecs.RunTaskInput{
  1578  		Cluster:        aws.String(clusterName),
  1579  		TaskDefinition: aws.String(taskDefinition),
  1580  		StartedBy:      aws.String(runTask.StartedBy),
  1581  	}
  1582  
  1583  	taskOverride := &ecs.TaskOverride{}
  1584  	var containerOverrides []*ecs.ContainerOverride
  1585  	for _, co := range runTask.ContainerOverrides {
  1586  		// environment variables
  1587  		var environment []*ecs.KeyValuePair
  1588  		if len(co.Environment) > 0 {
  1589  			for _, v := range co.Environment {
  1590  				environment = append(environment, &ecs.KeyValuePair{Name: aws.String(v.Name), Value: aws.String(v.Value)})
  1591  			}
  1592  		}
  1593  
  1594  		containerOverrides = append(containerOverrides, &ecs.ContainerOverride{
  1595  			Command:     aws.StringSlice(co.Command),
  1596  			Name:        aws.String(co.Name),
  1597  			Environment: environment,
  1598  		})
  1599  	}
  1600  	taskOverride.SetContainerOverrides(containerOverrides)
  1601  	input.SetOverrides(taskOverride)
  1602  
  1603  	// network configuration
  1604  	if d.NetworkMode == "awsvpc" && len(d.NetworkConfiguration.Subnets) > 0 {
  1605  		if strings.ToUpper(d.LaunchType) == "FARGATE" {
  1606  			input.SetLaunchType("FARGATE")
  1607  		}
  1608  		input.SetNetworkConfiguration(e.getNetworkConfiguration(d))
  1609  	}
  1610  
  1611  	ecsLogger.Debugf("Running ad-hoc task using taskdef %s and taskoverride: %+v", taskDefinition, taskOverride)
  1612  
  1613  	result, err := svc.RunTask(input)
  1614  	if err != nil {
  1615  		if aerr, ok := err.(awserr.Error); ok {
  1616  			ecsLogger.Errorf(aerr.Error())
  1617  		} else {
  1618  			ecsLogger.Errorf(err.Error())
  1619  		}
  1620  		return taskArn, err
  1621  	}
  1622  	if len(result.Tasks) == 0 {
  1623  		return taskArn, errors.New("No task arn returned")
  1624  	}
  1625  	return aws.StringValue(result.Tasks[0].TaskArn), nil
  1626  }
  1627  
  1628  func (e *ECS) GetTaskDefinition(clusterName, serviceName string) (string, error) {
  1629  	runningService, err := e.DescribeService(clusterName, serviceName, false, false, false)
  1630  	if err != nil {
  1631  		return "", nil
  1632  	}
  1633  	for _, d := range runningService.Deployments {
  1634  		if d.Status == "PRIMARY" {
  1635  			return d.TaskDefinition, nil
  1636  		}
  1637  	}
  1638  	return "", errors.New("No task definition found")
  1639  }
  1640  func (e *ECS) DescribeTaskDefinition(taskDefinitionNameOrArn string) (TaskDefinition, error) {
  1641  	var taskDefinition TaskDefinition
  1642  	svc := ecs.New(session.New())
  1643  	input := &ecs.DescribeTaskDefinitionInput{
  1644  		TaskDefinition: aws.String(taskDefinitionNameOrArn),
  1645  	}
  1646  
  1647  	result, err := svc.DescribeTaskDefinition(input)
  1648  	if err != nil {
  1649  		if aerr, ok := err.(awserr.Error); ok {
  1650  			ecsLogger.Errorf(aerr.Error())
  1651  		} else {
  1652  			ecsLogger.Errorf(err.Error())
  1653  		}
  1654  		return taskDefinition, err
  1655  	}
  1656  
  1657  	taskDefinition.Family = aws.StringValue(result.TaskDefinition.Family)
  1658  	taskDefinition.Revision = aws.Int64Value(result.TaskDefinition.Revision)
  1659  	taskDefinition.ExecutionRoleArn = aws.StringValue(result.TaskDefinition.ExecutionRoleArn)
  1660  	var containerDefinitions []ContainerDefinition
  1661  	for _, cd := range result.TaskDefinition.ContainerDefinitions {
  1662  		var containerDefinition ContainerDefinition
  1663  		containerDefinition.Name = aws.StringValue(cd.Name)
  1664  		containerDefinition.Essential = aws.BoolValue(cd.Essential)
  1665  		containerDefinitions = append(containerDefinitions, containerDefinition)
  1666  	}
  1667  	taskDefinition.ContainerDefinitions = containerDefinitions
  1668  
  1669  	return taskDefinition, nil
  1670  }
  1671  
  1672  func (e *ECS) GetContainerLimits(d service.Deploy) (int64, int64, int64, int64) {
  1673  	var cpuReservation, cpuLimit, memoryReservation, memoryLimit int64
  1674  	for _, c := range d.Containers {
  1675  		if c.MemoryReservation == 0 {
  1676  			memoryReservation += c.Memory
  1677  			memoryLimit += c.Memory
  1678  		} else {
  1679  			memoryReservation += c.MemoryReservation
  1680  			memoryLimit += c.Memory
  1681  		}
  1682  		if c.CPUReservation == 0 {
  1683  			cpuReservation += c.CPU
  1684  			cpuLimit += c.CPU
  1685  		} else {
  1686  			cpuReservation += c.CPUReservation
  1687  			cpuLimit += c.CPU
  1688  		}
  1689  	}
  1690  	return cpuReservation, cpuLimit, memoryReservation, memoryLimit
  1691  }
  1692  func (e *ECS) IsEqualContainerLimits(d1 service.Deploy, d2 service.Deploy) bool {
  1693  	cpuReservation1, cpuLimit1, memoryReservation1, memoryLimit1 := e.GetContainerLimits(d1)
  1694  	cpuReservation2, cpuLimit2, memoryReservation2, memoryLimit2 := e.GetContainerLimits(d2)
  1695  	if cpuReservation1 == cpuReservation2 && cpuLimit1 == cpuLimit2 && memoryReservation1 == memoryReservation2 && memoryLimit1 == memoryLimit2 {
  1696  		return true
  1697  	} else {
  1698  		return false
  1699  	}
  1700  }
  1701  
  1702  func (e *ECS) GetInstanceResources(clusterName string) ([]FreeInstanceResource, []RegisteredInstanceResource, error) {
  1703  	var firs []FreeInstanceResource
  1704  	var rirs []RegisteredInstanceResource
  1705  	ciArns, err := e.ListContainerInstances(clusterName)
  1706  	if err != nil {
  1707  		return firs, rirs, err
  1708  	}
  1709  	cis, err := e.DescribeContainerInstances(clusterName, ciArns)
  1710  	if err != nil {
  1711  		return firs, rirs, err
  1712  	}
  1713  	for _, ci := range cis {
  1714  		// free resources
  1715  		fir, err := e.ConvertResourceToFir(ci.RemainingResources)
  1716  		if err != nil {
  1717  			return firs, rirs, err
  1718  		}
  1719  		fir.InstanceId = ci.Ec2InstanceId
  1720  		fir.AvailabilityZone = ci.AvailabilityZone
  1721  		fir.Status = ci.Status
  1722  		firs = append(firs, fir)
  1723  		// registered resources
  1724  		rir, err := e.ConvertResourceToRir(ci.RegisteredResources)
  1725  		if err != nil {
  1726  			return firs, rirs, err
  1727  		}
  1728  		rir.InstanceId = ci.Ec2InstanceId
  1729  		rirs = append(rirs, rir)
  1730  	}
  1731  	return firs, rirs, nil
  1732  }
  1733  func (e *ECS) ConvertResourceToFir(cir []ContainerInstanceResource) (FreeInstanceResource, error) {
  1734  	var fir FreeInstanceResource
  1735  	for _, v := range cir {
  1736  		if v.Name == "MEMORY" {
  1737  			if v.Type != "INTEGER" && v.Type != "LONG" {
  1738  				return fir, errors.New("Memory return wrong type (" + v.Type + ")")
  1739  			}
  1740  			fir.FreeMemory = v.IntegerValue
  1741  		}
  1742  		if v.Name == "CPU" {
  1743  			if v.Type != "INTEGER" && v.Type != "LONG" {
  1744  				return fir, errors.New("CPU return wrong type (" + v.Type + ")")
  1745  			}
  1746  			fir.FreeCpu = v.IntegerValue
  1747  		}
  1748  	}
  1749  	return fir, nil
  1750  }
  1751  func (e *ECS) ConvertResourceToRir(cir []ContainerInstanceResource) (RegisteredInstanceResource, error) {
  1752  	var rir RegisteredInstanceResource
  1753  	for _, v := range cir {
  1754  		if v.Name == "MEMORY" {
  1755  			if v.Type != "INTEGER" && v.Type != "LONG" {
  1756  				return rir, errors.New("Memory return wrong type (" + v.Type + ")")
  1757  			}
  1758  			rir.RegisteredMemory = v.IntegerValue
  1759  		}
  1760  		if v.Name == "CPU" {
  1761  			if v.Type != "INTEGER" && v.Type != "LONG" {
  1762  				return rir, errors.New("CPU return wrong type (" + v.Type + ")")
  1763  			}
  1764  			rir.RegisteredCpu = v.IntegerValue
  1765  		}
  1766  	}
  1767  	return rir, nil
  1768  }
  1769  
  1770  func (e *ECS) DrainNode(clusterName, instance string) error {
  1771  	svc := ecs.New(session.New())
  1772  	input := &ecs.UpdateContainerInstancesStateInput{
  1773  		Cluster:            aws.String(clusterName),
  1774  		ContainerInstances: aws.StringSlice([]string{instance}),
  1775  		Status:             aws.String("DRAINING"),
  1776  	}
  1777  	_, err := svc.UpdateContainerInstancesState(input)
  1778  	if err != nil {
  1779  		if aerr, ok := err.(awserr.Error); ok {
  1780  			ecsLogger.Errorf("%v", aerr.Error())
  1781  		} else {
  1782  			ecsLogger.Errorf("%v", err.Error())
  1783  		}
  1784  		return err
  1785  	}
  1786  	return nil
  1787  }
  1788  func (e *ECS) GetClusterNameByInstanceId(instance string) (string, error) {
  1789  	var clusterName string
  1790  	svc := ec2.New(session.New())
  1791  	input := &ec2.DescribeTagsInput{
  1792  		Filters: []*ec2.Filter{
  1793  			{
  1794  				Name: aws.String("resource-id"),
  1795  				Values: []*string{
  1796  					aws.String(instance),
  1797  				},
  1798  			},
  1799  		},
  1800  	}
  1801  
  1802  	result, err := svc.DescribeTags(input)
  1803  	if err != nil {
  1804  		if aerr, ok := err.(awserr.Error); ok {
  1805  			ecsLogger.Errorf("%v", aerr.Error())
  1806  		} else {
  1807  			ecsLogger.Errorf("%v", err.Error())
  1808  		}
  1809  		return clusterName, err
  1810  	}
  1811  	for _, v := range result.Tags {
  1812  		if aws.StringValue(v.Key) == "Cluster" {
  1813  			return aws.StringValue(v.Value), nil
  1814  		}
  1815  	}
  1816  	return clusterName, errors.New("Could not determine clusterName. Is the EC2 instance tagged?")
  1817  }
  1818  func (e *ECS) GetContainerInstanceArnByInstanceId(clusterName, instanceId string) (string, error) {
  1819  	ciArns, err := e.ListContainerInstances(clusterName)
  1820  	if err != nil {
  1821  		return "", err
  1822  	}
  1823  	cis, err := e.DescribeContainerInstances(clusterName, ciArns)
  1824  	if err != nil {
  1825  		return "", err
  1826  	}
  1827  	for _, ci := range cis {
  1828  		if ci.Ec2InstanceId == instanceId {
  1829  			return ci.ContainerInstanceArn, nil
  1830  		}
  1831  	}
  1832  	return "", errors.New("Couldn't find container instance Arn (instanceId=" + instanceId + ")")
  1833  }
  1834  func (e *ECS) LaunchWaitForDrainedNode(clusterName, containerInstanceArn, instanceId, autoScalingGroupName, lifecycleHookName, lifecycleHookToken string) error {
  1835  	var tasksDrained bool
  1836  	var err error
  1837  	for i := 0; i < 80 && !tasksDrained; i++ {
  1838  		cis, err := e.DescribeContainerInstances(clusterName, []string{containerInstanceArn})
  1839  		if err != nil || len(cis) == 0 {
  1840  			ecsLogger.Errorf("launchWaitForDrainedNode: %v", err.Error())
  1841  			return err
  1842  		}
  1843  		ci := cis[0]
  1844  		if ci.RunningTasksCount == 0 {
  1845  			tasksDrained = true
  1846  		} else {
  1847  			ecsLogger.Infof("launchWaitForDrainedNode: still %d tasks running", ci.RunningTasksCount)
  1848  		}
  1849  		time.Sleep(15 * time.Second)
  1850  	}
  1851  	if !tasksDrained {
  1852  		ecsLogger.Errorf("launchWaitForDrainedNode: Not able to drain tasks: timeout of 20m reached")
  1853  	}
  1854  	// CompleteLifeCycleAction
  1855  	autoscaling := AutoScaling{}
  1856  	if lifecycleHookToken == "" {
  1857  		ecsLogger.Debugf("Running completePendingLifecycleAction")
  1858  		err = autoscaling.CompletePendingLifecycleAction(autoScalingGroupName, instanceId, "CONTINUE", lifecycleHookName)
  1859  	} else {
  1860  		ecsLogger.Debugf("Running completeLifecycleAction")
  1861  		err = autoscaling.CompleteLifecycleAction(autoScalingGroupName, instanceId, "CONTINUE", lifecycleHookName, lifecycleHookToken)
  1862  	}
  1863  	if err != nil {
  1864  		ecsLogger.Errorf("launchWaitForDrainedNode: Could not complete life cycle action: %v", err.Error())
  1865  		return err
  1866  	}
  1867  	ecsLogger.Infof("launchWaitForDrainedNode: Node drained, completed lifecycle action")
  1868  	return nil
  1869  }
  1870  
  1871  // list services
  1872  func (e *ECS) ListServices(clusterName string) ([]*string, error) {
  1873  	svc := ecs.New(session.New())
  1874  	var services []*string
  1875  
  1876  	input := &ecs.ListServicesInput{
  1877  		Cluster: aws.String(clusterName),
  1878  	}
  1879  
  1880  	pageNum := 0
  1881  	err := svc.ListServicesPages(input,
  1882  		func(page *ecs.ListServicesOutput, lastPage bool) bool {
  1883  			pageNum++
  1884  			services = append(services, page.ServiceArns...)
  1885  			return pageNum <= 100
  1886  		})
  1887  
  1888  	if err != nil {
  1889  		if aerr, ok := err.(awserr.Error); ok {
  1890  			ecsLogger.Errorf(aerr.Error())
  1891  		} else {
  1892  			ecsLogger.Errorf(err.Error())
  1893  		}
  1894  	}
  1895  	return services, err
  1896  }