github.com/in4it/ecs-deploy@v0.0.42-0.20240508120354-ed77ff16df25/provider/ecs/ecs.go (about) 1 package ecs 2 3 import ( 4 "github.com/aws/aws-sdk-go/aws" 5 "github.com/aws/aws-sdk-go/aws/awserr" 6 "github.com/aws/aws-sdk-go/aws/request" 7 "github.com/aws/aws-sdk-go/aws/session" 8 "github.com/aws/aws-sdk-go/service/ec2" 9 "github.com/aws/aws-sdk-go/service/ecs" 10 "github.com/in4it/ecs-deploy/integrations" 11 "github.com/in4it/ecs-deploy/service" 12 "github.com/in4it/ecs-deploy/util" 13 "github.com/juju/loggo" 14 15 "context" 16 "crypto/rsa" 17 "crypto/x509" 18 "encoding/base64" 19 "encoding/pem" 20 "errors" 21 "fmt" 22 "math" 23 "strconv" 24 "strings" 25 "time" 26 ) 27 28 // logging 29 var ecsLogger = loggo.GetLogger("ecs") 30 31 // ECS struct 32 type ECS struct { 33 ClusterName string 34 ServiceName string 35 IamRoleArn string 36 TaskDefinition *ecs.RegisterTaskDefinitionInput 37 TaskDefArn *string 38 TargetGroupArn *string 39 } 40 41 type ECSIf interface { 42 GetInstanceResources(clusterName string) ([]FreeInstanceResource, []RegisteredInstanceResource, error) 43 } 44 45 // Task definition and Container definition 46 type TaskDefinition struct { 47 Family string `json:"family"` 48 Revision int64 `json:"revision"` 49 ExecutionRoleArn string `json:"executionRole"` 50 ContainerDefinitions []ContainerDefinition `json:"containerDefinitions"` 51 } 52 type ContainerDefinition struct { 53 Name string `json:"name"` 54 Essential bool `json:"essential"` 55 } 56 57 // containerInstance 58 type ContainerInstance struct { 59 ContainerInstanceArn string 60 Ec2InstanceId string 61 AvailabilityZone string 62 PendingTasksCount int64 63 RegisteredAt time.Time 64 RegisteredResources []ContainerInstanceResource 65 RemainingResources []ContainerInstanceResource 66 RunningTasksCount int64 67 Status string 68 Version int64 69 } 70 type ContainerInstanceResource struct { 71 DoubleValue float64 `json:"doubleValue"` 72 IntegerValue int64 `json:"integerValue"` 73 Name string `json:"name"` 74 StringSetValue []string `json:"stringSetValue"` 75 Type string `json:"type"` 76 } 77 78 // free instance resource 79 type FreeInstanceResource struct { 80 InstanceId string 81 AvailabilityZone string 82 Status string 83 FreeMemory int64 84 FreeCpu int64 85 } 86 87 // registered instance resource 88 type RegisteredInstanceResource struct { 89 InstanceId string 90 RegisteredMemory int64 91 RegisteredCpu int64 92 } 93 94 // version info 95 type EcsVersionInfo struct { 96 AgentHash string `json:"agentHash"` 97 AgentVersion string `json:"agentVersion"` 98 DockerVersion string `json:"dockerVersion"` 99 } 100 101 // task metadata 102 type EcsTaskMetadata struct { 103 Cluster string `json:"Cluster"` 104 TaskARN string `json:"TaskARN"` 105 Family string `json:"Family"` 106 Revision string `json:"Revision"` 107 DesiredStatus string `json:"DesiredStatus"` 108 KnownStatus string `json:"KnownStatus"` 109 Containers []EcsTaskMetadataItemContainer `json:"Containers"` 110 } 111 type EcsTaskMetadataItemContainer struct { 112 DockerId string `json:"DockerId"` 113 DockerName string `json:"DockerName"` 114 Name string `json:"Name"` 115 Image string `json:"Image"` 116 ImageID string `json:"ImageID"` 117 Labels map[string]string `json:"Labels"` 118 DesiredStatus string `json:"DesiredStatus"` 119 KnownStatus string `json:"KnownStatus"` 120 CreatedAt time.Time `json:"CreatedAt"` 121 StartedAt time.Time `json:"CreatedAt"` 122 Limits EcsTaskMetadataItemContainerLimits `json:"Limits"` 123 Type string `json:"Type"` 124 Networks []EcsTaskMetadataItemContainerNetworks `json:"Networks"` 125 } 126 type EcsTaskMetadataItemContainerLimits struct { 127 CPU int64 `json:"CPU"` 128 Memory int64 `json:"Memory"` 129 } 130 type EcsTaskMetadataItemContainerNetworks struct { 131 NetworkMode string `json:"NetworkMode"` 132 Ipv4Addresses []string `json:"Ipv4Addresses"` 133 } 134 135 // create cluster 136 func (e *ECS) CreateCluster(clusterName string) (*string, error) { 137 svc := ecs.New(session.New()) 138 createClusterInput := &ecs.CreateClusterInput{ 139 ClusterName: aws.String(clusterName), 140 } 141 142 result, err := svc.CreateCluster(createClusterInput) 143 if err != nil { 144 if aerr, ok := err.(awserr.Error); ok { 145 ecsLogger.Errorf("%v", aerr.Error()) 146 } else { 147 ecsLogger.Errorf("%v", err.Error()) 148 } 149 return nil, err 150 } 151 return result.Cluster.ClusterArn, nil 152 } 153 func (e *ECS) GetECSAMI() (string, error) { 154 var amiId string 155 svc := ec2.New(session.New()) 156 input := &ec2.DescribeImagesInput{ 157 Owners: []*string{aws.String("591542846629")}, // AWS 158 Filters: []*ec2.Filter{ 159 {Name: aws.String("name"), Values: []*string{aws.String("amzn-ami-*-amazon-ecs-optimized")}}, 160 {Name: aws.String("virtualization-type"), Values: []*string{aws.String("hvm")}}, 161 }, 162 } 163 result, err := svc.DescribeImages(input) 164 if err != nil { 165 if aerr, ok := err.(awserr.Error); ok { 166 ecsLogger.Errorf("%v", aerr.Error()) 167 } else { 168 ecsLogger.Errorf("%v", err.Error()) 169 } 170 return amiId, err 171 } 172 if len(result.Images) == 0 { 173 return amiId, errors.New("No ECS AMI found") 174 } 175 layout := "2006-01-02T15:04:05.000Z" 176 var lastTime time.Time 177 for _, v := range result.Images { 178 t, err := time.Parse(layout, *v.CreationDate) 179 if err != nil { 180 return amiId, err 181 } 182 if t.After(lastTime) { 183 lastTime = t 184 amiId = *v.ImageId 185 } 186 } 187 return amiId, nil 188 } 189 func (e *ECS) ImportKeyPair(keyName string, publicKey []byte) error { 190 svc := ec2.New(session.New()) 191 input := &ec2.ImportKeyPairInput{ 192 KeyName: aws.String(keyName), 193 PublicKeyMaterial: publicKey, 194 } 195 _, err := svc.ImportKeyPair(input) 196 if err != nil { 197 if aerr, ok := err.(awserr.Error); ok { 198 ecsLogger.Errorf("%v", aerr.Error()) 199 } else { 200 ecsLogger.Errorf("%v", err.Error()) 201 } 202 return err 203 } 204 return nil 205 } 206 func (e *ECS) GetPubKeyFromPrivateKey(privateKey string) ([]byte, error) { 207 var pubASN1 []byte 208 var key *rsa.PrivateKey 209 block, _ := pem.Decode([]byte(privateKey)) 210 if block == nil { 211 return pubASN1, errors.New("No private key found") 212 } 213 if block.Type != "RSA PRIVATE KEY" { 214 return pubASN1, errors.New("Key not a RSA PRIVATE KEY") 215 } 216 key, err := x509.ParsePKCS1PrivateKey([]byte(block.Bytes)) 217 if err != nil { 218 return pubASN1, err 219 } 220 pubASN1, err = x509.MarshalPKIXPublicKey(&key.PublicKey) 221 if err != nil { 222 return pubASN1, err 223 } 224 return []byte(base64.StdEncoding.EncodeToString(pubASN1)), nil 225 } 226 func (e *ECS) DeleteKeyPair(keyName string) error { 227 svc := ec2.New(session.New()) 228 input := &ec2.DeleteKeyPairInput{ 229 KeyName: aws.String(keyName), 230 } 231 _, err := svc.DeleteKeyPair(input) 232 if err != nil { 233 if aerr, ok := err.(awserr.Error); ok { 234 ecsLogger.Errorf("%v", aerr.Error()) 235 } else { 236 ecsLogger.Errorf("%v", err.Error()) 237 } 238 return err 239 } 240 return nil 241 } 242 243 // delete cluster 244 func (e *ECS) DeleteCluster(clusterName string) error { 245 svc := ecs.New(session.New()) 246 deleteClusterInput := &ecs.DeleteClusterInput{ 247 Cluster: aws.String(clusterName), 248 } 249 250 _, err := svc.DeleteCluster(deleteClusterInput) 251 if err != nil { 252 if aerr, ok := err.(awserr.Error); ok { 253 ecsLogger.Errorf("%v", aerr.Error()) 254 } else { 255 ecsLogger.Errorf("%v", err.Error()) 256 } 257 return err 258 } 259 return nil 260 } 261 262 // Creates ECS repository 263 func (e *ECS) CreateTaskDefinitionInput(d service.Deploy, secrets map[string]string, accountId string) error { 264 e.TaskDefinition = &ecs.RegisterTaskDefinitionInput{ 265 Family: aws.String(e.ServiceName), 266 TaskRoleArn: aws.String(e.IamRoleArn), 267 } 268 269 // set network mode if set 270 if d.NetworkMode != "" { 271 e.TaskDefinition.SetNetworkMode(d.NetworkMode) 272 } 273 274 // placement constraints 275 if len(d.PlacementConstraints) > 0 { 276 var pcs []*ecs.TaskDefinitionPlacementConstraint 277 for _, pc := range d.PlacementConstraints { 278 tdpc := &ecs.TaskDefinitionPlacementConstraint{} 279 if pc.Expression != "" { 280 tdpc.SetExpression(pc.Expression) 281 } 282 if pc.Type != "" { 283 tdpc.SetType(pc.Type) 284 } 285 pcs = append(pcs, tdpc) 286 } 287 e.TaskDefinition.SetPlacementConstraints(pcs) 288 } 289 290 // volumes 291 if len(d.Volumes) > 0 { 292 var volumes []*ecs.Volume 293 for _, vol := range d.Volumes { 294 volume := &ecs.Volume{ 295 Name: aws.String(vol.Name), 296 } 297 if len(vol.Host.SourcePath) > 0 { 298 volume.SetHost(&ecs.HostVolumeProperties{ 299 SourcePath: aws.String(vol.Host.SourcePath), 300 }) 301 } 302 if len(vol.DockerVolumeConfiguration.Scope) > 0 { 303 volumeConfig := &ecs.DockerVolumeConfiguration{ 304 Autoprovision: aws.Bool(vol.DockerVolumeConfiguration.Autoprovision), 305 Driver: aws.String(vol.DockerVolumeConfiguration.Driver), 306 Scope: aws.String(vol.DockerVolumeConfiguration.Scope), 307 } 308 if len(vol.DockerVolumeConfiguration.DriverOpts) > 0 { 309 volumeConfig.SetDriverOpts(aws.StringMap(vol.DockerVolumeConfiguration.DriverOpts)) 310 } 311 if len(vol.DockerVolumeConfiguration.Labels) > 0 { 312 volumeConfig.SetLabels(aws.StringMap(vol.DockerVolumeConfiguration.Labels)) 313 } 314 volume.SetDockerVolumeConfiguration(volumeConfig) 315 } 316 volumes = append(volumes, volume) 317 } 318 e.TaskDefinition.SetVolumes(volumes) 319 } 320 321 // loop over containers 322 for _, container := range d.Containers { 323 324 // prepare image Uri 325 var imageUri string 326 if container.ContainerURI == "" { 327 if container.ContainerImage == "" { 328 imageUri = accountId + ".dkr.ecr." + util.GetEnv("AWS_REGION", "") + ".amazonaws.com" + "/" + container.ContainerName 329 } else { 330 imageUri = accountId + ".dkr.ecr." + util.GetEnv("AWS_REGION", "") + ".amazonaws.com" + "/" + container.ContainerImage 331 } 332 if container.ContainerTag != "" { 333 imageUri += ":" + container.ContainerTag 334 } 335 } else { 336 imageUri = container.ContainerURI 337 } 338 339 // prepare container definition 340 containerDefinition := &ecs.ContainerDefinition{ 341 Name: aws.String(container.ContainerName), 342 Image: aws.String(imageUri), 343 DockerLabels: aws.StringMap(container.DockerLabels), 344 } 345 346 if len(container.HealthCheck.Command) > 0 { 347 healthCheck := &ecs.HealthCheck{ 348 Command: container.HealthCheck.Command, 349 } 350 if container.HealthCheck.Interval > 0 { 351 healthCheck.SetInterval(container.HealthCheck.Interval) 352 } 353 if container.HealthCheck.Retries > 0 { 354 healthCheck.SetRetries(container.HealthCheck.Retries) 355 } 356 if container.HealthCheck.StartPeriod > 0 { 357 healthCheck.SetStartPeriod(container.HealthCheck.StartPeriod) 358 } 359 if container.HealthCheck.Timeout > 0 { 360 healthCheck.SetTimeout(container.HealthCheck.Timeout) 361 } 362 containerDefinition.SetHealthCheck(healthCheck) 363 } 364 // set containerPort if not empty 365 if container.ContainerPort > 0 { 366 if len(container.PortMappings) > 0 { 367 var portMapping []*ecs.PortMapping 368 for _, v := range container.PortMappings { 369 protocol := "tcp" 370 if v.Protocol != "" { 371 protocol = v.Protocol 372 } 373 if v.HostPort > 0 { 374 portMapping = append(portMapping, &ecs.PortMapping{ 375 ContainerPort: aws.Int64(v.ContainerPort), 376 HostPort: aws.Int64(v.HostPort), 377 Protocol: aws.String(protocol), 378 }) 379 } else { 380 portMapping = append(portMapping, &ecs.PortMapping{ 381 ContainerPort: aws.Int64(v.ContainerPort), 382 Protocol: aws.String(protocol), 383 }) 384 } 385 } 386 containerDefinition.SetPortMappings(portMapping) 387 } else { 388 containerDefinition.SetPortMappings([]*ecs.PortMapping{ 389 { 390 ContainerPort: aws.Int64(container.ContainerPort), 391 }, 392 }) 393 } 394 } 395 // set containerCommand if not empty 396 if len(container.ContainerCommand) > 0 { 397 containerDefinition.SetCommand(container.ContainerCommand) 398 } 399 // set containerEntryPoint if not empty 400 if len(container.ContainerEntryPoint) > 0 { 401 containerDefinition.SetEntryPoint(container.ContainerEntryPoint) 402 } 403 // set cloudwacht logs if enabled 404 if util.GetEnv("CLOUDWATCH_LOGS_ENABLED", "no") == "yes" { 405 var logPrefix string 406 if util.GetEnv("CLOUDWATCH_LOGS_PREFIX", "") != "" { 407 logPrefix = util.GetEnv("CLOUDWATCH_LOGS_PREFIX", "") + "-" + util.GetEnv("AWS_ACCOUNT_ENV", "") 408 } 409 containerDefinition.SetLogConfiguration(&ecs.LogConfiguration{ 410 LogDriver: aws.String("awslogs"), 411 Options: map[string]*string{ 412 "awslogs-group": aws.String(logPrefix), 413 "awslogs-region": aws.String(util.GetEnv("AWS_REGION", "")), 414 "awslogs-stream-prefix": aws.String(container.ContainerName), 415 }, 416 }) 417 } 418 // override logconfiguration if set in deploy config 419 if container.LogConfiguration.LogDriver != "" { 420 containerDefinition.SetLogConfiguration(&ecs.LogConfiguration{ 421 LogDriver: aws.String(container.LogConfiguration.LogDriver), 422 }) 423 options := map[string]*string{} 424 if container.LogConfiguration.Options.MaxFile != "" { 425 options["max-file"] = aws.String(container.LogConfiguration.Options.MaxFile) 426 } 427 if container.LogConfiguration.Options.MaxSize != "" { 428 options["max-size"] = aws.String(container.LogConfiguration.Options.MaxSize) 429 } 430 containerDefinition.LogConfiguration.SetOptions(options) 431 } 432 if container.Memory > 0 { 433 containerDefinition.Memory = aws.Int64(container.Memory) 434 } 435 if container.MemoryReservation > 0 { 436 containerDefinition.MemoryReservation = aws.Int64(container.MemoryReservation) 437 } 438 if container.CPU > 0 { 439 containerDefinition.Cpu = aws.Int64(container.CPU) 440 } else { 441 if container.CPU == 0 && util.GetEnv("DEFAULT_CONTAINER_CPU_LIMIT", "") != "" { 442 defaultCpuLimit, err := strconv.ParseInt(util.GetEnv("DEFAULT_CONTAINER_CPU_LIMIT", ""), 10, 64) 443 if err != nil { 444 return err 445 } 446 containerDefinition.Cpu = aws.Int64(defaultCpuLimit) 447 } 448 } 449 450 if container.Essential { 451 containerDefinition.Essential = aws.Bool(container.Essential) 452 } 453 454 // environment variables 455 var environment []*ecs.KeyValuePair 456 if len(container.Environment) > 0 { 457 for _, v := range container.Environment { 458 environment = append(environment, &ecs.KeyValuePair{Name: aws.String(v.Name), Value: aws.String(v.Value)}) 459 } 460 } 461 if util.GetEnv("PARAMSTORE_ENABLED", "no") == "yes" { 462 namespace := d.EnvNamespace 463 if namespace == "" { 464 namespace = e.ServiceName 465 } 466 environment = append(environment, &ecs.KeyValuePair{Name: aws.String("AWS_REGION"), Value: aws.String(util.GetEnv("AWS_REGION", ""))}) 467 environment = append(environment, &ecs.KeyValuePair{Name: aws.String("AWS_ENV_PATH"), Value: aws.String("/" + util.GetEnv("PARAMSTORE_PREFIX", "") + "-" + util.GetEnv("AWS_ACCOUNT_ENV", "") + "/" + namespace + "/")}) 468 } 469 470 if len(environment) > 0 { 471 containerDefinition.SetEnvironment(environment) 472 } 473 474 // ulimits 475 if len(container.Ulimits) > 0 { 476 var us []*ecs.Ulimit 477 for _, u := range container.Ulimits { 478 us = append(us, &ecs.Ulimit{ 479 Name: aws.String(u.Name), 480 SoftLimit: aws.Int64(u.SoftLimit), 481 HardLimit: aws.Int64(u.HardLimit), 482 }) 483 } 484 containerDefinition.SetUlimits(us) 485 } 486 487 // MountPoints 488 if len(container.MountPoints) > 0 { 489 var mps []*ecs.MountPoint 490 for _, mp := range container.MountPoints { 491 mps = append(mps, &ecs.MountPoint{ 492 ContainerPath: aws.String(mp.ContainerPath), 493 SourceVolume: aws.String(mp.SourceVolume), 494 ReadOnly: aws.Bool(mp.ReadOnly), 495 }) 496 } 497 containerDefinition.SetMountPoints(mps) 498 } 499 500 // Links 501 if len(container.Links) > 0 { 502 containerDefinition.SetLinks(container.Links) 503 } 504 505 // inject parameter store entries as secrets 506 if util.GetEnv("PARAMSTORE_INJECT", "no") == "yes" { 507 ecsSecrets := []*ecs.Secret{} 508 for k, v := range secrets { 509 ecsSecrets = append(ecsSecrets, &ecs.Secret{ 510 Name: aws.String(k), 511 ValueFrom: aws.String(v), 512 }) 513 } 514 containerDefinition.SetSecrets(ecsSecrets) 515 } 516 517 e.TaskDefinition.ContainerDefinitions = append(e.TaskDefinition.ContainerDefinitions, containerDefinition) 518 } 519 520 // add execution role 521 if util.GetEnv("PARAMSTORE_INJECT", "no") == "yes" { 522 iam := IAM{} 523 iamExecutionRoleName := util.GetEnv("AWS_ECS_EXECUTION_ROLE", "ecs-"+d.Cluster+"-task-execution-role") 524 iamExecutionRoleArn, err := iam.RoleExists(iamExecutionRoleName) 525 if err != nil { 526 return err 527 } 528 if iamExecutionRoleArn == nil { 529 return fmt.Errorf("Execution role %s not found and PARAMSTORE_INJECT enabled", iamExecutionRoleName) 530 } 531 e.TaskDefinition.SetExecutionRoleArn(aws.StringValue(iamExecutionRoleArn)) 532 } 533 534 // app mesh 535 if d.AppMesh.Name != "" && d.NetworkMode == "awsvpc" { 536 a := AppMesh{} 537 virtualNodeName := d.ServiceName 538 virtualNodeDNS := strings.ToLower(d.ServiceName + "." + d.ServiceRegistry) 539 virtualServiceName := strings.ToLower(d.ServiceName + "." + d.ServiceRegistry) 540 541 // list virtual nodes and services 542 virtualNodes, err := a.listVirtualNodes(d.AppMesh.Name) 543 if err != nil { 544 return err 545 } 546 virtualServices, err := a.listVirtualServices(d.AppMesh.Name) 547 if err != nil { 548 return err 549 } 550 // get healthcheck object 551 healthCheck, err := e.prepareAppMeshHealthcheck(d.HealthCheck, d.ServicePort, d.ServiceProtocol) 552 if err != nil { 553 return err 554 } 555 556 // create virtual node if it doesn't exist yet 557 if _, ok := virtualNodes[virtualNodeName]; !ok { 558 if err := a.createVirtualNode(virtualNodeName, virtualNodeDNS, d.AppMesh.Name, d.ServicePort, healthCheck, d.AppMesh.Backends); err != nil { 559 return err 560 } 561 } else { 562 // update 563 if err := a.updateVirtualNode(virtualNodeName, virtualNodeDNS, d.AppMesh.Name, d.ServicePort, healthCheck, d.AppMesh.Backends); err != nil { 564 return err 565 } 566 } 567 568 // retry policy 569 if d.AppMesh.RetryPolicy.MaxRetries > 0 { 570 virtualRouters, err := a.listVirtualRouters(d.AppMesh.Name) 571 if err != nil { 572 return err 573 } 574 virtualRouterName := "retries_" + d.ServiceName 575 if _, ok := virtualRouters[virtualRouterName]; !ok { 576 if err := a.createVirtualRouter(virtualRouterName, d.AppMesh.Name, d.ServicePort); err != nil { 577 return err 578 } 579 if err := a.createRoute("retries", virtualRouterName, virtualNodeName, virtualNodeDNS, d.AppMesh); err != nil { 580 return err 581 } 582 } 583 // create virtual service if it doesn't exist yet, or update existing with new virtualRouter 584 if _, ok := virtualServices[virtualServiceName]; !ok { 585 if err := a.createVirtualServiceWithVirtualRouter(virtualServiceName, virtualRouterName, d.AppMesh.Name); err != nil { 586 return err 587 } 588 } else { 589 if err := a.updateVirtualServiceWithVirtualRouter(virtualServiceName, virtualRouterName, d.AppMesh.Name); err != nil { 590 return err 591 } 592 } 593 } else { 594 // create virtual service if it doesn't exist yet 595 if _, ok := virtualServices[virtualServiceName]; !ok { 596 if err := a.createVirtualServiceWithVirtualNode(virtualServiceName, virtualNodeName, d.AppMesh.Name); err != nil { 597 return err 598 } 599 } 600 } 601 602 proxyConfiguration := &ecs.ProxyConfiguration{ 603 Type: aws.String("APPMESH"), 604 ContainerName: aws.String("envoy"), 605 Properties: []*ecs.KeyValuePair{ 606 { 607 Name: aws.String("IgnoredUID"), 608 Value: aws.String("1337"), 609 }, 610 { 611 Name: aws.String("ProxyIngressPort"), 612 Value: aws.String("15000"), 613 }, 614 { 615 Name: aws.String("ProxyEgressPort"), 616 Value: aws.String("15001"), 617 }, 618 { 619 Name: aws.String("AppPorts"), 620 Value: aws.String(strconv.FormatInt(d.ServicePort, 10)), 621 }, 622 { 623 Name: aws.String("EgressIgnoredIPs"), 624 Value: aws.String("169.254.170.2,169.254.169.254"), 625 }, 626 }, 627 } 628 if len(d.AppMesh.Configuration.EgressIgnoredPorts) != 0 { 629 proxyConfiguration.Properties = append(proxyConfiguration.Properties, &ecs.KeyValuePair{ 630 Name: aws.String("EgressIgnoredPorts"), 631 Value: aws.String(strings.Join(d.AppMesh.Configuration.EgressIgnoredPorts, ",")), 632 }) 633 } 634 e.TaskDefinition.SetProxyConfiguration(proxyConfiguration) 635 for k := range e.TaskDefinition.ContainerDefinitions { 636 e.TaskDefinition.ContainerDefinitions[k].SetDependsOn([]*ecs.ContainerDependency{ 637 { 638 Condition: aws.String("HEALTHY"), 639 ContainerName: aws.String("envoy"), 640 }, 641 }) 642 } 643 e.TaskDefinition.ContainerDefinitions = append(e.TaskDefinition.ContainerDefinitions, &ecs.ContainerDefinition{ 644 Name: aws.String("envoy"), 645 Image: aws.String(util.GetEnv("APPMESH_IMAGE", "111345817488.dkr.ecr."+util.GetEnv("AWS_REGION", "us-west-2")+".amazonaws.com/aws-appmesh-envoy:v1.11.1.1-prod")), 646 Essential: aws.Bool(true), 647 MemoryReservation: aws.Int64(256), 648 Environment: []*ecs.KeyValuePair{ 649 { 650 Name: aws.String("APPMESH_VIRTUAL_NODE_NAME"), 651 Value: aws.String("mesh/" + d.AppMesh.Name + "/virtualNode/" + virtualNodeName), 652 }, 653 }, 654 HealthCheck: &ecs.HealthCheck{ 655 Command: aws.StringSlice([]string{"CMD-SHELL", "curl -s http://localhost:9901/server_info | grep state | grep -q LIVE"}), 656 StartPeriod: aws.Int64(10), 657 Interval: aws.Int64(5), 658 Timeout: aws.Int64(2), 659 Retries: aws.Int64(3), 660 }, 661 User: aws.String("1337"), 662 }) 663 } 664 665 return nil 666 } 667 668 func (e *ECS) prepareAppMeshHealthcheck(healthCheck service.DeployHealthCheck, servicePort int64, serviceProtocol string) (AppMeshHealthCheck, error) { 669 var healthCheckPort int64 670 var err error 671 if healthCheck.HealthyThreshold == 0 { 672 healthCheck.HealthyThreshold = 3 673 } 674 if healthCheck.Interval == 0 { 675 healthCheck.Interval = 60 676 } 677 if healthCheck.Path == "" && strings.ToLower(healthCheck.Protocol) == "http" { 678 return AppMeshHealthCheck{}, fmt.Errorf("Healthcheck path must be set when enabling AppMesh and protocol is http") 679 } 680 if healthCheck.Port == "" { 681 healthCheckPort = servicePort 682 } else { 683 healthCheckPort, err = strconv.ParseInt(healthCheck.Port, 10, 64) 684 if err == nil { 685 return AppMeshHealthCheck{}, err 686 } 687 } 688 if healthCheck.Protocol == "" { 689 if serviceProtocol != "" { 690 healthCheck.Protocol = strings.ToLower(serviceProtocol) 691 } else { 692 healthCheck.Protocol = "http" 693 } 694 } 695 if healthCheck.Timeout == 0 { 696 healthCheck.Timeout = 30 697 } 698 if healthCheck.UnhealthyThreshold == 0 { 699 healthCheck.UnhealthyThreshold = 3 700 } 701 return AppMeshHealthCheck{ 702 HealthyThreshold: healthCheck.HealthyThreshold, 703 IntervalMillis: healthCheck.Interval * 1000, 704 Path: healthCheck.Path, 705 Port: healthCheckPort, 706 Protocol: strings.ToLower(healthCheck.Protocol), 707 TimeoutMillis: healthCheck.Timeout * 1000, 708 UnhealthyThreshold: healthCheck.UnhealthyThreshold, 709 }, nil 710 } 711 712 func (e *ECS) CreateTaskDefinition(d service.Deploy, secrets map[string]string) (*string, error) { 713 var err error 714 715 svc := ecs.New(session.New()) 716 717 // get account id 718 iam := IAM{} 719 err = iam.GetAccountId() 720 if err != nil { 721 return nil, errors.New("Could not get accountId during createTaskDefinition") 722 } 723 724 err = e.CreateTaskDefinitionInput(d, secrets, iam.AccountId) 725 if err != nil { 726 return nil, err 727 } 728 729 // going to register 730 ecsLogger.Debugf("Going to register: %+v", e.TaskDefinition) 731 732 result, err := svc.RegisterTaskDefinition(e.TaskDefinition) 733 if err != nil { 734 if aerr, ok := err.(awserr.Error); ok { 735 switch aerr.Code() { 736 case ecs.ErrCodeServerException: 737 ecsLogger.Errorf(ecs.ErrCodeServerException+": %v", aerr.Error()) 738 case ecs.ErrCodeClientException: 739 ecsLogger.Errorf(ecs.ErrCodeClientException+": %v", aerr.Error()) 740 case ecs.ErrCodeInvalidParameterException: 741 ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException+": %v", aerr.Error()) 742 default: 743 ecsLogger.Errorf(aerr.Error()) 744 } 745 } 746 // return error 747 return nil, errors.New("Could not register task definition") 748 } else { 749 return result.TaskDefinition.TaskDefinitionArn, nil 750 } 751 } 752 753 // check whether service exists 754 func (e *ECS) ServiceExists(serviceName string) (bool, error) { 755 svc := ecs.New(session.New()) 756 input := &ecs.DescribeServicesInput{ 757 Cluster: aws.String(e.ClusterName), 758 Services: []*string{ 759 aws.String(serviceName), 760 }, 761 } 762 763 result, err := svc.DescribeServices(input) 764 if err != nil { 765 if aerr, ok := err.(awserr.Error); ok { 766 switch aerr.Code() { 767 case ecs.ErrCodeServerException: 768 ecsLogger.Errorf(ecs.ErrCodeServerException, aerr.Error()) 769 case ecs.ErrCodeClientException: 770 ecsLogger.Errorf(ecs.ErrCodeClientException, aerr.Error()) 771 case ecs.ErrCodeInvalidParameterException: 772 ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException, aerr.Error()) 773 case ecs.ErrCodeClusterNotFoundException: 774 ecsLogger.Errorf(ecs.ErrCodeClusterNotFoundException, aerr.Error()) 775 default: 776 ecsLogger.Errorf(aerr.Error()) 777 } 778 } else { 779 // Print the error, cast err to awserr.Error to get the Code and 780 // Message from an error. 781 ecsLogger.Errorf(err.Error()) 782 } 783 return false, err 784 } 785 if len(result.Services) == 0 { 786 return false, nil 787 } else if len(result.Services) == 1 && *result.Services[0].Status == "INACTIVE" { 788 return false, nil 789 } else { 790 return true, nil 791 } 792 } 793 794 // Update ECS service 795 func (e *ECS) UpdateService(serviceName string, taskDefArn *string, d service.Deploy) (*string, error) { 796 svc := ecs.New(session.New()) 797 input := &ecs.UpdateServiceInput{ 798 Cluster: aws.String(e.ClusterName), 799 Service: aws.String(serviceName), 800 TaskDefinition: aws.String(*taskDefArn), 801 } 802 803 // network configuration 804 if d.NetworkMode == "awsvpc" && len(d.NetworkConfiguration.Subnets) > 0 { 805 input.SetNetworkConfiguration(e.getNetworkConfiguration(d)) 806 } 807 808 // set gracePeriodSeconds 809 if d.HealthCheck.GracePeriodSeconds > 0 { 810 input.SetHealthCheckGracePeriodSeconds(d.HealthCheck.GracePeriodSeconds) 811 } 812 813 ecsLogger.Debugf("Running UpdateService with input: %+v", input) 814 815 result, err := svc.UpdateService(input) 816 if err != nil { 817 if aerr, ok := err.(awserr.Error); ok { 818 switch aerr.Code() { 819 case ecs.ErrCodeServerException: 820 ecsLogger.Errorf(ecs.ErrCodeServerException+": %v", aerr.Error()) 821 case ecs.ErrCodeClientException: 822 ecsLogger.Errorf(ecs.ErrCodeClientException+": %v", aerr.Error()) 823 case ecs.ErrCodeInvalidParameterException: 824 ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException+": %v", aerr.Error()) 825 case ecs.ErrCodeClusterNotFoundException: 826 ecsLogger.Errorf(ecs.ErrCodeClusterNotFoundException+": %v", aerr.Error()) 827 case ecs.ErrCodeServiceNotFoundException: 828 ecsLogger.Infof(ecs.ErrCodeServiceNotFoundException+": %v", aerr.Error()) 829 // return error code to create new service 830 return nil, errors.New("ServiceNotFoundException") 831 case ecs.ErrCodeServiceNotActiveException: 832 ecsLogger.Errorf(ecs.ErrCodeServiceNotActiveException+": %v", aerr.Error()) 833 default: 834 ecsLogger.Errorf(aerr.Error()) 835 } 836 } else { 837 // Print the error, cast err to awserr.Error to get the Code and 838 // Message from an error. 839 ecsLogger.Errorf(err.Error()) 840 } 841 return nil, errors.New("Could not update service: " + serviceName) 842 } 843 return result.Service.ServiceName, nil 844 } 845 846 // delete ECS service 847 func (e *ECS) DeleteService(clusterName, serviceName string) error { 848 // first set desiredCount to 0 849 svc := ecs.New(session.New()) 850 input := &ecs.UpdateServiceInput{ 851 Cluster: aws.String(clusterName), 852 Service: aws.String(serviceName), 853 DesiredCount: aws.Int64(0), 854 } 855 856 _, err := svc.UpdateService(input) 857 if err != nil { 858 if aerr, ok := err.(awserr.Error); ok { 859 ecsLogger.Errorf("%v", aerr.Error()) 860 } else { 861 ecsLogger.Errorf("%v", err.Error()) 862 } 863 return err 864 } 865 // delete service 866 input2 := &ecs.DeleteServiceInput{ 867 Cluster: aws.String(clusterName), 868 Service: aws.String(serviceName), 869 } 870 871 _, err = svc.DeleteService(input2) 872 if err != nil { 873 if aerr, ok := err.(awserr.Error); ok { 874 ecsLogger.Errorf("%v", aerr.Error()) 875 } else { 876 ecsLogger.Errorf("%v", err.Error()) 877 } 878 return err 879 } 880 return nil 881 } 882 883 // create service 884 func (e *ECS) CreateService(d service.Deploy) error { 885 svc := ecs.New(session.New()) 886 887 // sanity checks 888 if len(d.Containers) == 0 { 889 return errors.New("No containers defined") 890 } 891 892 input := &ecs.CreateServiceInput{ 893 Cluster: aws.String(d.Cluster), 894 ServiceName: aws.String(e.ServiceName), 895 TaskDefinition: aws.String(*e.TaskDefArn), 896 } 897 898 if d.SchedulingStrategy != "DAEMON" { 899 input.SetPlacementStrategy([]*ecs.PlacementStrategy{ 900 { 901 Field: aws.String("attribute:ecs.availability-zone"), 902 Type: aws.String("spread"), 903 }, 904 { 905 Field: aws.String("memory"), 906 Type: aws.String("binpack"), 907 }, 908 }, 909 ) 910 input.SetDesiredCount(d.DesiredCount) 911 } 912 913 if d.SchedulingStrategy != "" { 914 input.SetSchedulingStrategy(d.SchedulingStrategy) 915 } 916 917 if strings.ToLower(d.ServiceProtocol) != "none" { 918 input.SetLoadBalancers([]*ecs.LoadBalancer{ 919 { 920 ContainerName: aws.String(e.ServiceName), 921 ContainerPort: aws.Int64(d.ServicePort), 922 TargetGroupArn: aws.String(*e.TargetGroupArn), 923 }, 924 }) 925 } 926 927 // network configuration 928 if d.NetworkMode == "awsvpc" && len(d.NetworkConfiguration.Subnets) > 0 { 929 if strings.ToUpper(d.LaunchType) == "FARGATE" { 930 input.SetLaunchType("FARGATE") 931 } 932 input.SetNetworkConfiguration(e.getNetworkConfiguration(d)) 933 } else { 934 // only set role if network mode is not awsvpc (it will be set automatically) 935 // only set role if serviceregistry is not defined 936 // only set the role if there's a loadbalancer necessary 937 if d.ServiceRegistry == "" && strings.ToLower(d.ServiceProtocol) != "none" { 938 input.SetRole(util.GetEnv("AWS_ECS_SERVICE_ROLE", "ecs-service-role")) 939 } 940 } 941 942 // check whether min/max is set 943 dc := &ecs.DeploymentConfiguration{} 944 if d.MinimumHealthyPercent > 0 { 945 dc.SetMinimumHealthyPercent(d.MinimumHealthyPercent) 946 } 947 if d.MaximumPercent > 0 { 948 dc.SetMaximumPercent(d.MaximumPercent) 949 } 950 if (ecs.DeploymentConfiguration{}) != *dc { 951 input.SetDeploymentConfiguration(dc) 952 } 953 954 // set gracePeriodSeconds 955 if d.HealthCheck.GracePeriodSeconds > 0 { 956 input.SetHealthCheckGracePeriodSeconds(d.HealthCheck.GracePeriodSeconds) 957 } 958 959 // set ServiceRegistry 960 if d.ServiceRegistry != "" && strings.ToLower(d.ServiceProtocol) != "none" { 961 sd := ServiceDiscovery{} 962 _, serviceDiscoveryNamespaceID, err := sd.getNamespaceArnAndId(d.ServiceRegistry) 963 if err != nil { 964 ecsLogger.Warningf("Could not apply ServiceRegistry Config: %s", err.Error()) 965 } else { 966 serviceDiscoveryServiceArn, err := sd.getServiceArn(d.ServiceName, serviceDiscoveryNamespaceID) 967 if err != nil && strings.HasPrefix(err.Error(), "Service not found") { 968 // Service not found, create service in service registry 969 serviceDiscoveryServiceArn, err = sd.createService(d.ServiceName, serviceDiscoveryNamespaceID) 970 } 971 // check for error, else set service registry 972 if err != nil && !strings.HasPrefix(err.Error(), "Service not found") { 973 ecsLogger.Warningf("Could not get service from ServiceRegistry: %s", err.Error()) 974 } else { 975 ecsLogger.Debugf("Applying ServiceRegistry for %s with Arn %s", e.ServiceName, serviceDiscoveryServiceArn) 976 input.SetServiceRegistries([]*ecs.ServiceRegistry{ 977 { 978 ContainerName: aws.String(e.ServiceName), 979 ContainerPort: aws.Int64(d.ServicePort), 980 RegistryArn: aws.String(serviceDiscoveryServiceArn), 981 }, 982 }) 983 } 984 } 985 } 986 987 // create service 988 _, err := svc.CreateService(input) 989 if err != nil { 990 if aerr, ok := err.(awserr.Error); ok { 991 switch aerr.Code() { 992 case ecs.ErrCodeServerException: 993 ecsLogger.Errorf(ecs.ErrCodeServerException+": %v", aerr.Error()) 994 case ecs.ErrCodeClientException: 995 ecsLogger.Errorf(ecs.ErrCodeClientException+": %v", aerr.Error()) 996 case ecs.ErrCodeInvalidParameterException: 997 ecsLogger.Errorf(ecs.ErrCodeInvalidParameterException+": %v", aerr.Error()) 998 case ecs.ErrCodeClusterNotFoundException: 999 ecsLogger.Errorf(ecs.ErrCodeClusterNotFoundException+": %v", aerr.Error()) 1000 default: 1001 ecsLogger.Errorf(aerr.Error()) 1002 } 1003 } else { 1004 ecsLogger.Errorf(err.Error()) 1005 } 1006 return errors.New("Could not create service") 1007 } 1008 return nil 1009 } 1010 1011 // wait until service is inactive 1012 func (e *ECS) WaitUntilServicesInactive(clusterName, serviceName string) error { 1013 svc := ecs.New(session.New()) 1014 input := &ecs.DescribeServicesInput{ 1015 Cluster: aws.String(clusterName), 1016 Services: []*string{aws.String(serviceName)}, 1017 } 1018 1019 ecsLogger.Debugf("Waiting for service %v on %v to become inactive", serviceName, clusterName) 1020 1021 err := svc.WaitUntilServicesInactive(input) 1022 if err != nil { 1023 if aerr, ok := err.(awserr.Error); ok { 1024 ecsLogger.Errorf(aerr.Error()) 1025 } else { 1026 ecsLogger.Errorf(err.Error()) 1027 } 1028 return err 1029 } 1030 return nil 1031 } 1032 1033 // wait until service is stable 1034 func (e *ECS) WaitUntilServicesStable(clusterName, serviceName string, maxWaitMinutes int) error { 1035 svc := ecs.New(session.New()) 1036 maxAttempts := maxWaitMinutes * 4 1037 input := &ecs.DescribeServicesInput{ 1038 Cluster: aws.String(clusterName), 1039 Services: []*string{aws.String(serviceName)}, 1040 } 1041 1042 ecsLogger.Debugf("Waiting for service %v on %v to become stable", serviceName, clusterName) 1043 1044 err := svc.WaitUntilServicesStableWithContext(context.Background(), input, request.WithWaiterMaxAttempts(maxAttempts)) 1045 if err != nil { 1046 if aerr, ok := err.(awserr.Error); ok { 1047 ecsLogger.Errorf(aerr.Error()) 1048 } else { 1049 ecsLogger.Errorf(err.Error()) 1050 } 1051 return err 1052 } 1053 return nil 1054 } 1055 1056 func (e *ECS) getMaxWaitMinutes(gracePeriodSeconds int64) int { 1057 // check whether service exists, otherwise wait might give error 1058 if maxWaitSecondsString := util.GetEnv("DEPLOY_MAX_WAIT_SECONDS", "900"); maxWaitSecondsString != "900" { 1059 maxWaitSeconds, err := strconv.Atoi(maxWaitSecondsString) 1060 if err != nil { 1061 return 15 1062 } else { 1063 return int(math.Ceil(float64(maxWaitSeconds) / 60)) 1064 } 1065 } else { 1066 if gracePeriodSeconds > 0 { 1067 return (1 + int(math.Ceil(float64(gracePeriodSeconds)/60/10))) * 10 1068 } 1069 } 1070 return 15 1071 } 1072 1073 func (e *ECS) LaunchWaitUntilServicesStable(dd, ddLast *service.DynamoDeployment, notification integrations.Notification) error { 1074 var failed bool 1075 1076 s := service.NewService() 1077 err := e.WaitUntilServicesStable(dd.DeployData.Cluster, dd.ServiceName, e.getMaxWaitMinutes(dd.DeployData.HealthCheck.GracePeriodSeconds)) 1078 if err != nil { 1079 ecsLogger.Debugf("waitUntilServiceStable didn't succeed: %v", err) 1080 failed = true 1081 } 1082 // check whether deployment has latest task definition 1083 runningService, err := e.DescribeService(dd.DeployData.Cluster, dd.ServiceName, false, true, true) 1084 if err != nil { 1085 return err 1086 } 1087 if len(runningService.Deployments) != 1 { 1088 reason := "Deployment failed: deployment was still running after 10 minutes" 1089 ecsLogger.Debugf(reason) 1090 err := s.SetDeploymentStatusWithReason(dd, "failed", reason) 1091 if err != nil { 1092 return err 1093 } 1094 err = notification.LogFailure(dd.ServiceName + ": " + reason) 1095 if err != nil { 1096 ecsLogger.Errorf("Could not send notification: %s", err) 1097 } 1098 err = e.Rollback(dd.DeployData.Cluster, dd.ServiceName) 1099 if err != nil { 1100 return err 1101 } 1102 return nil 1103 } 1104 if runningService.Deployments[0].TaskDefinition != *dd.TaskDefinitionArn { 1105 reason := "Deployment failed: Still running old task definition" 1106 ecsLogger.Debugf(reason) 1107 err := s.SetDeploymentStatusWithReason(dd, "failed", reason) 1108 if err != nil { 1109 return err 1110 } 1111 err = notification.LogFailure(dd.ServiceName + ": " + reason) 1112 if err != nil { 1113 ecsLogger.Errorf("Could not send notification: %s", err) 1114 } 1115 err = e.Rollback(dd.DeployData.Cluster, dd.ServiceName) 1116 if err != nil { 1117 return err 1118 } 1119 return nil 1120 } 1121 if len(runningService.Tasks) == 0 { 1122 reason := "Deployment failed: no tasks running" 1123 ecsLogger.Debugf(reason) 1124 err := s.SetDeploymentStatusWithReason(dd, "failed", reason) 1125 if err != nil { 1126 return err 1127 } 1128 err = notification.LogFailure(dd.ServiceName + ": " + reason) 1129 if err != nil { 1130 ecsLogger.Errorf("Could not send notification: %s", err) 1131 } 1132 err = e.Rollback(dd.DeployData.Cluster, dd.ServiceName) 1133 if err != nil { 1134 return err 1135 } 1136 return nil 1137 } 1138 if failed { 1139 reason := "Deployment timed out" 1140 s.SetDeploymentStatusWithReason(dd, "failed", reason) 1141 err = notification.LogFailure(dd.ServiceName + ": " + reason) 1142 if err != nil { 1143 ecsLogger.Errorf("Could not send notification: %s", err) 1144 } 1145 return nil 1146 } 1147 // set success 1148 s.SetDeploymentStatus(dd, "success") 1149 if ddLast != nil && ddLast.Status != "success" && ddLast.Status != "aborted" { 1150 err = notification.LogRecovery(dd.ServiceName + ": Deployed successfully") 1151 if err != nil { 1152 ecsLogger.Errorf("Could not send notification: %s", err) 1153 } 1154 } 1155 return nil 1156 } 1157 func (e *ECS) Rollback(clusterName, serviceName string) error { 1158 ecsLogger.Debugf("Starting rollback") 1159 s := service.NewService() 1160 s.ServiceName = serviceName 1161 dd, err := s.GetDeploys("secondToLast", 1) 1162 if err != nil { 1163 ecsLogger.Errorf("Error: %v", err.Error()) 1164 return err 1165 } 1166 if len(dd) == 0 || dd[0].Status != "success" { 1167 ecsLogger.Debugf("Rollback: Previous deploy was not successful") 1168 dd, err := s.GetDeploys("byMonth", 10) 1169 if err != nil { 1170 return err 1171 } 1172 ecsLogger.Debugf("Rollback: checking last %d deploys", len(dd)) 1173 } 1174 for _, v := range dd { 1175 ecsLogger.Debugf("Looping previous deployments: %v with status %v", *v.TaskDefinitionArn, v.Status) 1176 if v.Status == "success" { 1177 ecsLogger.Debugf("Rollback: rolling back to %v", *v.TaskDefinitionArn) 1178 e.UpdateService(v.ServiceName, v.TaskDefinitionArn, *v.DeployData) 1179 return nil 1180 } 1181 } 1182 ecsLogger.Debugf("Could not rollback, no stable version found") 1183 return errors.New("Could not rollback, no stable version found") 1184 } 1185 1186 // describe services 1187 func (e *ECS) DescribeService(clusterName string, serviceName string, showEvents bool, showTasks bool, showStoppedTasks bool) (service.RunningService, error) { 1188 s, err := e.DescribeServices(clusterName, []*string{aws.String(serviceName)}, showEvents, showTasks, showStoppedTasks) 1189 if err == nil && len(s) == 1 { 1190 return s[0], nil 1191 } else { 1192 if err == nil { 1193 return service.RunningService{}, errors.New("describeService: No error, but array length != 1") 1194 } else { 1195 return service.RunningService{}, err 1196 } 1197 } 1198 } 1199 func (e *ECS) DescribeServices(clusterName string, serviceNames []*string, showEvents bool, showTasks bool, showStoppedTasks bool) ([]service.RunningService, error) { 1200 return e.DescribeServicesWithOptions(clusterName, serviceNames, showEvents, showTasks, showStoppedTasks, map[string]string{}) 1201 } 1202 func (e *ECS) DescribeServicesWithOptions(clusterName string, serviceNames []*string, showEvents bool, showTasks bool, showStoppedTasks bool, options map[string]string) ([]service.RunningService, error) { 1203 var rss []service.RunningService 1204 svc := ecs.New(session.New()) 1205 1206 // fetch per 10 1207 var y float64 = float64(len(serviceNames)) / 10 1208 for i := 0; i < int(math.Ceil(y)); i++ { 1209 1210 f := i * 10 1211 t := int(math.Min(float64(10+10*i), float64(len(serviceNames)))) 1212 1213 input := &ecs.DescribeServicesInput{ 1214 Cluster: aws.String(clusterName), 1215 Services: serviceNames[f:t], 1216 } 1217 1218 result, err := svc.DescribeServices(input) 1219 if err != nil { 1220 if aerr, ok := err.(awserr.Error); ok { 1221 ecsLogger.Errorf(aerr.Error()) 1222 } else { 1223 ecsLogger.Errorf(err.Error()) 1224 } 1225 return rss, err 1226 } 1227 for _, ecsService := range result.Services { 1228 rs := service.RunningService{ServiceName: *ecsService.ServiceName, ClusterName: clusterName} 1229 rs.RunningCount = *ecsService.RunningCount 1230 rs.PendingCount = *ecsService.PendingCount 1231 rs.DesiredCount = *ecsService.DesiredCount 1232 rs.Status = *ecsService.Status 1233 for _, deployment := range ecsService.Deployments { 1234 var ds service.RunningServiceDeployment 1235 ds.Status = *deployment.Status 1236 ds.RunningCount = *deployment.RunningCount 1237 ds.PendingCount = *deployment.PendingCount 1238 ds.DesiredCount = *deployment.DesiredCount 1239 ds.CreatedAt = *deployment.CreatedAt 1240 ds.UpdatedAt = *deployment.UpdatedAt 1241 ds.TaskDefinition = *deployment.TaskDefinition 1242 rs.Deployments = append(rs.Deployments, ds) 1243 } 1244 if showEvents { 1245 for _, event := range ecsService.Events { 1246 event := service.RunningServiceEvent{ 1247 Id: *event.Id, 1248 CreatedAt: *event.CreatedAt, 1249 Message: *event.Message, 1250 } 1251 rs.Events = append(rs.Events, event) 1252 } 1253 } 1254 if showTasks { 1255 taskArns, err := e.ListTasks(clusterName, *ecsService.ServiceName, "RUNNING", "service") 1256 if err != nil { 1257 return rss, err 1258 } 1259 if showStoppedTasks { 1260 taskArnsStopped, err := e.ListTasks(clusterName, *ecsService.ServiceName, "STOPPED", "service") 1261 if err != nil { 1262 return rss, err 1263 } 1264 taskArns = append(taskArns, taskArnsStopped...) 1265 } 1266 runningTasks, err := e.DescribeTasks(clusterName, taskArns) 1267 if err != nil { 1268 return rss, err 1269 } 1270 rs.Tasks = runningTasks 1271 } 1272 rss = append(rss, rs) 1273 } 1274 // check whether to sleep between calls 1275 for k, v := range options { 1276 if k == "sleep" { 1277 seconds, err := strconv.Atoi(v) 1278 if err != nil { 1279 return rss, fmt.Errorf("Couldn't convert sleep value to int (in options)") 1280 } 1281 time.Sleep(time.Duration(seconds) * time.Second) 1282 } 1283 } 1284 } 1285 return rss, nil 1286 } 1287 1288 // list tasks 1289 func (e *ECS) ListTasks(clusterName, name, desiredStatus, filterBy string) ([]*string, error) { 1290 svc := ecs.New(session.New()) 1291 var tasks []*string 1292 1293 input := &ecs.ListTasksInput{ 1294 Cluster: aws.String(clusterName), 1295 } 1296 if filterBy == "service" { 1297 input.SetServiceName(name) 1298 } else if filterBy == "family" { 1299 input.SetFamily(name) 1300 } else { 1301 return tasks, errors.New("Invalid filterBy") 1302 } 1303 if desiredStatus == "STOPPED" { 1304 input.SetDesiredStatus(desiredStatus) 1305 } 1306 1307 pageNum := 0 1308 err := svc.ListTasksPages(input, 1309 func(page *ecs.ListTasksOutput, lastPage bool) bool { 1310 pageNum++ 1311 tasks = append(tasks, page.TaskArns...) 1312 return pageNum <= 100 1313 }) 1314 1315 if err != nil { 1316 if aerr, ok := err.(awserr.Error); ok { 1317 ecsLogger.Errorf(aerr.Error()) 1318 } else { 1319 ecsLogger.Errorf(err.Error()) 1320 } 1321 } 1322 return tasks, err 1323 } 1324 func (e *ECS) DescribeTasks(clusterName string, tasks []*string) ([]service.RunningTask, error) { 1325 var rts []service.RunningTask 1326 svc := ecs.New(session.New()) 1327 1328 // fetch per 100 1329 var y float64 = float64(len(tasks)) / 100 1330 for i := 0; i < int(math.Ceil(y)); i++ { 1331 1332 f := i * 100 1333 t := int(math.Min(float64(100+100*i), float64(len(tasks)))) 1334 1335 input := &ecs.DescribeTasksInput{ 1336 Cluster: aws.String(clusterName), 1337 Tasks: tasks[f:t], 1338 } 1339 1340 result, err := svc.DescribeTasks(input) 1341 if err != nil { 1342 if aerr, ok := err.(awserr.Error); ok { 1343 ecsLogger.Errorf(aerr.Error()) 1344 } else { 1345 ecsLogger.Errorf(err.Error()) 1346 } 1347 return rts, err 1348 } 1349 for _, task := range result.Tasks { 1350 rs := service.RunningTask{} 1351 rs.ContainerInstanceArn = *task.ContainerInstanceArn 1352 rs.Cpu = *task.Cpu 1353 rs.CreatedAt = *task.CreatedAt 1354 rs.DesiredStatus = *task.DesiredStatus 1355 if task.ExecutionStoppedAt != nil { 1356 rs.ExecutionStoppedAt = *task.ExecutionStoppedAt 1357 } 1358 if task.Group != nil { 1359 rs.Group = *task.Group 1360 } 1361 rs.LastStatus = *task.LastStatus 1362 rs.LaunchType = *task.LaunchType 1363 rs.Memory = *task.Memory 1364 if task.PullStartedAt != nil { 1365 rs.PullStartedAt = *task.PullStartedAt 1366 } 1367 if task.PullStoppedAt != nil { 1368 rs.PullStoppedAt = *task.PullStoppedAt 1369 } 1370 if task.StartedAt != nil { 1371 rs.StartedAt = *task.StartedAt 1372 } 1373 if task.StartedBy != nil { 1374 rs.StartedBy = *task.StartedBy 1375 } 1376 if task.StoppedAt != nil { 1377 rs.StoppedAt = *task.StoppedAt 1378 } 1379 if task.StoppedReason != nil { 1380 rs.StoppedReason = *task.StoppedReason 1381 } 1382 if task.StoppingAt != nil { 1383 rs.StoppingAt = *task.StoppingAt 1384 } 1385 rs.TaskArn = *task.TaskArn 1386 rs.TaskDefinitionArn = *task.TaskDefinitionArn 1387 rs.Version = *task.Version 1388 for _, container := range task.Containers { 1389 var tc service.RunningTaskContainer 1390 tc.ContainerArn = *container.ContainerArn 1391 if container.ExitCode != nil { 1392 tc.ExitCode = *container.ExitCode 1393 } 1394 if container.LastStatus != nil { 1395 tc.LastStatus = *container.LastStatus 1396 } 1397 tc.Name = *container.Name 1398 if container.Reason != nil { 1399 tc.Reason = *container.Reason 1400 } 1401 rs.Containers = append(rs.Containers, tc) 1402 } 1403 rts = append(rts, rs) 1404 } 1405 } 1406 return rts, nil 1407 } 1408 1409 func (e *ECS) ListContainerInstances(clusterName string) ([]string, error) { 1410 svc := ecs.New(session.New()) 1411 input := &ecs.ListContainerInstancesInput{ 1412 Cluster: aws.String(clusterName), 1413 } 1414 var instanceArns []*string 1415 1416 pageNum := 0 1417 err := svc.ListContainerInstancesPages(input, 1418 func(page *ecs.ListContainerInstancesOutput, lastPage bool) bool { 1419 pageNum++ 1420 instanceArns = append(instanceArns, page.ContainerInstanceArns...) 1421 return pageNum <= 100 1422 }) 1423 1424 if err != nil { 1425 if aerr, ok := err.(awserr.Error); ok { 1426 ecsLogger.Errorf("%v", aerr.Error()) 1427 } else { 1428 ecsLogger.Errorf("%v", err.Error()) 1429 } 1430 return aws.StringValueSlice(instanceArns), err 1431 } 1432 return aws.StringValueSlice(instanceArns), nil 1433 } 1434 1435 // describe container instances 1436 func (e *ECS) DescribeContainerInstances(clusterName string, containerInstances []string) ([]ContainerInstance, error) { 1437 var cis []ContainerInstance 1438 svc := ecs.New(session.New()) 1439 input := &ecs.DescribeContainerInstancesInput{ 1440 Cluster: aws.String(clusterName), 1441 ContainerInstances: aws.StringSlice(containerInstances), 1442 } 1443 1444 result, err := svc.DescribeContainerInstances(input) 1445 if err != nil { 1446 if aerr, ok := err.(awserr.Error); ok { 1447 ecsLogger.Errorf(aerr.Error()) 1448 } else { 1449 ecsLogger.Errorf(err.Error()) 1450 } 1451 return cis, err 1452 } 1453 if len(result.ContainerInstances) == 0 { 1454 return cis, errors.New("No container instances returned") 1455 } 1456 for _, ci := range result.ContainerInstances { 1457 var c ContainerInstance 1458 c.ContainerInstanceArn = aws.StringValue(ci.ContainerInstanceArn) 1459 c.Ec2InstanceId = aws.StringValue(ci.Ec2InstanceId) 1460 c.PendingTasksCount = aws.Int64Value(ci.PendingTasksCount) 1461 c.RegisteredAt = aws.TimeValue(ci.RegisteredAt) 1462 c.RunningTasksCount = aws.Int64Value(ci.RunningTasksCount) 1463 c.Status = aws.StringValue(ci.Status) 1464 c.Version = aws.Int64Value(ci.Version) 1465 for _, v := range ci.RegisteredResources { 1466 var vv ContainerInstanceResource 1467 switch aws.StringValue(v.Type) { 1468 case "INTEGER": 1469 vv.IntegerValue = aws.Int64Value(v.IntegerValue) 1470 case "DOUBLE": 1471 vv.DoubleValue = aws.Float64Value(v.DoubleValue) 1472 case "LONG": 1473 vv.IntegerValue = aws.Int64Value(v.IntegerValue) 1474 case "STRINGSET": 1475 vv.StringSetValue = aws.StringValueSlice(v.StringSetValue) 1476 } 1477 vv.Name = aws.StringValue(v.Name) 1478 vv.Type = aws.StringValue(v.Type) 1479 c.RegisteredResources = append(c.RegisteredResources, vv) 1480 } 1481 for _, v := range ci.RemainingResources { 1482 var vv ContainerInstanceResource 1483 switch aws.StringValue(v.Type) { 1484 case "INTEGER": 1485 vv.IntegerValue = aws.Int64Value(v.IntegerValue) 1486 case "DOUBLE": 1487 vv.DoubleValue = aws.Float64Value(v.DoubleValue) 1488 case "LONG": 1489 vv.IntegerValue = aws.Int64Value(v.IntegerValue) 1490 case "STRINGSET": 1491 vv.StringSetValue = aws.StringValueSlice(v.StringSetValue) 1492 } 1493 vv.Name = aws.StringValue(v.Name) 1494 vv.Type = aws.StringValue(v.Type) 1495 c.RemainingResources = append(c.RemainingResources, vv) 1496 } 1497 // get AZ 1498 for _, ciAttr := range ci.Attributes { 1499 if aws.StringValue(ciAttr.Name) == "ecs.availability-zone" { 1500 c.AvailabilityZone = aws.StringValue(ciAttr.Value) 1501 } 1502 } 1503 cis = append(cis, c) 1504 } 1505 return cis, nil 1506 } 1507 1508 // manual scale ECS service 1509 func (e *ECS) ManualScaleService(clusterName, serviceName string, desiredCount int64) error { 1510 svc := ecs.New(session.New()) 1511 input := &ecs.UpdateServiceInput{ 1512 Cluster: aws.String(clusterName), 1513 Service: aws.String(serviceName), 1514 DesiredCount: aws.Int64(desiredCount), 1515 } 1516 1517 ecsLogger.Debugf("Manually scaling %v to a count of %d", serviceName, desiredCount) 1518 1519 _, err := svc.UpdateService(input) 1520 if err != nil { 1521 if aerr, ok := err.(awserr.Error); ok { 1522 ecsLogger.Errorf(aerr.Error()) 1523 } else { 1524 ecsLogger.Errorf(err.Error()) 1525 } 1526 return err 1527 } 1528 return nil 1529 } 1530 1531 func (e *ECS) getNetworkConfiguration(d service.Deploy) *ecs.NetworkConfiguration { 1532 var sns []*string 1533 var sgs []*string 1534 var aIp string 1535 nc := &ecs.NetworkConfiguration{AwsvpcConfiguration: &ecs.AwsVpcConfiguration{}} 1536 ec2 := EC2{} 1537 for i := range d.NetworkConfiguration.Subnets { 1538 if strings.HasPrefix(d.NetworkConfiguration.Subnets[i], "subnet-") { 1539 sns = append(sns, &d.NetworkConfiguration.Subnets[i]) 1540 } else { 1541 subnetID, err := ec2.GetSubnetID(d.NetworkConfiguration.Subnets[i]) 1542 if err != nil { 1543 ecsLogger.Errorf("Couldn't retrieve subnet name %s: %s", d.NetworkConfiguration.Subnets[i], err) 1544 } else { 1545 sns = append(sns, &subnetID) 1546 } 1547 1548 } 1549 } 1550 nc.AwsvpcConfiguration.SetSubnets(sns) 1551 for i := range d.NetworkConfiguration.SecurityGroups { 1552 if strings.HasPrefix(d.NetworkConfiguration.SecurityGroups[i], "sg-") { 1553 sgs = append(sgs, &d.NetworkConfiguration.SecurityGroups[i]) 1554 } else { 1555 securityGroupID, err := ec2.GetSecurityGroupID(d.NetworkConfiguration.SecurityGroups[i]) 1556 if err != nil { 1557 ecsLogger.Errorf("Couldn't retrieve subnet name %s: %s", d.NetworkConfiguration.Subnets[i], err) 1558 } else { 1559 sgs = append(sgs, &securityGroupID) 1560 } 1561 } 1562 } 1563 nc.AwsvpcConfiguration.SetSecurityGroups(sgs) 1564 if d.NetworkConfiguration.AssignPublicIp == "" { 1565 aIp = "DISABLED" 1566 } else { 1567 aIp = d.NetworkConfiguration.AssignPublicIp 1568 } 1569 nc.AwsvpcConfiguration.SetAssignPublicIp(aIp) 1570 return nc 1571 } 1572 1573 // run one-off task 1574 func (e *ECS) RunTask(clusterName, taskDefinition string, runTask service.RunTask, d service.Deploy) (string, error) { 1575 var taskArn string 1576 svc := ecs.New(session.New()) 1577 input := &ecs.RunTaskInput{ 1578 Cluster: aws.String(clusterName), 1579 TaskDefinition: aws.String(taskDefinition), 1580 StartedBy: aws.String(runTask.StartedBy), 1581 } 1582 1583 taskOverride := &ecs.TaskOverride{} 1584 var containerOverrides []*ecs.ContainerOverride 1585 for _, co := range runTask.ContainerOverrides { 1586 // environment variables 1587 var environment []*ecs.KeyValuePair 1588 if len(co.Environment) > 0 { 1589 for _, v := range co.Environment { 1590 environment = append(environment, &ecs.KeyValuePair{Name: aws.String(v.Name), Value: aws.String(v.Value)}) 1591 } 1592 } 1593 1594 containerOverrides = append(containerOverrides, &ecs.ContainerOverride{ 1595 Command: aws.StringSlice(co.Command), 1596 Name: aws.String(co.Name), 1597 Environment: environment, 1598 }) 1599 } 1600 taskOverride.SetContainerOverrides(containerOverrides) 1601 input.SetOverrides(taskOverride) 1602 1603 // network configuration 1604 if d.NetworkMode == "awsvpc" && len(d.NetworkConfiguration.Subnets) > 0 { 1605 if strings.ToUpper(d.LaunchType) == "FARGATE" { 1606 input.SetLaunchType("FARGATE") 1607 } 1608 input.SetNetworkConfiguration(e.getNetworkConfiguration(d)) 1609 } 1610 1611 ecsLogger.Debugf("Running ad-hoc task using taskdef %s and taskoverride: %+v", taskDefinition, taskOverride) 1612 1613 result, err := svc.RunTask(input) 1614 if err != nil { 1615 if aerr, ok := err.(awserr.Error); ok { 1616 ecsLogger.Errorf(aerr.Error()) 1617 } else { 1618 ecsLogger.Errorf(err.Error()) 1619 } 1620 return taskArn, err 1621 } 1622 if len(result.Tasks) == 0 { 1623 return taskArn, errors.New("No task arn returned") 1624 } 1625 return aws.StringValue(result.Tasks[0].TaskArn), nil 1626 } 1627 1628 func (e *ECS) GetTaskDefinition(clusterName, serviceName string) (string, error) { 1629 runningService, err := e.DescribeService(clusterName, serviceName, false, false, false) 1630 if err != nil { 1631 return "", nil 1632 } 1633 for _, d := range runningService.Deployments { 1634 if d.Status == "PRIMARY" { 1635 return d.TaskDefinition, nil 1636 } 1637 } 1638 return "", errors.New("No task definition found") 1639 } 1640 func (e *ECS) DescribeTaskDefinition(taskDefinitionNameOrArn string) (TaskDefinition, error) { 1641 var taskDefinition TaskDefinition 1642 svc := ecs.New(session.New()) 1643 input := &ecs.DescribeTaskDefinitionInput{ 1644 TaskDefinition: aws.String(taskDefinitionNameOrArn), 1645 } 1646 1647 result, err := svc.DescribeTaskDefinition(input) 1648 if err != nil { 1649 if aerr, ok := err.(awserr.Error); ok { 1650 ecsLogger.Errorf(aerr.Error()) 1651 } else { 1652 ecsLogger.Errorf(err.Error()) 1653 } 1654 return taskDefinition, err 1655 } 1656 1657 taskDefinition.Family = aws.StringValue(result.TaskDefinition.Family) 1658 taskDefinition.Revision = aws.Int64Value(result.TaskDefinition.Revision) 1659 taskDefinition.ExecutionRoleArn = aws.StringValue(result.TaskDefinition.ExecutionRoleArn) 1660 var containerDefinitions []ContainerDefinition 1661 for _, cd := range result.TaskDefinition.ContainerDefinitions { 1662 var containerDefinition ContainerDefinition 1663 containerDefinition.Name = aws.StringValue(cd.Name) 1664 containerDefinition.Essential = aws.BoolValue(cd.Essential) 1665 containerDefinitions = append(containerDefinitions, containerDefinition) 1666 } 1667 taskDefinition.ContainerDefinitions = containerDefinitions 1668 1669 return taskDefinition, nil 1670 } 1671 1672 func (e *ECS) GetContainerLimits(d service.Deploy) (int64, int64, int64, int64) { 1673 var cpuReservation, cpuLimit, memoryReservation, memoryLimit int64 1674 for _, c := range d.Containers { 1675 if c.MemoryReservation == 0 { 1676 memoryReservation += c.Memory 1677 memoryLimit += c.Memory 1678 } else { 1679 memoryReservation += c.MemoryReservation 1680 memoryLimit += c.Memory 1681 } 1682 if c.CPUReservation == 0 { 1683 cpuReservation += c.CPU 1684 cpuLimit += c.CPU 1685 } else { 1686 cpuReservation += c.CPUReservation 1687 cpuLimit += c.CPU 1688 } 1689 } 1690 return cpuReservation, cpuLimit, memoryReservation, memoryLimit 1691 } 1692 func (e *ECS) IsEqualContainerLimits(d1 service.Deploy, d2 service.Deploy) bool { 1693 cpuReservation1, cpuLimit1, memoryReservation1, memoryLimit1 := e.GetContainerLimits(d1) 1694 cpuReservation2, cpuLimit2, memoryReservation2, memoryLimit2 := e.GetContainerLimits(d2) 1695 if cpuReservation1 == cpuReservation2 && cpuLimit1 == cpuLimit2 && memoryReservation1 == memoryReservation2 && memoryLimit1 == memoryLimit2 { 1696 return true 1697 } else { 1698 return false 1699 } 1700 } 1701 1702 func (e *ECS) GetInstanceResources(clusterName string) ([]FreeInstanceResource, []RegisteredInstanceResource, error) { 1703 var firs []FreeInstanceResource 1704 var rirs []RegisteredInstanceResource 1705 ciArns, err := e.ListContainerInstances(clusterName) 1706 if err != nil { 1707 return firs, rirs, err 1708 } 1709 cis, err := e.DescribeContainerInstances(clusterName, ciArns) 1710 if err != nil { 1711 return firs, rirs, err 1712 } 1713 for _, ci := range cis { 1714 // free resources 1715 fir, err := e.ConvertResourceToFir(ci.RemainingResources) 1716 if err != nil { 1717 return firs, rirs, err 1718 } 1719 fir.InstanceId = ci.Ec2InstanceId 1720 fir.AvailabilityZone = ci.AvailabilityZone 1721 fir.Status = ci.Status 1722 firs = append(firs, fir) 1723 // registered resources 1724 rir, err := e.ConvertResourceToRir(ci.RegisteredResources) 1725 if err != nil { 1726 return firs, rirs, err 1727 } 1728 rir.InstanceId = ci.Ec2InstanceId 1729 rirs = append(rirs, rir) 1730 } 1731 return firs, rirs, nil 1732 } 1733 func (e *ECS) ConvertResourceToFir(cir []ContainerInstanceResource) (FreeInstanceResource, error) { 1734 var fir FreeInstanceResource 1735 for _, v := range cir { 1736 if v.Name == "MEMORY" { 1737 if v.Type != "INTEGER" && v.Type != "LONG" { 1738 return fir, errors.New("Memory return wrong type (" + v.Type + ")") 1739 } 1740 fir.FreeMemory = v.IntegerValue 1741 } 1742 if v.Name == "CPU" { 1743 if v.Type != "INTEGER" && v.Type != "LONG" { 1744 return fir, errors.New("CPU return wrong type (" + v.Type + ")") 1745 } 1746 fir.FreeCpu = v.IntegerValue 1747 } 1748 } 1749 return fir, nil 1750 } 1751 func (e *ECS) ConvertResourceToRir(cir []ContainerInstanceResource) (RegisteredInstanceResource, error) { 1752 var rir RegisteredInstanceResource 1753 for _, v := range cir { 1754 if v.Name == "MEMORY" { 1755 if v.Type != "INTEGER" && v.Type != "LONG" { 1756 return rir, errors.New("Memory return wrong type (" + v.Type + ")") 1757 } 1758 rir.RegisteredMemory = v.IntegerValue 1759 } 1760 if v.Name == "CPU" { 1761 if v.Type != "INTEGER" && v.Type != "LONG" { 1762 return rir, errors.New("CPU return wrong type (" + v.Type + ")") 1763 } 1764 rir.RegisteredCpu = v.IntegerValue 1765 } 1766 } 1767 return rir, nil 1768 } 1769 1770 func (e *ECS) DrainNode(clusterName, instance string) error { 1771 svc := ecs.New(session.New()) 1772 input := &ecs.UpdateContainerInstancesStateInput{ 1773 Cluster: aws.String(clusterName), 1774 ContainerInstances: aws.StringSlice([]string{instance}), 1775 Status: aws.String("DRAINING"), 1776 } 1777 _, err := svc.UpdateContainerInstancesState(input) 1778 if err != nil { 1779 if aerr, ok := err.(awserr.Error); ok { 1780 ecsLogger.Errorf("%v", aerr.Error()) 1781 } else { 1782 ecsLogger.Errorf("%v", err.Error()) 1783 } 1784 return err 1785 } 1786 return nil 1787 } 1788 func (e *ECS) GetClusterNameByInstanceId(instance string) (string, error) { 1789 var clusterName string 1790 svc := ec2.New(session.New()) 1791 input := &ec2.DescribeTagsInput{ 1792 Filters: []*ec2.Filter{ 1793 { 1794 Name: aws.String("resource-id"), 1795 Values: []*string{ 1796 aws.String(instance), 1797 }, 1798 }, 1799 }, 1800 } 1801 1802 result, err := svc.DescribeTags(input) 1803 if err != nil { 1804 if aerr, ok := err.(awserr.Error); ok { 1805 ecsLogger.Errorf("%v", aerr.Error()) 1806 } else { 1807 ecsLogger.Errorf("%v", err.Error()) 1808 } 1809 return clusterName, err 1810 } 1811 for _, v := range result.Tags { 1812 if aws.StringValue(v.Key) == "Cluster" { 1813 return aws.StringValue(v.Value), nil 1814 } 1815 } 1816 return clusterName, errors.New("Could not determine clusterName. Is the EC2 instance tagged?") 1817 } 1818 func (e *ECS) GetContainerInstanceArnByInstanceId(clusterName, instanceId string) (string, error) { 1819 ciArns, err := e.ListContainerInstances(clusterName) 1820 if err != nil { 1821 return "", err 1822 } 1823 cis, err := e.DescribeContainerInstances(clusterName, ciArns) 1824 if err != nil { 1825 return "", err 1826 } 1827 for _, ci := range cis { 1828 if ci.Ec2InstanceId == instanceId { 1829 return ci.ContainerInstanceArn, nil 1830 } 1831 } 1832 return "", errors.New("Couldn't find container instance Arn (instanceId=" + instanceId + ")") 1833 } 1834 func (e *ECS) LaunchWaitForDrainedNode(clusterName, containerInstanceArn, instanceId, autoScalingGroupName, lifecycleHookName, lifecycleHookToken string) error { 1835 var tasksDrained bool 1836 var err error 1837 for i := 0; i < 80 && !tasksDrained; i++ { 1838 cis, err := e.DescribeContainerInstances(clusterName, []string{containerInstanceArn}) 1839 if err != nil || len(cis) == 0 { 1840 ecsLogger.Errorf("launchWaitForDrainedNode: %v", err.Error()) 1841 return err 1842 } 1843 ci := cis[0] 1844 if ci.RunningTasksCount == 0 { 1845 tasksDrained = true 1846 } else { 1847 ecsLogger.Infof("launchWaitForDrainedNode: still %d tasks running", ci.RunningTasksCount) 1848 } 1849 time.Sleep(15 * time.Second) 1850 } 1851 if !tasksDrained { 1852 ecsLogger.Errorf("launchWaitForDrainedNode: Not able to drain tasks: timeout of 20m reached") 1853 } 1854 // CompleteLifeCycleAction 1855 autoscaling := AutoScaling{} 1856 if lifecycleHookToken == "" { 1857 ecsLogger.Debugf("Running completePendingLifecycleAction") 1858 err = autoscaling.CompletePendingLifecycleAction(autoScalingGroupName, instanceId, "CONTINUE", lifecycleHookName) 1859 } else { 1860 ecsLogger.Debugf("Running completeLifecycleAction") 1861 err = autoscaling.CompleteLifecycleAction(autoScalingGroupName, instanceId, "CONTINUE", lifecycleHookName, lifecycleHookToken) 1862 } 1863 if err != nil { 1864 ecsLogger.Errorf("launchWaitForDrainedNode: Could not complete life cycle action: %v", err.Error()) 1865 return err 1866 } 1867 ecsLogger.Infof("launchWaitForDrainedNode: Node drained, completed lifecycle action") 1868 return nil 1869 } 1870 1871 // list services 1872 func (e *ECS) ListServices(clusterName string) ([]*string, error) { 1873 svc := ecs.New(session.New()) 1874 var services []*string 1875 1876 input := &ecs.ListServicesInput{ 1877 Cluster: aws.String(clusterName), 1878 } 1879 1880 pageNum := 0 1881 err := svc.ListServicesPages(input, 1882 func(page *ecs.ListServicesOutput, lastPage bool) bool { 1883 pageNum++ 1884 services = append(services, page.ServiceArns...) 1885 return pageNum <= 100 1886 }) 1887 1888 if err != nil { 1889 if aerr, ok := err.(awserr.Error); ok { 1890 ecsLogger.Errorf(aerr.Error()) 1891 } else { 1892 ecsLogger.Errorf(err.Error()) 1893 } 1894 } 1895 return services, err 1896 }