github.com/cbroglie/terraform@v0.7.0-rc3.0.20170410193827-735dfc416d46/builtin/providers/aws/resource_aws_emr_cluster_test.go (about) 1 package aws 2 3 import ( 4 "fmt" 5 "log" 6 "testing" 7 8 "github.com/aws/aws-sdk-go/aws" 9 "github.com/aws/aws-sdk-go/aws/awserr" 10 "github.com/aws/aws-sdk-go/service/emr" 11 "github.com/hashicorp/terraform/helper/acctest" 12 "github.com/hashicorp/terraform/helper/resource" 13 "github.com/hashicorp/terraform/terraform" 14 ) 15 16 func TestAccAWSEMRCluster_basic(t *testing.T) { 17 var jobFlow emr.RunJobFlowOutput 18 r := acctest.RandInt() 19 resource.Test(t, resource.TestCase{ 20 PreCheck: func() { testAccPreCheck(t) }, 21 Providers: testAccProviders, 22 CheckDestroy: testAccCheckAWSEmrDestroy, 23 Steps: []resource.TestStep{ 24 { 25 Config: testAccAWSEmrClusterConfig(r), 26 Check: testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 27 }, 28 }, 29 }) 30 } 31 32 func TestAccAWSEMRCluster_terminationProtected(t *testing.T) { 33 var jobFlow emr.RunJobFlowOutput 34 r := acctest.RandInt() 35 resource.Test(t, resource.TestCase{ 36 PreCheck: func() { testAccPreCheck(t) }, 37 Providers: testAccProviders, 38 CheckDestroy: testAccCheckAWSEmrDestroy, 39 Steps: []resource.TestStep{ 40 { 41 Config: testAccAWSEmrClusterConfig(r), 42 Check: resource.ComposeTestCheckFunc( 43 testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 44 resource.TestCheckResourceAttr( 45 "aws_emr_cluster.tf-test-cluster", "termination_protection", "false"), 46 ), 47 }, 48 { 49 Config: testAccAWSEmrClusterConfigTerminationPolicyUpdated(r), 50 Check: resource.ComposeTestCheckFunc( 51 testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 52 resource.TestCheckResourceAttr( 53 "aws_emr_cluster.tf-test-cluster", "termination_protection", "true"), 54 ), 55 }, 56 { 57 //Need to turn off termination_protection to allow the job to be deleted 58 Config: testAccAWSEmrClusterConfig(r), 59 Check: resource.ComposeTestCheckFunc( 60 testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 61 ), 62 }, 63 }, 64 }) 65 } 66 67 func TestAccAWSEMRCluster_visibleToAllUsers(t *testing.T) { 68 var jobFlow emr.RunJobFlowOutput 69 r := acctest.RandInt() 70 resource.Test(t, resource.TestCase{ 71 PreCheck: func() { testAccPreCheck(t) }, 72 Providers: testAccProviders, 73 CheckDestroy: testAccCheckAWSEmrDestroy, 74 Steps: []resource.TestStep{ 75 { 76 Config: testAccAWSEmrClusterConfig(r), 77 Check: resource.ComposeTestCheckFunc( 78 testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 79 resource.TestCheckResourceAttr( 80 "aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "true"), 81 ), 82 }, 83 { 84 Config: testAccAWSEmrClusterConfigVisibleToAllUsersUpdated(r), 85 Check: resource.ComposeTestCheckFunc( 86 testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 87 resource.TestCheckResourceAttr( 88 "aws_emr_cluster.tf-test-cluster", "visible_to_all_users", "false"), 89 ), 90 }, 91 }, 92 }) 93 } 94 95 func TestAccAWSEMRCluster_tags(t *testing.T) { 96 var jobFlow emr.RunJobFlowOutput 97 r := acctest.RandInt() 98 resource.Test(t, resource.TestCase{ 99 PreCheck: func() { testAccPreCheck(t) }, 100 Providers: testAccProviders, 101 CheckDestroy: testAccCheckAWSEmrDestroy, 102 Steps: []resource.TestStep{ 103 { 104 Config: testAccAWSEmrClusterConfig(r), 105 Check: resource.ComposeTestCheckFunc( 106 testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 107 resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "4"), 108 resource.TestCheckResourceAttr( 109 "aws_emr_cluster.tf-test-cluster", "tags.role", "rolename"), 110 resource.TestCheckResourceAttr( 111 "aws_emr_cluster.tf-test-cluster", "tags.dns_zone", "env_zone"), 112 resource.TestCheckResourceAttr( 113 "aws_emr_cluster.tf-test-cluster", "tags.env", "env"), 114 resource.TestCheckResourceAttr( 115 "aws_emr_cluster.tf-test-cluster", "tags.name", "name-env")), 116 }, 117 { 118 Config: testAccAWSEmrClusterConfigUpdatedTags(r), 119 Check: resource.ComposeTestCheckFunc( 120 testAccCheckAWSEmrClusterExists("aws_emr_cluster.tf-test-cluster", &jobFlow), 121 resource.TestCheckResourceAttr("aws_emr_cluster.tf-test-cluster", "tags.%", "3"), 122 resource.TestCheckResourceAttr( 123 "aws_emr_cluster.tf-test-cluster", "tags.dns_zone", "new_zone"), 124 resource.TestCheckResourceAttr( 125 "aws_emr_cluster.tf-test-cluster", "tags.Env", "production"), 126 resource.TestCheckResourceAttr( 127 "aws_emr_cluster.tf-test-cluster", "tags.name", "name-env"), 128 ), 129 }, 130 }, 131 }) 132 } 133 134 func testAccCheckAWSEmrDestroy(s *terraform.State) error { 135 conn := testAccProvider.Meta().(*AWSClient).emrconn 136 137 for _, rs := range s.RootModule().Resources { 138 if rs.Type != "aws_emr_cluster" { 139 continue 140 } 141 142 params := &emr.DescribeClusterInput{ 143 ClusterId: aws.String(rs.Primary.ID), 144 } 145 146 describe, err := conn.DescribeCluster(params) 147 148 if err == nil { 149 if describe.Cluster != nil && 150 *describe.Cluster.Status.State == "WAITING" { 151 return fmt.Errorf("EMR Cluster still exists") 152 } 153 } 154 155 providerErr, ok := err.(awserr.Error) 156 if !ok { 157 return err 158 } 159 160 log.Printf("[ERROR] %v", providerErr) 161 } 162 163 return nil 164 } 165 166 func testAccCheckAWSEmrClusterExists(n string, v *emr.RunJobFlowOutput) resource.TestCheckFunc { 167 return func(s *terraform.State) error { 168 rs, ok := s.RootModule().Resources[n] 169 if !ok { 170 return fmt.Errorf("Not found: %s", n) 171 } 172 if rs.Primary.ID == "" { 173 return fmt.Errorf("No cluster id set") 174 } 175 conn := testAccProvider.Meta().(*AWSClient).emrconn 176 describe, err := conn.DescribeCluster(&emr.DescribeClusterInput{ 177 ClusterId: aws.String(rs.Primary.ID), 178 }) 179 if err != nil { 180 return fmt.Errorf("EMR error: %v", err) 181 } 182 183 if describe.Cluster != nil && 184 *describe.Cluster.Id != rs.Primary.ID { 185 return fmt.Errorf("EMR cluser not found") 186 } 187 188 if describe.Cluster != nil && 189 *describe.Cluster.Status.State != "WAITING" { 190 return fmt.Errorf("EMR cluser is not up yet") 191 } 192 193 return nil 194 } 195 } 196 197 func testAccAWSEmrClusterConfig(r int) string { 198 return fmt.Sprintf(` 199 provider "aws" { 200 region = "us-west-2" 201 } 202 203 resource "aws_emr_cluster" "tf-test-cluster" { 204 name = "emr-test-%d" 205 release_label = "emr-4.6.0" 206 applications = ["Spark"] 207 208 ec2_attributes { 209 subnet_id = "${aws_subnet.main.id}" 210 emr_managed_master_security_group = "${aws_security_group.allow_all.id}" 211 emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" 212 instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" 213 } 214 215 master_instance_type = "m3.xlarge" 216 core_instance_type = "m3.xlarge" 217 core_instance_count = 1 218 219 tags { 220 role = "rolename" 221 dns_zone = "env_zone" 222 env = "env" 223 name = "name-env" 224 } 225 226 keep_job_flow_alive_when_no_steps = true 227 termination_protection = false 228 229 bootstrap_action { 230 path = "s3://elasticmapreduce/bootstrap-actions/run-if" 231 name = "runif" 232 args = ["instance.isMaster=true", "echo running on master node"] 233 } 234 235 configurations = "test-fixtures/emr_configurations.json" 236 237 depends_on = ["aws_main_route_table_association.a"] 238 239 service_role = "${aws_iam_role.iam_emr_default_role.arn}" 240 autoscaling_role = "${aws_iam_role.emr-autoscaling-role.arn}" 241 } 242 243 resource "aws_security_group" "allow_all" { 244 name = "allow_all_%d" 245 description = "Allow all inbound traffic" 246 vpc_id = "${aws_vpc.main.id}" 247 248 ingress { 249 from_port = 0 250 to_port = 0 251 protocol = "-1" 252 cidr_blocks = ["0.0.0.0/0"] 253 } 254 255 egress { 256 from_port = 0 257 to_port = 0 258 protocol = "-1" 259 cidr_blocks = ["0.0.0.0/0"] 260 } 261 262 depends_on = ["aws_subnet.main"] 263 264 lifecycle { 265 ignore_changes = ["ingress", "egress"] 266 } 267 268 tags { 269 name = "emr_test" 270 } 271 } 272 273 resource "aws_vpc" "main" { 274 cidr_block = "168.31.0.0/16" 275 enable_dns_hostnames = true 276 277 tags { 278 name = "emr_test_%d" 279 } 280 } 281 282 resource "aws_subnet" "main" { 283 vpc_id = "${aws_vpc.main.id}" 284 cidr_block = "168.31.0.0/20" 285 286 tags { 287 name = "emr_test_%d" 288 } 289 } 290 291 resource "aws_internet_gateway" "gw" { 292 vpc_id = "${aws_vpc.main.id}" 293 } 294 295 resource "aws_route_table" "r" { 296 vpc_id = "${aws_vpc.main.id}" 297 298 route { 299 cidr_block = "0.0.0.0/0" 300 gateway_id = "${aws_internet_gateway.gw.id}" 301 } 302 } 303 304 resource "aws_main_route_table_association" "a" { 305 vpc_id = "${aws_vpc.main.id}" 306 route_table_id = "${aws_route_table.r.id}" 307 } 308 309 ### 310 311 # IAM things 312 313 ### 314 315 # IAM role for EMR Service 316 resource "aws_iam_role" "iam_emr_default_role" { 317 name = "iam_emr_default_role_%d" 318 319 assume_role_policy = <<EOT 320 { 321 "Version": "2008-10-17", 322 "Statement": [ 323 { 324 "Sid": "", 325 "Effect": "Allow", 326 "Principal": { 327 "Service": "elasticmapreduce.amazonaws.com" 328 }, 329 "Action": "sts:AssumeRole" 330 } 331 ] 332 } 333 EOT 334 } 335 336 resource "aws_iam_role_policy_attachment" "service-attach" { 337 role = "${aws_iam_role.iam_emr_default_role.id}" 338 policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}" 339 } 340 341 resource "aws_iam_policy" "iam_emr_default_policy" { 342 name = "iam_emr_default_policy_%d" 343 344 policy = <<EOT 345 { 346 "Version": "2012-10-17", 347 "Statement": [{ 348 "Effect": "Allow", 349 "Resource": "*", 350 "Action": [ 351 "ec2:AuthorizeSecurityGroupEgress", 352 "ec2:AuthorizeSecurityGroupIngress", 353 "ec2:CancelSpotInstanceRequests", 354 "ec2:CreateNetworkInterface", 355 "ec2:CreateSecurityGroup", 356 "ec2:CreateTags", 357 "ec2:DeleteNetworkInterface", 358 "ec2:DeleteSecurityGroup", 359 "ec2:DeleteTags", 360 "ec2:DescribeAvailabilityZones", 361 "ec2:DescribeAccountAttributes", 362 "ec2:DescribeDhcpOptions", 363 "ec2:DescribeInstanceStatus", 364 "ec2:DescribeInstances", 365 "ec2:DescribeKeyPairs", 366 "ec2:DescribeNetworkAcls", 367 "ec2:DescribeNetworkInterfaces", 368 "ec2:DescribePrefixLists", 369 "ec2:DescribeRouteTables", 370 "ec2:DescribeSecurityGroups", 371 "ec2:DescribeSpotInstanceRequests", 372 "ec2:DescribeSpotPriceHistory", 373 "ec2:DescribeSubnets", 374 "ec2:DescribeVpcAttribute", 375 "ec2:DescribeVpcEndpoints", 376 "ec2:DescribeVpcEndpointServices", 377 "ec2:DescribeVpcs", 378 "ec2:DetachNetworkInterface", 379 "ec2:ModifyImageAttribute", 380 "ec2:ModifyInstanceAttribute", 381 "ec2:RequestSpotInstances", 382 "ec2:RevokeSecurityGroupEgress", 383 "ec2:RunInstances", 384 "ec2:TerminateInstances", 385 "ec2:DeleteVolume", 386 "ec2:DescribeVolumeStatus", 387 "ec2:DescribeVolumes", 388 "ec2:DetachVolume", 389 "iam:GetRole", 390 "iam:GetRolePolicy", 391 "iam:ListInstanceProfiles", 392 "iam:ListRolePolicies", 393 "iam:PassRole", 394 "s3:CreateBucket", 395 "s3:Get*", 396 "s3:List*", 397 "sdb:BatchPutAttributes", 398 "sdb:Select", 399 "sqs:CreateQueue", 400 "sqs:Delete*", 401 "sqs:GetQueue*", 402 "sqs:PurgeQueue", 403 "sqs:ReceiveMessage" 404 ] 405 }] 406 } 407 EOT 408 } 409 410 # IAM Role for EC2 Instance Profile 411 resource "aws_iam_role" "iam_emr_profile_role" { 412 name = "iam_emr_profile_role_%d" 413 414 assume_role_policy = <<EOT 415 { 416 "Version": "2008-10-17", 417 "Statement": [ 418 { 419 "Sid": "", 420 "Effect": "Allow", 421 "Principal": { 422 "Service": "ec2.amazonaws.com" 423 }, 424 "Action": "sts:AssumeRole" 425 } 426 ] 427 } 428 EOT 429 } 430 431 resource "aws_iam_instance_profile" "emr_profile" { 432 name = "emr_profile_%d" 433 roles = ["${aws_iam_role.iam_emr_profile_role.name}"] 434 } 435 436 resource "aws_iam_role_policy_attachment" "profile-attach" { 437 role = "${aws_iam_role.iam_emr_profile_role.id}" 438 policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}" 439 } 440 441 resource "aws_iam_policy" "iam_emr_profile_policy" { 442 name = "iam_emr_profile_policy_%d" 443 444 policy = <<EOT 445 { 446 "Version": "2012-10-17", 447 "Statement": [{ 448 "Effect": "Allow", 449 "Resource": "*", 450 "Action": [ 451 "cloudwatch:*", 452 "dynamodb:*", 453 "ec2:Describe*", 454 "elasticmapreduce:Describe*", 455 "elasticmapreduce:ListBootstrapActions", 456 "elasticmapreduce:ListClusters", 457 "elasticmapreduce:ListInstanceGroups", 458 "elasticmapreduce:ListInstances", 459 "elasticmapreduce:ListSteps", 460 "kinesis:CreateStream", 461 "kinesis:DeleteStream", 462 "kinesis:DescribeStream", 463 "kinesis:GetRecords", 464 "kinesis:GetShardIterator", 465 "kinesis:MergeShards", 466 "kinesis:PutRecord", 467 "kinesis:SplitShard", 468 "rds:Describe*", 469 "s3:*", 470 "sdb:*", 471 "sns:*", 472 "sqs:*" 473 ] 474 }] 475 } 476 EOT 477 } 478 479 # IAM Role for autoscaling 480 resource "aws_iam_role" "emr-autoscaling-role" { 481 name = "EMR_AutoScaling_DefaultRole_%d" 482 assume_role_policy = "${data.aws_iam_policy_document.emr-autoscaling-role-policy.json}" 483 } 484 485 data "aws_iam_policy_document" "emr-autoscaling-role-policy" { 486 statement { 487 effect = "Allow" 488 actions = ["sts:AssumeRole"] 489 principals = { 490 type = "Service" 491 identifiers = ["elasticmapreduce.amazonaws.com","application-autoscaling.amazonaws.com"] 492 } 493 } 494 } 495 496 resource "aws_iam_role_policy_attachment" "emr-autoscaling-role" { 497 role = "${aws_iam_role.emr-autoscaling-role.name}" 498 policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" 499 } 500 `, r, r, r, r, r, r, r, r, r, r) 501 } 502 503 func testAccAWSEmrClusterConfigTerminationPolicyUpdated(r int) string { 504 return fmt.Sprintf(` 505 provider "aws" { 506 region = "us-west-2" 507 } 508 509 resource "aws_emr_cluster" "tf-test-cluster" { 510 name = "emr-test-%d" 511 release_label = "emr-4.6.0" 512 applications = ["Spark"] 513 514 ec2_attributes { 515 subnet_id = "${aws_subnet.main.id}" 516 emr_managed_master_security_group = "${aws_security_group.allow_all.id}" 517 emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" 518 instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" 519 } 520 521 master_instance_type = "m3.xlarge" 522 core_instance_type = "m3.xlarge" 523 core_instance_count = 1 524 525 tags { 526 role = "rolename" 527 dns_zone = "env_zone" 528 env = "env" 529 name = "name-env" 530 } 531 532 keep_job_flow_alive_when_no_steps = true 533 termination_protection = true 534 535 bootstrap_action { 536 path = "s3://elasticmapreduce/bootstrap-actions/run-if" 537 name = "runif" 538 args = ["instance.isMaster=true", "echo running on master node"] 539 } 540 541 configurations = "test-fixtures/emr_configurations.json" 542 543 depends_on = ["aws_main_route_table_association.a"] 544 545 service_role = "${aws_iam_role.iam_emr_default_role.arn}" 546 autoscaling_role = "${aws_iam_role.emr-autoscaling-role.arn}" 547 } 548 549 resource "aws_security_group" "allow_all" { 550 name = "allow_all_%d" 551 description = "Allow all inbound traffic" 552 vpc_id = "${aws_vpc.main.id}" 553 554 ingress { 555 from_port = 0 556 to_port = 0 557 protocol = "-1" 558 cidr_blocks = ["0.0.0.0/0"] 559 } 560 561 egress { 562 from_port = 0 563 to_port = 0 564 protocol = "-1" 565 cidr_blocks = ["0.0.0.0/0"] 566 } 567 568 depends_on = ["aws_subnet.main"] 569 570 lifecycle { 571 ignore_changes = ["ingress", "egress"] 572 } 573 574 tags { 575 name = "emr_test" 576 } 577 } 578 579 resource "aws_vpc" "main" { 580 cidr_block = "168.31.0.0/16" 581 enable_dns_hostnames = true 582 583 tags { 584 name = "emr_test_%d" 585 } 586 } 587 588 resource "aws_subnet" "main" { 589 vpc_id = "${aws_vpc.main.id}" 590 cidr_block = "168.31.0.0/20" 591 592 tags { 593 name = "emr_test_%d" 594 } 595 } 596 597 resource "aws_internet_gateway" "gw" { 598 vpc_id = "${aws_vpc.main.id}" 599 } 600 601 resource "aws_route_table" "r" { 602 vpc_id = "${aws_vpc.main.id}" 603 604 route { 605 cidr_block = "0.0.0.0/0" 606 gateway_id = "${aws_internet_gateway.gw.id}" 607 } 608 } 609 610 resource "aws_main_route_table_association" "a" { 611 vpc_id = "${aws_vpc.main.id}" 612 route_table_id = "${aws_route_table.r.id}" 613 } 614 615 ### 616 617 # IAM things 618 619 ### 620 621 # IAM role for EMR Service 622 resource "aws_iam_role" "iam_emr_default_role" { 623 name = "iam_emr_default_role_%d" 624 625 assume_role_policy = <<EOT 626 { 627 "Version": "2008-10-17", 628 "Statement": [ 629 { 630 "Sid": "", 631 "Effect": "Allow", 632 "Principal": { 633 "Service": "elasticmapreduce.amazonaws.com" 634 }, 635 "Action": "sts:AssumeRole" 636 } 637 ] 638 } 639 EOT 640 } 641 642 resource "aws_iam_role_policy_attachment" "service-attach" { 643 role = "${aws_iam_role.iam_emr_default_role.id}" 644 policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}" 645 } 646 647 resource "aws_iam_policy" "iam_emr_default_policy" { 648 name = "iam_emr_default_policy_%d" 649 650 policy = <<EOT 651 { 652 "Version": "2012-10-17", 653 "Statement": [{ 654 "Effect": "Allow", 655 "Resource": "*", 656 "Action": [ 657 "ec2:AuthorizeSecurityGroupEgress", 658 "ec2:AuthorizeSecurityGroupIngress", 659 "ec2:CancelSpotInstanceRequests", 660 "ec2:CreateNetworkInterface", 661 "ec2:CreateSecurityGroup", 662 "ec2:CreateTags", 663 "ec2:DeleteNetworkInterface", 664 "ec2:DeleteSecurityGroup", 665 "ec2:DeleteTags", 666 "ec2:DescribeAvailabilityZones", 667 "ec2:DescribeAccountAttributes", 668 "ec2:DescribeDhcpOptions", 669 "ec2:DescribeInstanceStatus", 670 "ec2:DescribeInstances", 671 "ec2:DescribeKeyPairs", 672 "ec2:DescribeNetworkAcls", 673 "ec2:DescribeNetworkInterfaces", 674 "ec2:DescribePrefixLists", 675 "ec2:DescribeRouteTables", 676 "ec2:DescribeSecurityGroups", 677 "ec2:DescribeSpotInstanceRequests", 678 "ec2:DescribeSpotPriceHistory", 679 "ec2:DescribeSubnets", 680 "ec2:DescribeVpcAttribute", 681 "ec2:DescribeVpcEndpoints", 682 "ec2:DescribeVpcEndpointServices", 683 "ec2:DescribeVpcs", 684 "ec2:DetachNetworkInterface", 685 "ec2:ModifyImageAttribute", 686 "ec2:ModifyInstanceAttribute", 687 "ec2:RequestSpotInstances", 688 "ec2:RevokeSecurityGroupEgress", 689 "ec2:RunInstances", 690 "ec2:TerminateInstances", 691 "ec2:DeleteVolume", 692 "ec2:DescribeVolumeStatus", 693 "ec2:DescribeVolumes", 694 "ec2:DetachVolume", 695 "iam:GetRole", 696 "iam:GetRolePolicy", 697 "iam:ListInstanceProfiles", 698 "iam:ListRolePolicies", 699 "iam:PassRole", 700 "s3:CreateBucket", 701 "s3:Get*", 702 "s3:List*", 703 "sdb:BatchPutAttributes", 704 "sdb:Select", 705 "sqs:CreateQueue", 706 "sqs:Delete*", 707 "sqs:GetQueue*", 708 "sqs:PurgeQueue", 709 "sqs:ReceiveMessage" 710 ] 711 }] 712 } 713 EOT 714 } 715 716 # IAM Role for EC2 Instance Profile 717 resource "aws_iam_role" "iam_emr_profile_role" { 718 name = "iam_emr_profile_role_%d" 719 720 assume_role_policy = <<EOT 721 { 722 "Version": "2008-10-17", 723 "Statement": [ 724 { 725 "Sid": "", 726 "Effect": "Allow", 727 "Principal": { 728 "Service": "ec2.amazonaws.com" 729 }, 730 "Action": "sts:AssumeRole" 731 } 732 ] 733 } 734 EOT 735 } 736 737 resource "aws_iam_instance_profile" "emr_profile" { 738 name = "emr_profile_%d" 739 roles = ["${aws_iam_role.iam_emr_profile_role.name}"] 740 } 741 742 resource "aws_iam_role_policy_attachment" "profile-attach" { 743 role = "${aws_iam_role.iam_emr_profile_role.id}" 744 policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}" 745 } 746 747 resource "aws_iam_policy" "iam_emr_profile_policy" { 748 name = "iam_emr_profile_policy_%d" 749 750 policy = <<EOT 751 { 752 "Version": "2012-10-17", 753 "Statement": [{ 754 "Effect": "Allow", 755 "Resource": "*", 756 "Action": [ 757 "cloudwatch:*", 758 "dynamodb:*", 759 "ec2:Describe*", 760 "elasticmapreduce:Describe*", 761 "elasticmapreduce:ListBootstrapActions", 762 "elasticmapreduce:ListClusters", 763 "elasticmapreduce:ListInstanceGroups", 764 "elasticmapreduce:ListInstances", 765 "elasticmapreduce:ListSteps", 766 "kinesis:CreateStream", 767 "kinesis:DeleteStream", 768 "kinesis:DescribeStream", 769 "kinesis:GetRecords", 770 "kinesis:GetShardIterator", 771 "kinesis:MergeShards", 772 "kinesis:PutRecord", 773 "kinesis:SplitShard", 774 "rds:Describe*", 775 "s3:*", 776 "sdb:*", 777 "sns:*", 778 "sqs:*" 779 ] 780 }] 781 } 782 EOT 783 } 784 785 # IAM Role for autoscaling 786 resource "aws_iam_role" "emr-autoscaling-role" { 787 name = "EMR_AutoScaling_DefaultRole_%d" 788 assume_role_policy = "${data.aws_iam_policy_document.emr-autoscaling-role-policy.json}" 789 } 790 791 data "aws_iam_policy_document" "emr-autoscaling-role-policy" { 792 statement { 793 effect = "Allow" 794 actions = ["sts:AssumeRole"] 795 796 principals = { 797 type = "Service" 798 identifiers = ["elasticmapreduce.amazonaws.com","application-autoscaling.amazonaws.com"] 799 } 800 } 801 } 802 803 resource "aws_iam_role_policy_attachment" "emr-autoscaling-role" { 804 role = "${aws_iam_role.emr-autoscaling-role.name}" 805 policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" 806 } 807 `, r, r, r, r, r, r, r, r, r, r) 808 } 809 810 func testAccAWSEmrClusterConfigVisibleToAllUsersUpdated(r int) string { 811 return fmt.Sprintf(` 812 provider "aws" { 813 region = "us-west-2" 814 } 815 816 resource "aws_emr_cluster" "tf-test-cluster" { 817 name = "emr-test-%d" 818 release_label = "emr-4.6.0" 819 applications = ["Spark"] 820 821 ec2_attributes { 822 subnet_id = "${aws_subnet.main.id}" 823 emr_managed_master_security_group = "${aws_security_group.allow_all.id}" 824 emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" 825 instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" 826 } 827 828 master_instance_type = "m3.xlarge" 829 core_instance_type = "m3.xlarge" 830 core_instance_count = 1 831 832 tags { 833 role = "rolename" 834 dns_zone = "env_zone" 835 env = "env" 836 name = "name-env" 837 } 838 839 keep_job_flow_alive_when_no_steps = true 840 visible_to_all_users = false 841 842 bootstrap_action { 843 path = "s3://elasticmapreduce/bootstrap-actions/run-if" 844 name = "runif" 845 args = ["instance.isMaster=true", "echo running on master node"] 846 } 847 848 configurations = "test-fixtures/emr_configurations.json" 849 850 depends_on = ["aws_main_route_table_association.a"] 851 852 service_role = "${aws_iam_role.iam_emr_default_role.arn}" 853 autoscaling_role = "${aws_iam_role.emr-autoscaling-role.arn}" 854 } 855 856 resource "aws_security_group" "allow_all" { 857 name = "allow_all_%d" 858 description = "Allow all inbound traffic" 859 vpc_id = "${aws_vpc.main.id}" 860 861 ingress { 862 from_port = 0 863 to_port = 0 864 protocol = "-1" 865 cidr_blocks = ["0.0.0.0/0"] 866 } 867 868 egress { 869 from_port = 0 870 to_port = 0 871 protocol = "-1" 872 cidr_blocks = ["0.0.0.0/0"] 873 } 874 875 depends_on = ["aws_subnet.main"] 876 877 lifecycle { 878 ignore_changes = ["ingress", "egress"] 879 } 880 881 tags { 882 name = "emr_test" 883 } 884 } 885 886 resource "aws_vpc" "main" { 887 cidr_block = "168.31.0.0/16" 888 enable_dns_hostnames = true 889 890 tags { 891 name = "emr_test_%d" 892 } 893 } 894 895 resource "aws_subnet" "main" { 896 vpc_id = "${aws_vpc.main.id}" 897 cidr_block = "168.31.0.0/20" 898 899 tags { 900 name = "emr_test_%d" 901 } 902 } 903 904 resource "aws_internet_gateway" "gw" { 905 vpc_id = "${aws_vpc.main.id}" 906 } 907 908 resource "aws_route_table" "r" { 909 vpc_id = "${aws_vpc.main.id}" 910 911 route { 912 cidr_block = "0.0.0.0/0" 913 gateway_id = "${aws_internet_gateway.gw.id}" 914 } 915 } 916 917 resource "aws_main_route_table_association" "a" { 918 vpc_id = "${aws_vpc.main.id}" 919 route_table_id = "${aws_route_table.r.id}" 920 } 921 922 ### 923 924 # IAM things 925 926 ### 927 928 # IAM role for EMR Service 929 resource "aws_iam_role" "iam_emr_default_role" { 930 name = "iam_emr_default_role_%d" 931 932 assume_role_policy = <<EOT 933 { 934 "Version": "2008-10-17", 935 "Statement": [ 936 { 937 "Sid": "", 938 "Effect": "Allow", 939 "Principal": { 940 "Service": "elasticmapreduce.amazonaws.com" 941 }, 942 "Action": "sts:AssumeRole" 943 } 944 ] 945 } 946 EOT 947 } 948 949 resource "aws_iam_role_policy_attachment" "service-attach" { 950 role = "${aws_iam_role.iam_emr_default_role.id}" 951 policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}" 952 } 953 954 resource "aws_iam_policy" "iam_emr_default_policy" { 955 name = "iam_emr_default_policy_%d" 956 957 policy = <<EOT 958 { 959 "Version": "2012-10-17", 960 "Statement": [{ 961 "Effect": "Allow", 962 "Resource": "*", 963 "Action": [ 964 "ec2:AuthorizeSecurityGroupEgress", 965 "ec2:AuthorizeSecurityGroupIngress", 966 "ec2:CancelSpotInstanceRequests", 967 "ec2:CreateNetworkInterface", 968 "ec2:CreateSecurityGroup", 969 "ec2:CreateTags", 970 "ec2:DeleteNetworkInterface", 971 "ec2:DeleteSecurityGroup", 972 "ec2:DeleteTags", 973 "ec2:DescribeAvailabilityZones", 974 "ec2:DescribeAccountAttributes", 975 "ec2:DescribeDhcpOptions", 976 "ec2:DescribeInstanceStatus", 977 "ec2:DescribeInstances", 978 "ec2:DescribeKeyPairs", 979 "ec2:DescribeNetworkAcls", 980 "ec2:DescribeNetworkInterfaces", 981 "ec2:DescribePrefixLists", 982 "ec2:DescribeRouteTables", 983 "ec2:DescribeSecurityGroups", 984 "ec2:DescribeSpotInstanceRequests", 985 "ec2:DescribeSpotPriceHistory", 986 "ec2:DescribeSubnets", 987 "ec2:DescribeVpcAttribute", 988 "ec2:DescribeVpcEndpoints", 989 "ec2:DescribeVpcEndpointServices", 990 "ec2:DescribeVpcs", 991 "ec2:DetachNetworkInterface", 992 "ec2:ModifyImageAttribute", 993 "ec2:ModifyInstanceAttribute", 994 "ec2:RequestSpotInstances", 995 "ec2:RevokeSecurityGroupEgress", 996 "ec2:RunInstances", 997 "ec2:TerminateInstances", 998 "ec2:DeleteVolume", 999 "ec2:DescribeVolumeStatus", 1000 "ec2:DescribeVolumes", 1001 "ec2:DetachVolume", 1002 "iam:GetRole", 1003 "iam:GetRolePolicy", 1004 "iam:ListInstanceProfiles", 1005 "iam:ListRolePolicies", 1006 "iam:PassRole", 1007 "s3:CreateBucket", 1008 "s3:Get*", 1009 "s3:List*", 1010 "sdb:BatchPutAttributes", 1011 "sdb:Select", 1012 "sqs:CreateQueue", 1013 "sqs:Delete*", 1014 "sqs:GetQueue*", 1015 "sqs:PurgeQueue", 1016 "sqs:ReceiveMessage" 1017 ] 1018 }] 1019 } 1020 EOT 1021 } 1022 1023 # IAM Role for EC2 Instance Profile 1024 resource "aws_iam_role" "iam_emr_profile_role" { 1025 name = "iam_emr_profile_role_%d" 1026 1027 assume_role_policy = <<EOT 1028 { 1029 "Version": "2008-10-17", 1030 "Statement": [ 1031 { 1032 "Sid": "", 1033 "Effect": "Allow", 1034 "Principal": { 1035 "Service": "ec2.amazonaws.com" 1036 }, 1037 "Action": "sts:AssumeRole" 1038 } 1039 ] 1040 } 1041 EOT 1042 } 1043 1044 resource "aws_iam_instance_profile" "emr_profile" { 1045 name = "emr_profile_%d" 1046 roles = ["${aws_iam_role.iam_emr_profile_role.name}"] 1047 } 1048 1049 resource "aws_iam_role_policy_attachment" "profile-attach" { 1050 role = "${aws_iam_role.iam_emr_profile_role.id}" 1051 policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}" 1052 } 1053 1054 resource "aws_iam_policy" "iam_emr_profile_policy" { 1055 name = "iam_emr_profile_policy_%d" 1056 1057 policy = <<EOT 1058 { 1059 "Version": "2012-10-17", 1060 "Statement": [{ 1061 "Effect": "Allow", 1062 "Resource": "*", 1063 "Action": [ 1064 "cloudwatch:*", 1065 "dynamodb:*", 1066 "ec2:Describe*", 1067 "elasticmapreduce:Describe*", 1068 "elasticmapreduce:ListBootstrapActions", 1069 "elasticmapreduce:ListClusters", 1070 "elasticmapreduce:ListInstanceGroups", 1071 "elasticmapreduce:ListInstances", 1072 "elasticmapreduce:ListSteps", 1073 "kinesis:CreateStream", 1074 "kinesis:DeleteStream", 1075 "kinesis:DescribeStream", 1076 "kinesis:GetRecords", 1077 "kinesis:GetShardIterator", 1078 "kinesis:MergeShards", 1079 "kinesis:PutRecord", 1080 "kinesis:SplitShard", 1081 "rds:Describe*", 1082 "s3:*", 1083 "sdb:*", 1084 "sns:*", 1085 "sqs:*" 1086 ] 1087 }] 1088 } 1089 EOT 1090 } 1091 1092 # IAM Role for autoscaling 1093 resource "aws_iam_role" "emr-autoscaling-role" { 1094 name = "EMR_AutoScaling_DefaultRole_%d" 1095 assume_role_policy = "${data.aws_iam_policy_document.emr-autoscaling-role-policy.json}" 1096 } 1097 1098 data "aws_iam_policy_document" "emr-autoscaling-role-policy" { 1099 statement { 1100 effect = "Allow" 1101 actions = ["sts:AssumeRole"] 1102 1103 principals = { 1104 type = "Service" 1105 identifiers = ["elasticmapreduce.amazonaws.com","application-autoscaling.amazonaws.com"] 1106 } 1107 } 1108 } 1109 1110 resource "aws_iam_role_policy_attachment" "emr-autoscaling-role" { 1111 role = "${aws_iam_role.emr-autoscaling-role.name}" 1112 policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" 1113 } 1114 `, r, r, r, r, r, r, r, r, r, r) 1115 } 1116 1117 func testAccAWSEmrClusterConfigUpdatedTags(r int) string { 1118 return fmt.Sprintf(` 1119 provider "aws" { 1120 region = "us-west-2" 1121 } 1122 1123 resource "aws_emr_cluster" "tf-test-cluster" { 1124 name = "emr-test-%d" 1125 release_label = "emr-4.6.0" 1126 applications = ["Spark"] 1127 1128 ec2_attributes { 1129 subnet_id = "${aws_subnet.main.id}" 1130 emr_managed_master_security_group = "${aws_security_group.allow_all.id}" 1131 emr_managed_slave_security_group = "${aws_security_group.allow_all.id}" 1132 instance_profile = "${aws_iam_instance_profile.emr_profile.arn}" 1133 } 1134 1135 master_instance_type = "m3.xlarge" 1136 core_instance_type = "m3.xlarge" 1137 core_instance_count = 1 1138 1139 tags { 1140 dns_zone = "new_zone" 1141 Env = "production" 1142 name = "name-env" 1143 } 1144 1145 keep_job_flow_alive_when_no_steps = true 1146 termination_protection = false 1147 1148 bootstrap_action { 1149 path = "s3://elasticmapreduce/bootstrap-actions/run-if" 1150 name = "runif" 1151 args = ["instance.isMaster=true", "echo running on master node"] 1152 } 1153 1154 configurations = "test-fixtures/emr_configurations.json" 1155 1156 depends_on = ["aws_main_route_table_association.a"] 1157 1158 service_role = "${aws_iam_role.iam_emr_default_role.arn}" 1159 autoscaling_role = "${aws_iam_role.emr-autoscaling-role.arn}" 1160 } 1161 1162 resource "aws_security_group" "allow_all" { 1163 name = "allow_all_%d" 1164 description = "Allow all inbound traffic" 1165 vpc_id = "${aws_vpc.main.id}" 1166 1167 ingress { 1168 from_port = 0 1169 to_port = 0 1170 protocol = "-1" 1171 cidr_blocks = ["0.0.0.0/0"] 1172 } 1173 1174 egress { 1175 from_port = 0 1176 to_port = 0 1177 protocol = "-1" 1178 cidr_blocks = ["0.0.0.0/0"] 1179 } 1180 1181 depends_on = ["aws_subnet.main"] 1182 1183 lifecycle { 1184 ignore_changes = ["ingress", "egress"] 1185 } 1186 1187 tags { 1188 name = "emr_test" 1189 } 1190 } 1191 1192 resource "aws_vpc" "main" { 1193 cidr_block = "168.31.0.0/16" 1194 enable_dns_hostnames = true 1195 1196 tags { 1197 name = "emr_test_%d" 1198 } 1199 } 1200 1201 resource "aws_subnet" "main" { 1202 vpc_id = "${aws_vpc.main.id}" 1203 cidr_block = "168.31.0.0/20" 1204 1205 tags { 1206 name = "emr_test_%d" 1207 } 1208 } 1209 1210 resource "aws_internet_gateway" "gw" { 1211 vpc_id = "${aws_vpc.main.id}" 1212 } 1213 1214 resource "aws_route_table" "r" { 1215 vpc_id = "${aws_vpc.main.id}" 1216 1217 route { 1218 cidr_block = "0.0.0.0/0" 1219 gateway_id = "${aws_internet_gateway.gw.id}" 1220 } 1221 } 1222 1223 resource "aws_main_route_table_association" "a" { 1224 vpc_id = "${aws_vpc.main.id}" 1225 route_table_id = "${aws_route_table.r.id}" 1226 } 1227 1228 ### 1229 1230 # IAM things 1231 1232 ### 1233 1234 # IAM role for EMR Service 1235 resource "aws_iam_role" "iam_emr_default_role" { 1236 name = "iam_emr_default_role_%d" 1237 1238 assume_role_policy = <<EOT 1239 { 1240 "Version": "2008-10-17", 1241 "Statement": [ 1242 { 1243 "Sid": "", 1244 "Effect": "Allow", 1245 "Principal": { 1246 "Service": "elasticmapreduce.amazonaws.com" 1247 }, 1248 "Action": "sts:AssumeRole" 1249 } 1250 ] 1251 } 1252 EOT 1253 } 1254 1255 resource "aws_iam_role_policy_attachment" "service-attach" { 1256 role = "${aws_iam_role.iam_emr_default_role.id}" 1257 policy_arn = "${aws_iam_policy.iam_emr_default_policy.arn}" 1258 } 1259 1260 resource "aws_iam_policy" "iam_emr_default_policy" { 1261 name = "iam_emr_default_policy_%d" 1262 1263 policy = <<EOT 1264 { 1265 "Version": "2012-10-17", 1266 "Statement": [{ 1267 "Effect": "Allow", 1268 "Resource": "*", 1269 "Action": [ 1270 "ec2:AuthorizeSecurityGroupEgress", 1271 "ec2:AuthorizeSecurityGroupIngress", 1272 "ec2:CancelSpotInstanceRequests", 1273 "ec2:CreateNetworkInterface", 1274 "ec2:CreateSecurityGroup", 1275 "ec2:CreateTags", 1276 "ec2:DeleteNetworkInterface", 1277 "ec2:DeleteSecurityGroup", 1278 "ec2:DeleteTags", 1279 "ec2:DescribeAvailabilityZones", 1280 "ec2:DescribeAccountAttributes", 1281 "ec2:DescribeDhcpOptions", 1282 "ec2:DescribeInstanceStatus", 1283 "ec2:DescribeInstances", 1284 "ec2:DescribeKeyPairs", 1285 "ec2:DescribeNetworkAcls", 1286 "ec2:DescribeNetworkInterfaces", 1287 "ec2:DescribePrefixLists", 1288 "ec2:DescribeRouteTables", 1289 "ec2:DescribeSecurityGroups", 1290 "ec2:DescribeSpotInstanceRequests", 1291 "ec2:DescribeSpotPriceHistory", 1292 "ec2:DescribeSubnets", 1293 "ec2:DescribeVpcAttribute", 1294 "ec2:DescribeVpcEndpoints", 1295 "ec2:DescribeVpcEndpointServices", 1296 "ec2:DescribeVpcs", 1297 "ec2:DetachNetworkInterface", 1298 "ec2:ModifyImageAttribute", 1299 "ec2:ModifyInstanceAttribute", 1300 "ec2:RequestSpotInstances", 1301 "ec2:RevokeSecurityGroupEgress", 1302 "ec2:RunInstances", 1303 "ec2:TerminateInstances", 1304 "ec2:DeleteVolume", 1305 "ec2:DescribeVolumeStatus", 1306 "ec2:DescribeVolumes", 1307 "ec2:DetachVolume", 1308 "iam:GetRole", 1309 "iam:GetRolePolicy", 1310 "iam:ListInstanceProfiles", 1311 "iam:ListRolePolicies", 1312 "iam:PassRole", 1313 "s3:CreateBucket", 1314 "s3:Get*", 1315 "s3:List*", 1316 "sdb:BatchPutAttributes", 1317 "sdb:Select", 1318 "sqs:CreateQueue", 1319 "sqs:Delete*", 1320 "sqs:GetQueue*", 1321 "sqs:PurgeQueue", 1322 "sqs:ReceiveMessage" 1323 ] 1324 }] 1325 } 1326 EOT 1327 } 1328 1329 # IAM Role for EC2 Instance Profile 1330 resource "aws_iam_role" "iam_emr_profile_role" { 1331 name = "iam_emr_profile_role_%d" 1332 1333 assume_role_policy = <<EOT 1334 { 1335 "Version": "2008-10-17", 1336 "Statement": [ 1337 { 1338 "Sid": "", 1339 "Effect": "Allow", 1340 "Principal": { 1341 "Service": "ec2.amazonaws.com" 1342 }, 1343 "Action": "sts:AssumeRole" 1344 } 1345 ] 1346 } 1347 EOT 1348 } 1349 1350 resource "aws_iam_instance_profile" "emr_profile" { 1351 name = "emr_profile_%d" 1352 roles = ["${aws_iam_role.iam_emr_profile_role.name}"] 1353 } 1354 1355 resource "aws_iam_role_policy_attachment" "profile-attach" { 1356 role = "${aws_iam_role.iam_emr_profile_role.id}" 1357 policy_arn = "${aws_iam_policy.iam_emr_profile_policy.arn}" 1358 } 1359 1360 resource "aws_iam_policy" "iam_emr_profile_policy" { 1361 name = "iam_emr_profile_policy_%d" 1362 1363 policy = <<EOT 1364 { 1365 "Version": "2012-10-17", 1366 "Statement": [{ 1367 "Effect": "Allow", 1368 "Resource": "*", 1369 "Action": [ 1370 "cloudwatch:*", 1371 "dynamodb:*", 1372 "ec2:Describe*", 1373 "elasticmapreduce:Describe*", 1374 "elasticmapreduce:ListBootstrapActions", 1375 "elasticmapreduce:ListClusters", 1376 "elasticmapreduce:ListInstanceGroups", 1377 "elasticmapreduce:ListInstances", 1378 "elasticmapreduce:ListSteps", 1379 "kinesis:CreateStream", 1380 "kinesis:DeleteStream", 1381 "kinesis:DescribeStream", 1382 "kinesis:GetRecords", 1383 "kinesis:GetShardIterator", 1384 "kinesis:MergeShards", 1385 "kinesis:PutRecord", 1386 "kinesis:SplitShard", 1387 "rds:Describe*", 1388 "s3:*", 1389 "sdb:*", 1390 "sns:*", 1391 "sqs:*" 1392 ] 1393 }] 1394 } 1395 EOT 1396 } 1397 1398 # IAM Role for autoscaling 1399 resource "aws_iam_role" "emr-autoscaling-role" { 1400 name = "EMR_AutoScaling_DefaultRole_%d" 1401 assume_role_policy = "${data.aws_iam_policy_document.emr-autoscaling-role-policy.json}" 1402 } 1403 1404 data "aws_iam_policy_document" "emr-autoscaling-role-policy" { 1405 statement { 1406 effect = "Allow" 1407 actions = ["sts:AssumeRole"] 1408 1409 principals = { 1410 type = "Service" 1411 identifiers = ["elasticmapreduce.amazonaws.com","application-autoscaling.amazonaws.com"] 1412 } 1413 } 1414 } 1415 1416 resource "aws_iam_role_policy_attachment" "emr-autoscaling-role" { 1417 role = "${aws_iam_role.emr-autoscaling-role.name}" 1418 policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonElasticMapReduceforAutoScalingRole" 1419 } 1420 `, r, r, r, r, r, r, r, r, r, r) 1421 }