github.com/gabrielperezs/terraform@v0.7.0-rc2.0.20160715084931-f7da2612946f/builtin/providers/aws/resource_aws_kinesis_firehose_delivery_stream.go (about) 1 package aws 2 3 import ( 4 "fmt" 5 "log" 6 "strings" 7 "time" 8 9 "github.com/aws/aws-sdk-go/aws" 10 "github.com/aws/aws-sdk-go/aws/awserr" 11 "github.com/aws/aws-sdk-go/service/firehose" 12 "github.com/hashicorp/terraform/helper/resource" 13 "github.com/hashicorp/terraform/helper/schema" 14 ) 15 16 func resourceAwsKinesisFirehoseDeliveryStream() *schema.Resource { 17 return &schema.Resource{ 18 Create: resourceAwsKinesisFirehoseDeliveryStreamCreate, 19 Read: resourceAwsKinesisFirehoseDeliveryStreamRead, 20 Update: resourceAwsKinesisFirehoseDeliveryStreamUpdate, 21 Delete: resourceAwsKinesisFirehoseDeliveryStreamDelete, 22 23 SchemaVersion: 1, 24 MigrateState: resourceAwsKinesisFirehoseMigrateState, 25 Schema: map[string]*schema.Schema{ 26 "name": &schema.Schema{ 27 Type: schema.TypeString, 28 Required: true, 29 ForceNew: true, 30 }, 31 32 "destination": &schema.Schema{ 33 Type: schema.TypeString, 34 Required: true, 35 ForceNew: true, 36 StateFunc: func(v interface{}) string { 37 value := v.(string) 38 return strings.ToLower(value) 39 }, 40 }, 41 42 // elements removed in v0.7.0 43 "role_arn": &schema.Schema{ 44 Type: schema.TypeString, 45 Optional: true, 46 Removed: "role_arn has been removed. Use a s3_configuration block instead. See https://terraform.io/docs/providers/aws/r/kinesis_firehose_delivery_stream.html", 47 }, 48 49 "s3_bucket_arn": &schema.Schema{ 50 Type: schema.TypeString, 51 Optional: true, 52 Removed: "s3_bucket_arn has been removed. Use a s3_configuration block instead. See https://terraform.io/docs/providers/aws/r/kinesis_firehose_delivery_stream.html", 53 }, 54 55 "s3_prefix": &schema.Schema{ 56 Type: schema.TypeString, 57 Optional: true, 58 Removed: "s3_prefix has been removed. Use a s3_configuration block instead. See https://terraform.io/docs/providers/aws/r/kinesis_firehose_delivery_stream.html", 59 }, 60 61 "s3_buffer_size": &schema.Schema{ 62 Type: schema.TypeInt, 63 Optional: true, 64 Removed: "s3_buffer_size has been removed. Use a s3_configuration block instead. See https://terraform.io/docs/providers/aws/r/kinesis_firehose_delivery_stream.html", 65 }, 66 67 "s3_buffer_interval": &schema.Schema{ 68 Type: schema.TypeInt, 69 Optional: true, 70 Removed: "s3_buffer_interval has been removed. Use a s3_configuration block instead. See https://terraform.io/docs/providers/aws/r/kinesis_firehose_delivery_stream.html", 71 }, 72 73 "s3_data_compression": &schema.Schema{ 74 Type: schema.TypeString, 75 Optional: true, 76 Removed: "s3_data_compression has been removed. Use a s3_configuration block instead. See https://terraform.io/docs/providers/aws/r/kinesis_firehose_delivery_stream.html", 77 }, 78 79 "s3_configuration": &schema.Schema{ 80 Type: schema.TypeList, 81 Required: true, 82 MaxItems: 1, 83 Elem: &schema.Resource{ 84 Schema: map[string]*schema.Schema{ 85 "bucket_arn": &schema.Schema{ 86 Type: schema.TypeString, 87 Required: true, 88 }, 89 90 "buffer_size": &schema.Schema{ 91 Type: schema.TypeInt, 92 Optional: true, 93 Default: 5, 94 }, 95 96 "buffer_interval": &schema.Schema{ 97 Type: schema.TypeInt, 98 Optional: true, 99 Default: 300, 100 }, 101 102 "compression_format": &schema.Schema{ 103 Type: schema.TypeString, 104 Optional: true, 105 Default: "UNCOMPRESSED", 106 }, 107 108 "kms_key_arn": &schema.Schema{ 109 Type: schema.TypeString, 110 Optional: true, 111 }, 112 113 "role_arn": &schema.Schema{ 114 Type: schema.TypeString, 115 Required: true, 116 }, 117 118 "prefix": &schema.Schema{ 119 Type: schema.TypeString, 120 Optional: true, 121 }, 122 }, 123 }, 124 }, 125 126 "redshift_configuration": &schema.Schema{ 127 Type: schema.TypeList, 128 Optional: true, 129 MaxItems: 1, 130 Elem: &schema.Resource{ 131 Schema: map[string]*schema.Schema{ 132 "cluster_jdbcurl": &schema.Schema{ 133 Type: schema.TypeString, 134 Required: true, 135 }, 136 137 "username": &schema.Schema{ 138 Type: schema.TypeString, 139 Required: true, 140 }, 141 142 "password": &schema.Schema{ 143 Type: schema.TypeString, 144 Required: true, 145 }, 146 147 "role_arn": &schema.Schema{ 148 Type: schema.TypeString, 149 Required: true, 150 }, 151 152 "copy_options": &schema.Schema{ 153 Type: schema.TypeString, 154 Optional: true, 155 }, 156 157 "data_table_columns": &schema.Schema{ 158 Type: schema.TypeString, 159 Optional: true, 160 }, 161 162 "data_table_name": &schema.Schema{ 163 Type: schema.TypeString, 164 Required: true, 165 }, 166 }, 167 }, 168 }, 169 170 "arn": &schema.Schema{ 171 Type: schema.TypeString, 172 Optional: true, 173 Computed: true, 174 }, 175 176 "version_id": &schema.Schema{ 177 Type: schema.TypeString, 178 Optional: true, 179 Computed: true, 180 }, 181 182 "destination_id": &schema.Schema{ 183 Type: schema.TypeString, 184 Optional: true, 185 Computed: true, 186 }, 187 }, 188 } 189 } 190 191 func validateConfiguration(d *schema.ResourceData) error { 192 destination := d.Get("destination").(string) 193 if destination != "s3" && destination != "redshift" { 194 return fmt.Errorf("[ERROR] Destination must be s3 or redshift") 195 } 196 197 return nil 198 } 199 200 func createS3Config(d *schema.ResourceData) *firehose.S3DestinationConfiguration { 201 s3 := d.Get("s3_configuration").([]interface{})[0].(map[string]interface{}) 202 203 return &firehose.S3DestinationConfiguration{ 204 BucketARN: aws.String(s3["bucket_arn"].(string)), 205 RoleARN: aws.String(s3["role_arn"].(string)), 206 BufferingHints: &firehose.BufferingHints{ 207 IntervalInSeconds: aws.Int64(int64(s3["buffer_interval"].(int))), 208 SizeInMBs: aws.Int64(int64(s3["buffer_size"].(int))), 209 }, 210 Prefix: extractPrefixConfiguration(s3), 211 CompressionFormat: aws.String(s3["compression_format"].(string)), 212 EncryptionConfiguration: extractEncryptionConfiguration(s3), 213 } 214 } 215 216 func updateS3Config(d *schema.ResourceData) *firehose.S3DestinationUpdate { 217 s3 := d.Get("s3_configuration").([]interface{})[0].(map[string]interface{}) 218 219 return &firehose.S3DestinationUpdate{ 220 BucketARN: aws.String(s3["bucket_arn"].(string)), 221 RoleARN: aws.String(s3["role_arn"].(string)), 222 BufferingHints: &firehose.BufferingHints{ 223 IntervalInSeconds: aws.Int64((int64)(s3["buffer_interval"].(int))), 224 SizeInMBs: aws.Int64((int64)(s3["buffer_size"].(int))), 225 }, 226 Prefix: extractPrefixConfiguration(s3), 227 CompressionFormat: aws.String(s3["compression_format"].(string)), 228 EncryptionConfiguration: extractEncryptionConfiguration(s3), 229 } 230 } 231 232 func extractEncryptionConfiguration(s3 map[string]interface{}) *firehose.EncryptionConfiguration { 233 if key, ok := s3["kms_key_arn"]; ok && len(key.(string)) > 0 { 234 return &firehose.EncryptionConfiguration{ 235 KMSEncryptionConfig: &firehose.KMSEncryptionConfig{ 236 AWSKMSKeyARN: aws.String(key.(string)), 237 }, 238 } 239 } 240 241 return &firehose.EncryptionConfiguration{ 242 NoEncryptionConfig: aws.String("NoEncryption"), 243 } 244 } 245 246 func extractPrefixConfiguration(s3 map[string]interface{}) *string { 247 if v, ok := s3["prefix"]; ok { 248 return aws.String(v.(string)) 249 } 250 251 return nil 252 } 253 254 func createRedshiftConfig(d *schema.ResourceData, s3Config *firehose.S3DestinationConfiguration) (*firehose.RedshiftDestinationConfiguration, error) { 255 redshiftRaw, ok := d.GetOk("redshift_configuration") 256 if !ok { 257 return nil, fmt.Errorf("[ERR] Error loading Redshift Configuration for Kinesis Firehose: redshift_configuration not found") 258 } 259 rl := redshiftRaw.([]interface{}) 260 261 redshift := rl[0].(map[string]interface{}) 262 263 return &firehose.RedshiftDestinationConfiguration{ 264 ClusterJDBCURL: aws.String(redshift["cluster_jdbcurl"].(string)), 265 Password: aws.String(redshift["password"].(string)), 266 Username: aws.String(redshift["username"].(string)), 267 RoleARN: aws.String(redshift["role_arn"].(string)), 268 CopyCommand: extractCopyCommandConfiguration(redshift), 269 S3Configuration: s3Config, 270 }, nil 271 } 272 273 func updateRedshiftConfig(d *schema.ResourceData, s3Update *firehose.S3DestinationUpdate) (*firehose.RedshiftDestinationUpdate, error) { 274 redshiftRaw, ok := d.GetOk("redshift_configuration") 275 if !ok { 276 return nil, fmt.Errorf("[ERR] Error loading Redshift Configuration for Kinesis Firehose: redshift_configuration not found") 277 } 278 rl := redshiftRaw.([]interface{}) 279 280 redshift := rl[0].(map[string]interface{}) 281 282 return &firehose.RedshiftDestinationUpdate{ 283 ClusterJDBCURL: aws.String(redshift["cluster_jdbcurl"].(string)), 284 Password: aws.String(redshift["password"].(string)), 285 Username: aws.String(redshift["username"].(string)), 286 RoleARN: aws.String(redshift["role_arn"].(string)), 287 CopyCommand: extractCopyCommandConfiguration(redshift), 288 S3Update: s3Update, 289 }, nil 290 } 291 292 func extractCopyCommandConfiguration(redshift map[string]interface{}) *firehose.CopyCommand { 293 cmd := &firehose.CopyCommand{ 294 DataTableName: aws.String(redshift["data_table_name"].(string)), 295 } 296 if copyOptions, ok := redshift["copy_options"]; ok { 297 cmd.CopyOptions = aws.String(copyOptions.(string)) 298 } 299 if columns, ok := redshift["data_table_columns"]; ok { 300 cmd.DataTableColumns = aws.String(columns.(string)) 301 } 302 303 return cmd 304 } 305 306 func resourceAwsKinesisFirehoseDeliveryStreamCreate(d *schema.ResourceData, meta interface{}) error { 307 conn := meta.(*AWSClient).firehoseconn 308 309 if err := validateConfiguration(d); err != nil { 310 return err 311 } 312 313 sn := d.Get("name").(string) 314 s3Config := createS3Config(d) 315 316 createInput := &firehose.CreateDeliveryStreamInput{ 317 DeliveryStreamName: aws.String(sn), 318 } 319 320 if d.Get("destination").(string) == "s3" { 321 createInput.S3DestinationConfiguration = s3Config 322 } else { 323 rc, err := createRedshiftConfig(d, s3Config) 324 if err != nil { 325 return err 326 } 327 createInput.RedshiftDestinationConfiguration = rc 328 } 329 330 var lastError error 331 err := resource.Retry(1*time.Minute, func() *resource.RetryError { 332 _, err := conn.CreateDeliveryStream(createInput) 333 if err != nil { 334 log.Printf("[DEBUG] Error creating Firehose Delivery Stream: %s", err) 335 lastError = err 336 337 if awsErr, ok := err.(awserr.Error); ok { 338 // IAM roles can take ~10 seconds to propagate in AWS: 339 // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html#launch-instance-with-role-console 340 if awsErr.Code() == "InvalidArgumentException" && strings.Contains(awsErr.Message(), "Firehose is unable to assume role") { 341 log.Printf("[DEBUG] Firehose could not assume role referenced, retrying...") 342 return resource.RetryableError(awsErr) 343 } 344 } 345 // Not retryable 346 return resource.NonRetryableError(err) 347 } 348 349 return nil 350 }) 351 if err != nil { 352 if awsErr, ok := lastError.(awserr.Error); ok { 353 return fmt.Errorf("[WARN] Error creating Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code()) 354 } 355 return err 356 } 357 358 stateConf := &resource.StateChangeConf{ 359 Pending: []string{"CREATING"}, 360 Target: []string{"ACTIVE"}, 361 Refresh: firehoseStreamStateRefreshFunc(conn, sn), 362 Timeout: 5 * time.Minute, 363 Delay: 10 * time.Second, 364 MinTimeout: 3 * time.Second, 365 } 366 367 firehoseStream, err := stateConf.WaitForState() 368 if err != nil { 369 return fmt.Errorf( 370 "Error waiting for Kinesis Stream (%s) to become active: %s", 371 sn, err) 372 } 373 374 s := firehoseStream.(*firehose.DeliveryStreamDescription) 375 d.SetId(*s.DeliveryStreamARN) 376 d.Set("arn", s.DeliveryStreamARN) 377 378 return resourceAwsKinesisFirehoseDeliveryStreamRead(d, meta) 379 } 380 381 func resourceAwsKinesisFirehoseDeliveryStreamUpdate(d *schema.ResourceData, meta interface{}) error { 382 conn := meta.(*AWSClient).firehoseconn 383 384 if err := validateConfiguration(d); err != nil { 385 return err 386 } 387 388 sn := d.Get("name").(string) 389 s3Config := updateS3Config(d) 390 391 updateInput := &firehose.UpdateDestinationInput{ 392 DeliveryStreamName: aws.String(sn), 393 CurrentDeliveryStreamVersionId: aws.String(d.Get("version_id").(string)), 394 DestinationId: aws.String(d.Get("destination_id").(string)), 395 } 396 397 if d.Get("destination").(string) == "s3" { 398 updateInput.S3DestinationUpdate = s3Config 399 } else { 400 rc, err := updateRedshiftConfig(d, s3Config) 401 if err != nil { 402 return err 403 } 404 updateInput.RedshiftDestinationUpdate = rc 405 } 406 407 _, err := conn.UpdateDestination(updateInput) 408 if err != nil { 409 return fmt.Errorf( 410 "Error Updating Kinesis Firehose Delivery Stream: \"%s\"\n%s", 411 sn, err) 412 } 413 414 return resourceAwsKinesisFirehoseDeliveryStreamRead(d, meta) 415 } 416 417 func resourceAwsKinesisFirehoseDeliveryStreamRead(d *schema.ResourceData, meta interface{}) error { 418 conn := meta.(*AWSClient).firehoseconn 419 420 resp, err := conn.DescribeDeliveryStream(&firehose.DescribeDeliveryStreamInput{ 421 DeliveryStreamName: aws.String(d.Get("name").(string)), 422 }) 423 424 if err != nil { 425 if awsErr, ok := err.(awserr.Error); ok { 426 if awsErr.Code() == "ResourceNotFoundException" { 427 d.SetId("") 428 return nil 429 } 430 return fmt.Errorf("[WARN] Error reading Kinesis Firehose Delivery Stream: \"%s\", code: \"%s\"", awsErr.Message(), awsErr.Code()) 431 } 432 return err 433 } 434 435 s := resp.DeliveryStreamDescription 436 d.Set("version_id", s.VersionId) 437 d.Set("arn", *s.DeliveryStreamARN) 438 if len(s.Destinations) > 0 { 439 destination := s.Destinations[0] 440 d.Set("destination_id", *destination.DestinationId) 441 } 442 443 return nil 444 } 445 446 func resourceAwsKinesisFirehoseDeliveryStreamDelete(d *schema.ResourceData, meta interface{}) error { 447 conn := meta.(*AWSClient).firehoseconn 448 449 sn := d.Get("name").(string) 450 _, err := conn.DeleteDeliveryStream(&firehose.DeleteDeliveryStreamInput{ 451 DeliveryStreamName: aws.String(sn), 452 }) 453 454 if err != nil { 455 return err 456 } 457 458 stateConf := &resource.StateChangeConf{ 459 Pending: []string{"DELETING"}, 460 Target: []string{"DESTROYED"}, 461 Refresh: firehoseStreamStateRefreshFunc(conn, sn), 462 Timeout: 5 * time.Minute, 463 Delay: 10 * time.Second, 464 MinTimeout: 3 * time.Second, 465 } 466 467 _, err = stateConf.WaitForState() 468 if err != nil { 469 return fmt.Errorf( 470 "Error waiting for Delivery Stream (%s) to be destroyed: %s", 471 sn, err) 472 } 473 474 d.SetId("") 475 return nil 476 } 477 478 func firehoseStreamStateRefreshFunc(conn *firehose.Firehose, sn string) resource.StateRefreshFunc { 479 return func() (interface{}, string, error) { 480 describeOpts := &firehose.DescribeDeliveryStreamInput{ 481 DeliveryStreamName: aws.String(sn), 482 } 483 resp, err := conn.DescribeDeliveryStream(describeOpts) 484 if err != nil { 485 if awsErr, ok := err.(awserr.Error); ok { 486 if awsErr.Code() == "ResourceNotFoundException" { 487 return 42, "DESTROYED", nil 488 } 489 return nil, awsErr.Code(), err 490 } 491 return nil, "failed", err 492 } 493 494 return resp.DeliveryStreamDescription, *resp.DeliveryStreamDescription.DeliveryStreamStatus, nil 495 } 496 }