github.com/lymingtonprecision/terraform@v0.9.9-0.20170613092852-62acef9611a9/backend/remote-state/s3/client.go (about) 1 package s3 2 3 import ( 4 "bytes" 5 "crypto/md5" 6 "encoding/hex" 7 "encoding/json" 8 "errors" 9 "fmt" 10 "io" 11 "log" 12 "time" 13 14 "github.com/aws/aws-sdk-go/aws" 15 "github.com/aws/aws-sdk-go/aws/awserr" 16 "github.com/aws/aws-sdk-go/service/dynamodb" 17 "github.com/aws/aws-sdk-go/service/s3" 18 multierror "github.com/hashicorp/go-multierror" 19 uuid "github.com/hashicorp/go-uuid" 20 "github.com/hashicorp/terraform/state" 21 "github.com/hashicorp/terraform/state/remote" 22 ) 23 24 // Store the last saved serial in dynamo with this suffix for consistency checks. 25 const stateIDSuffix = "-md5" 26 27 type RemoteClient struct { 28 s3Client *s3.S3 29 dynClient *dynamodb.DynamoDB 30 bucketName string 31 path string 32 serverSideEncryption bool 33 acl string 34 kmsKeyID string 35 ddbTable string 36 } 37 38 var ( 39 // The amount of time we will retry a state waiting for it to match the 40 // expected checksum. 41 consistencyRetryTimeout = 10 * time.Second 42 43 // delay when polling the state 44 consistencyRetryPollInterval = 2 * time.Second 45 ) 46 47 // test hook called when checksums don't match 48 var testChecksumHook func() 49 50 func (c *RemoteClient) Get() (payload *remote.Payload, err error) { 51 deadline := time.Now().Add(consistencyRetryTimeout) 52 53 // If we have a checksum, and the returned payload doesn't match, we retry 54 // up until deadline. 55 for { 56 payload, err = c.get() 57 if err != nil { 58 return nil, err 59 } 60 61 // If the remote state was manually removed the payload will be nil, 62 // but if there's still a digest entry for that state we will still try 63 // to compare the MD5 below. 64 var digest []byte 65 if payload != nil { 66 digest = payload.MD5 67 } 68 69 // verify that this state is what we expect 70 if expected, err := c.getMD5(); err != nil { 71 log.Printf("[WARNING] failed to fetch state md5: %s", err) 72 } else if len(expected) > 0 && !bytes.Equal(expected, digest) { 73 log.Printf("[WARNING] state md5 mismatch: expected '%x', got '%x'", expected, digest) 74 75 if testChecksumHook != nil { 76 testChecksumHook() 77 } 78 79 if time.Now().Before(deadline) { 80 time.Sleep(consistencyRetryPollInterval) 81 log.Println("[INFO] retrying S3 RemoteClient.Get...") 82 continue 83 } 84 85 return nil, fmt.Errorf(errBadChecksumFmt, digest) 86 } 87 88 break 89 } 90 91 return payload, err 92 } 93 94 func (c *RemoteClient) get() (*remote.Payload, error) { 95 output, err := c.s3Client.GetObject(&s3.GetObjectInput{ 96 Bucket: &c.bucketName, 97 Key: &c.path, 98 }) 99 100 if err != nil { 101 if awserr := err.(awserr.Error); awserr != nil { 102 if awserr.Code() == "NoSuchKey" { 103 return nil, nil 104 } else { 105 return nil, err 106 } 107 } else { 108 return nil, err 109 } 110 } 111 112 defer output.Body.Close() 113 114 buf := bytes.NewBuffer(nil) 115 if _, err := io.Copy(buf, output.Body); err != nil { 116 return nil, fmt.Errorf("Failed to read remote state: %s", err) 117 } 118 119 sum := md5.Sum(buf.Bytes()) 120 payload := &remote.Payload{ 121 Data: buf.Bytes(), 122 MD5: sum[:], 123 } 124 125 // If there was no data, then return nil 126 if len(payload.Data) == 0 { 127 return nil, nil 128 } 129 130 return payload, nil 131 } 132 133 func (c *RemoteClient) Put(data []byte) error { 134 contentType := "application/json" 135 contentLength := int64(len(data)) 136 137 i := &s3.PutObjectInput{ 138 ContentType: &contentType, 139 ContentLength: &contentLength, 140 Body: bytes.NewReader(data), 141 Bucket: &c.bucketName, 142 Key: &c.path, 143 } 144 145 if c.serverSideEncryption { 146 if c.kmsKeyID != "" { 147 i.SSEKMSKeyId = &c.kmsKeyID 148 i.ServerSideEncryption = aws.String("aws:kms") 149 } else { 150 i.ServerSideEncryption = aws.String("AES256") 151 } 152 } 153 154 if c.acl != "" { 155 i.ACL = aws.String(c.acl) 156 } 157 158 log.Printf("[DEBUG] Uploading remote state to S3: %#v", i) 159 160 _, err := c.s3Client.PutObject(i) 161 if err != nil { 162 return fmt.Errorf("Failed to upload state: %v", err) 163 } 164 165 sum := md5.Sum(data) 166 if err := c.putMD5(sum[:]); err != nil { 167 // if this errors out, we unfortunately have to error out altogether, 168 // since the next Get will inevitably fail. 169 return fmt.Errorf("failed to store state MD5: %s", err) 170 171 } 172 173 return nil 174 } 175 176 func (c *RemoteClient) Delete() error { 177 _, err := c.s3Client.DeleteObject(&s3.DeleteObjectInput{ 178 Bucket: &c.bucketName, 179 Key: &c.path, 180 }) 181 182 if err != nil { 183 return err 184 } 185 186 if err := c.deleteMD5(); err != nil { 187 log.Printf("error deleting state md5: %s", err) 188 } 189 190 return nil 191 } 192 193 func (c *RemoteClient) Lock(info *state.LockInfo) (string, error) { 194 if c.ddbTable == "" { 195 return "", nil 196 } 197 198 info.Path = c.lockPath() 199 200 if info.ID == "" { 201 lockID, err := uuid.GenerateUUID() 202 if err != nil { 203 return "", err 204 } 205 206 info.ID = lockID 207 } 208 209 putParams := &dynamodb.PutItemInput{ 210 Item: map[string]*dynamodb.AttributeValue{ 211 "LockID": {S: aws.String(c.lockPath())}, 212 "Info": {S: aws.String(string(info.Marshal()))}, 213 }, 214 TableName: aws.String(c.ddbTable), 215 ConditionExpression: aws.String("attribute_not_exists(LockID)"), 216 } 217 _, err := c.dynClient.PutItem(putParams) 218 219 if err != nil { 220 lockInfo, infoErr := c.getLockInfo() 221 if infoErr != nil { 222 err = multierror.Append(err, infoErr) 223 } 224 225 lockErr := &state.LockError{ 226 Err: err, 227 Info: lockInfo, 228 } 229 return "", lockErr 230 } 231 232 return info.ID, nil 233 } 234 235 func (c *RemoteClient) getMD5() ([]byte, error) { 236 if c.ddbTable == "" { 237 return nil, nil 238 } 239 240 getParams := &dynamodb.GetItemInput{ 241 Key: map[string]*dynamodb.AttributeValue{ 242 "LockID": {S: aws.String(c.lockPath() + stateIDSuffix)}, 243 }, 244 ProjectionExpression: aws.String("LockID, Digest"), 245 TableName: aws.String(c.ddbTable), 246 } 247 248 resp, err := c.dynClient.GetItem(getParams) 249 if err != nil { 250 return nil, err 251 } 252 253 var val string 254 if v, ok := resp.Item["Digest"]; ok && v.S != nil { 255 val = *v.S 256 } 257 258 sum, err := hex.DecodeString(val) 259 if err != nil || len(sum) != md5.Size { 260 return nil, errors.New("invalid md5") 261 } 262 263 return sum, nil 264 } 265 266 // store the hash of the state to that clients can check for stale state files. 267 func (c *RemoteClient) putMD5(sum []byte) error { 268 if c.ddbTable == "" { 269 return nil 270 } 271 272 if len(sum) != md5.Size { 273 return errors.New("invalid payload md5") 274 } 275 276 putParams := &dynamodb.PutItemInput{ 277 Item: map[string]*dynamodb.AttributeValue{ 278 "LockID": {S: aws.String(c.lockPath() + stateIDSuffix)}, 279 "Digest": {S: aws.String(hex.EncodeToString(sum))}, 280 }, 281 TableName: aws.String(c.ddbTable), 282 } 283 _, err := c.dynClient.PutItem(putParams) 284 if err != nil { 285 log.Printf("[WARNING] failed to record state serial in dynamodb: %s", err) 286 } 287 288 return nil 289 } 290 291 // remove the hash value for a deleted state 292 func (c *RemoteClient) deleteMD5() error { 293 if c.ddbTable == "" { 294 return nil 295 } 296 297 params := &dynamodb.DeleteItemInput{ 298 Key: map[string]*dynamodb.AttributeValue{ 299 "LockID": {S: aws.String(c.lockPath() + stateIDSuffix)}, 300 }, 301 TableName: aws.String(c.ddbTable), 302 } 303 if _, err := c.dynClient.DeleteItem(params); err != nil { 304 return err 305 } 306 return nil 307 } 308 309 func (c *RemoteClient) getLockInfo() (*state.LockInfo, error) { 310 getParams := &dynamodb.GetItemInput{ 311 Key: map[string]*dynamodb.AttributeValue{ 312 "LockID": {S: aws.String(c.lockPath())}, 313 }, 314 ProjectionExpression: aws.String("LockID, Info"), 315 TableName: aws.String(c.ddbTable), 316 } 317 318 resp, err := c.dynClient.GetItem(getParams) 319 if err != nil { 320 return nil, err 321 } 322 323 var infoData string 324 if v, ok := resp.Item["Info"]; ok && v.S != nil { 325 infoData = *v.S 326 } 327 328 lockInfo := &state.LockInfo{} 329 err = json.Unmarshal([]byte(infoData), lockInfo) 330 if err != nil { 331 return nil, err 332 } 333 334 return lockInfo, nil 335 } 336 337 func (c *RemoteClient) Unlock(id string) error { 338 if c.ddbTable == "" { 339 return nil 340 } 341 342 lockErr := &state.LockError{} 343 344 // TODO: store the path and lock ID in separate fields, and have proper 345 // projection expression only delete the lock if both match, rather than 346 // checking the ID from the info field first. 347 lockInfo, err := c.getLockInfo() 348 if err != nil { 349 lockErr.Err = fmt.Errorf("failed to retrieve lock info: %s", err) 350 return lockErr 351 } 352 lockErr.Info = lockInfo 353 354 if lockInfo.ID != id { 355 lockErr.Err = fmt.Errorf("lock id %q does not match existing lock", id) 356 return lockErr 357 } 358 359 params := &dynamodb.DeleteItemInput{ 360 Key: map[string]*dynamodb.AttributeValue{ 361 "LockID": {S: aws.String(c.lockPath())}, 362 }, 363 TableName: aws.String(c.ddbTable), 364 } 365 _, err = c.dynClient.DeleteItem(params) 366 367 if err != nil { 368 lockErr.Err = err 369 return lockErr 370 } 371 return nil 372 } 373 374 func (c *RemoteClient) lockPath() string { 375 return fmt.Sprintf("%s/%s", c.bucketName, c.path) 376 } 377 378 const errBadChecksumFmt = `state data in S3 does not have the expected content. 379 380 This may be caused by unusually long delays in S3 processing a previous state 381 update. Please wait for a minute or two and try again. If this problem 382 persists, and neither S3 nor DynamoDB are experiencing an outage, you may need 383 to manually verify the remote state and update the Digest value stored in the 384 DynamoDB table to the following value: %x 385 `