github.com/emc-advanced-dev/unik@v0.0.0-20190717152701-a58d3e8e33b7/pkg/providers/aws/aws_import.go (about) 1 package aws 2 3 import ( 4 "encoding/xml" 5 "os" 6 "time" 7 8 "bytes" 9 "io" 10 11 "fmt" 12 13 log "github.com/sirupsen/logrus" 14 15 "github.com/aws/aws-sdk-go/aws" 16 "github.com/aws/aws-sdk-go/service/ec2" 17 "github.com/aws/aws-sdk-go/service/s3" 18 19 "math/rand" 20 21 "github.com/emc-advanced-dev/pkg/errors" 22 "github.com/solo-io/unik/pkg/types" 23 "strings" 24 ) 25 26 func init() { 27 rand.Seed(time.Now().UnixNano()) 28 } 29 30 func createDataVolumeFromRawImage(s3svc *s3.S3, ec2svc *ec2.EC2, imgFile string, imageSize int64, imageFormat types.ImageFormat, az string) (string, error) { 31 fileInfo, err := os.Stat(imgFile) 32 if err != nil { 33 return "", err 34 } 35 36 // upload the image file to aws 37 bucket := fmt.Sprintf("unik-tmp-%d", rand.Int63()) 38 39 if err := createBucket(s3svc, bucket); err != nil { 40 return "", err 41 } 42 defer deleteBucket(s3svc, bucket) 43 44 pathInBucket := "disk.img" 45 46 log.Debug("Uploading image to aws") 47 48 if err := uploadFileToAws(s3svc, imgFile, fileInfo.Size(), bucket, pathInBucket); err != nil { 49 return "", err 50 } 51 52 log.Debug("Creating self sign urls") 53 54 // create signed urls for the file (get, head, delete) 55 // s.s3svc. 56 57 getReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{ 58 Bucket: aws.String(bucket), 59 Key: aws.String(pathInBucket), 60 }) 61 getUrlStr, err := getReq.Presign(24 * time.Hour) 62 if err != nil { 63 return "", err 64 } 65 66 headReq, _ := s3svc.HeadObjectRequest(&s3.HeadObjectInput{ 67 Bucket: aws.String(bucket), 68 Key: aws.String(pathInBucket), 69 }) 70 71 headUrlStr, err := headReq.Presign(24 * time.Hour) 72 if err != nil { 73 return "", err 74 } 75 76 deleteReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{ 77 Bucket: aws.String(bucket), 78 Key: aws.String(pathInBucket), 79 }) 80 81 deleteUrlStr, err := deleteReq.Presign(24 * time.Hour) 82 if err != nil { 83 return "", err 84 } 85 86 log.Debug("Creating manifest") 87 88 // create manifest 89 manifestName := "upload-manifest.xml" 90 91 deleteManiReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{ 92 Bucket: aws.String(bucket), 93 Key: aws.String(manifestName), 94 }) 95 96 deleteManiUrlStr, err := deleteManiReq.Presign(24 * time.Hour) 97 if err != nil { 98 return "", err 99 } 100 101 m := manifest{ 102 Version: "2010-11-15", 103 FileFormat: strings.ToUpper(string(imageFormat)), 104 Importer: importer{"unik", "1", "2016-04-01"}, 105 SelfDestructUrl: deleteManiUrlStr, 106 ImportSpec: importSpec{ 107 Size: fileInfo.Size(), 108 VolumeSize: toGigs(imageSize), 109 Parts: parts{ 110 Count: 1, 111 Parts: []part{ 112 part{ 113 Index: 0, 114 ByteRange: byteRange{ 115 Start: 0, 116 End: fileInfo.Size(), 117 }, 118 Key: pathInBucket, 119 HeadUrl: headUrlStr, 120 GetUrl: getUrlStr, 121 DeleteUrl: deleteUrlStr, 122 }, 123 }, 124 }, 125 }, 126 } 127 // write manifest 128 buf := new(bytes.Buffer) 129 enc := xml.NewEncoder(buf) 130 if err := enc.Encode(m); err != nil { 131 return "", err 132 } 133 log.Debug("Uploading manifest") 134 135 // upload manifest 136 manifestBytes := buf.Bytes() 137 err = uploadToAws(s3svc, bytes.NewReader(manifestBytes), int64(len(manifestBytes)), bucket, manifestName) 138 if err != nil { 139 return "", err 140 } 141 142 getManiReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{ 143 Bucket: aws.String(bucket), 144 Key: aws.String(manifestName), 145 }) 146 getManiUrlStr, err := getManiReq.Presign(24 * time.Hour) 147 if err != nil { 148 return "", err 149 } 150 151 log.Debug("Importing volume") 152 153 // finally import the image 154 volparams := &ec2.ImportVolumeInput{ 155 AvailabilityZone: aws.String(az), // Required 156 Image: &ec2.DiskImageDetail{ // Required 157 Bytes: aws.Int64(toGigs(imageSize)), // Required 158 Format: aws.String(strings.ToUpper(string(imageFormat))), // Required 159 ImportManifestUrl: aws.String(getManiUrlStr), // Required 160 }, 161 Volume: &ec2.VolumeDetail{ // Required 162 Size: aws.Int64(toGigs(imageSize)), // Required 163 }, 164 } 165 task, err := ec2svc.ImportVolume(volparams) 166 if err != nil { 167 return "", err 168 } 169 log.WithFields(log.Fields{"task": *task}).Debug("Import task result") 170 171 taskInput := &ec2.DescribeConversionTasksInput{ 172 ConversionTaskIds: []*string{task.ConversionTask.ConversionTaskId}, 173 } 174 175 log.Debug("Waiting for task") 176 err = ec2svc.WaitUntilConversionTaskCompleted(taskInput) 177 178 if err != nil { 179 return "", err 180 } 181 182 log.Debug("Task done") 183 // hopefully successful! 184 convTaskOutput, err := ec2svc.DescribeConversionTasks(taskInput) 185 186 if err != nil { 187 return "", err 188 } 189 190 log.WithFields(log.Fields{"task": *convTaskOutput}).Debug("Convertion task result") 191 192 if len(convTaskOutput.ConversionTasks) != 1 { 193 return "", errors.New("Unexpected number of tasks", nil) 194 } 195 convTask := convTaskOutput.ConversionTasks[0] 196 197 if convTask.ImportVolume == nil { 198 return "", errors.New("No volume information", nil) 199 } 200 201 return *convTask.ImportVolume.Volume.Id, nil 202 203 } 204 205 func uploadFileToAws(s3svc *s3.S3, file string, fileSize int64, bucket, path string) error { 206 reader, err := os.Open(file) 207 if err != nil { 208 return nil 209 } 210 defer reader.Close() 211 return uploadToAws(s3svc, reader, fileSize, bucket, path) 212 } 213 214 func uploadToAws(s3svc *s3.S3, body io.ReadSeeker, size int64, bucket, path string) error { 215 216 // upload 217 params := &s3.PutObjectInput{ 218 Bucket: aws.String(bucket), // required 219 Key: aws.String(path), // required 220 ACL: aws.String("private"), 221 Body: body, 222 ContentLength: aws.Int64(size), 223 ContentType: aws.String("application/octet-stream"), 224 } 225 226 _, err := s3svc.PutObject(params) 227 228 if err != nil { 229 return err 230 } 231 return nil 232 } 233 234 func toGigs(i int64) int64 { 235 return 1 + (i >> 30) 236 } 237 238 func createBucket(s3svc *s3.S3, bucketName string) error { 239 240 log.WithFields(log.Fields{"name": bucketName}).Debug("Creating Bucket ") 241 242 params := &s3.CreateBucketInput{ 243 Bucket: aws.String(bucketName), // Required 244 } 245 _, err := s3svc.CreateBucket(params) 246 247 if err != nil { 248 return err 249 } 250 251 return nil 252 } 253 254 func deleteBucket(s3svc *s3.S3, bucketName string) error { 255 log.WithFields(log.Fields{"name": bucketName}).Debug("Deleting Bucket ") 256 //first delete objects 257 listObjectParams := &s3.ListObjectsInput{ 258 Bucket: aws.String(bucketName), 259 } 260 objects, err := s3svc.ListObjects(listObjectParams) 261 if err != nil { 262 return err 263 } 264 for _, object := range objects.Contents { 265 deleteObjectParams := &s3.DeleteObjectInput{ 266 Bucket: aws.String(bucketName), 267 Key: object.Key, 268 } 269 _, err := s3svc.DeleteObject(deleteObjectParams) 270 if err != nil { 271 return err 272 } 273 } 274 275 params := &s3.DeleteBucketInput{ 276 Bucket: aws.String(bucketName), // Required 277 } 278 _, err = s3svc.DeleteBucket(params) 279 return err 280 } 281 282 func deleteSnapshot(e2svc *ec2.EC2, snapshotId string) error { 283 param := &ec2.DeleteSnapshotInput{ 284 SnapshotId: aws.String(snapshotId), 285 } 286 _, err := e2svc.DeleteSnapshot(param) 287 return err 288 } 289 290 func deleteVolume(e2svc *ec2.EC2, volumeId string) error { 291 param := &ec2.DeleteVolumeInput{ 292 VolumeId: aws.String(volumeId), 293 } 294 _, err := e2svc.DeleteVolume(param) 295 return err 296 } 297 298 type manifest struct { 299 XMLName xml.Name `xml:"manifest"` 300 301 Version string `xml:"version"` 302 FileFormat string `xml:"file-format"` 303 Importer importer `xml:"importer"` 304 SelfDestructUrl string `xml:"self-destruct-url"` 305 306 ImportSpec importSpec `xml:"import"` 307 } 308 309 type importer struct { 310 Name string `xml:"name"` 311 Version string `xml:"version"` 312 Release string `xml:"release"` 313 } 314 315 type importSpec struct { 316 Size int64 `xml:"size"` 317 VolumeSize int64 `xml:"volume-size"` 318 Parts parts `xml:"parts"` 319 } 320 type parts struct { 321 Count int `xml:"count,attr"` 322 Parts []part `xml:"part"` 323 } 324 325 type part struct { 326 Index int `xml:"index,attr"` 327 ByteRange byteRange `xml:"byte-range"` 328 Key string `xml:"key"` 329 HeadUrl string `xml:"head-url"` 330 GetUrl string `xml:"get-url"` 331 DeleteUrl string `xml:"delete-url"` 332 } 333 type byteRange struct { 334 Start int64 `xml:"start,attr"` 335 End int64 `xml:"end,attr"` 336 }