github.com/emc-advanced-dev/unik@v0.0.0-20190717152701-a58d3e8e33b7/pkg/providers/aws/aws_import.go (about)

     1  package aws
     2  
     3  import (
     4  	"encoding/xml"
     5  	"os"
     6  	"time"
     7  
     8  	"bytes"
     9  	"io"
    10  
    11  	"fmt"
    12  
    13  	log "github.com/sirupsen/logrus"
    14  
    15  	"github.com/aws/aws-sdk-go/aws"
    16  	"github.com/aws/aws-sdk-go/service/ec2"
    17  	"github.com/aws/aws-sdk-go/service/s3"
    18  
    19  	"math/rand"
    20  
    21  	"github.com/emc-advanced-dev/pkg/errors"
    22  	"github.com/solo-io/unik/pkg/types"
    23  	"strings"
    24  )
    25  
    26  func init() {
    27  	rand.Seed(time.Now().UnixNano())
    28  }
    29  
    30  func createDataVolumeFromRawImage(s3svc *s3.S3, ec2svc *ec2.EC2, imgFile string, imageSize int64, imageFormat types.ImageFormat, az string) (string, error) {
    31  	fileInfo, err := os.Stat(imgFile)
    32  	if err != nil {
    33  		return "", err
    34  	}
    35  
    36  	// upload the image file to aws
    37  	bucket := fmt.Sprintf("unik-tmp-%d", rand.Int63())
    38  
    39  	if err := createBucket(s3svc, bucket); err != nil {
    40  		return "", err
    41  	}
    42  	defer deleteBucket(s3svc, bucket)
    43  
    44  	pathInBucket := "disk.img"
    45  
    46  	log.Debug("Uploading image to aws")
    47  
    48  	if err := uploadFileToAws(s3svc, imgFile, fileInfo.Size(), bucket, pathInBucket); err != nil {
    49  		return "", err
    50  	}
    51  
    52  	log.Debug("Creating self sign urls")
    53  
    54  	// create signed urls for the file (get, head, delete)
    55  	// s.s3svc.
    56  
    57  	getReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{
    58  		Bucket: aws.String(bucket),
    59  		Key:    aws.String(pathInBucket),
    60  	})
    61  	getUrlStr, err := getReq.Presign(24 * time.Hour)
    62  	if err != nil {
    63  		return "", err
    64  	}
    65  
    66  	headReq, _ := s3svc.HeadObjectRequest(&s3.HeadObjectInput{
    67  		Bucket: aws.String(bucket),
    68  		Key:    aws.String(pathInBucket),
    69  	})
    70  
    71  	headUrlStr, err := headReq.Presign(24 * time.Hour)
    72  	if err != nil {
    73  		return "", err
    74  	}
    75  
    76  	deleteReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{
    77  		Bucket: aws.String(bucket),
    78  		Key:    aws.String(pathInBucket),
    79  	})
    80  
    81  	deleteUrlStr, err := deleteReq.Presign(24 * time.Hour)
    82  	if err != nil {
    83  		return "", err
    84  	}
    85  
    86  	log.Debug("Creating manifest")
    87  
    88  	// create manifest
    89  	manifestName := "upload-manifest.xml"
    90  
    91  	deleteManiReq, _ := s3svc.DeleteObjectRequest(&s3.DeleteObjectInput{
    92  		Bucket: aws.String(bucket),
    93  		Key:    aws.String(manifestName),
    94  	})
    95  
    96  	deleteManiUrlStr, err := deleteManiReq.Presign(24 * time.Hour)
    97  	if err != nil {
    98  		return "", err
    99  	}
   100  
   101  	m := manifest{
   102  		Version:         "2010-11-15",
   103  		FileFormat:      strings.ToUpper(string(imageFormat)),
   104  		Importer:        importer{"unik", "1", "2016-04-01"},
   105  		SelfDestructUrl: deleteManiUrlStr,
   106  		ImportSpec: importSpec{
   107  			Size:       fileInfo.Size(),
   108  			VolumeSize: toGigs(imageSize),
   109  			Parts: parts{
   110  				Count: 1,
   111  				Parts: []part{
   112  					part{
   113  						Index: 0,
   114  						ByteRange: byteRange{
   115  							Start: 0,
   116  							End:   fileInfo.Size(),
   117  						},
   118  						Key:       pathInBucket,
   119  						HeadUrl:   headUrlStr,
   120  						GetUrl:    getUrlStr,
   121  						DeleteUrl: deleteUrlStr,
   122  					},
   123  				},
   124  			},
   125  		},
   126  	}
   127  	// write manifest
   128  	buf := new(bytes.Buffer)
   129  	enc := xml.NewEncoder(buf)
   130  	if err := enc.Encode(m); err != nil {
   131  		return "", err
   132  	}
   133  	log.Debug("Uploading manifest")
   134  
   135  	// upload manifest
   136  	manifestBytes := buf.Bytes()
   137  	err = uploadToAws(s3svc, bytes.NewReader(manifestBytes), int64(len(manifestBytes)), bucket, manifestName)
   138  	if err != nil {
   139  		return "", err
   140  	}
   141  
   142  	getManiReq, _ := s3svc.GetObjectRequest(&s3.GetObjectInput{
   143  		Bucket: aws.String(bucket),
   144  		Key:    aws.String(manifestName),
   145  	})
   146  	getManiUrlStr, err := getManiReq.Presign(24 * time.Hour)
   147  	if err != nil {
   148  		return "", err
   149  	}
   150  
   151  	log.Debug("Importing volume")
   152  
   153  	// finally import the image
   154  	volparams := &ec2.ImportVolumeInput{
   155  		AvailabilityZone: aws.String(az), // Required
   156  		Image: &ec2.DiskImageDetail{ // Required
   157  			Bytes:             aws.Int64(toGigs(imageSize)),                     // Required
   158  			Format:            aws.String(strings.ToUpper(string(imageFormat))), // Required
   159  			ImportManifestUrl: aws.String(getManiUrlStr),                        // Required
   160  		},
   161  		Volume: &ec2.VolumeDetail{ // Required
   162  			Size: aws.Int64(toGigs(imageSize)), // Required
   163  		},
   164  	}
   165  	task, err := ec2svc.ImportVolume(volparams)
   166  	if err != nil {
   167  		return "", err
   168  	}
   169  	log.WithFields(log.Fields{"task": *task}).Debug("Import task result")
   170  
   171  	taskInput := &ec2.DescribeConversionTasksInput{
   172  		ConversionTaskIds: []*string{task.ConversionTask.ConversionTaskId},
   173  	}
   174  
   175  	log.Debug("Waiting for task")
   176  	err = ec2svc.WaitUntilConversionTaskCompleted(taskInput)
   177  
   178  	if err != nil {
   179  		return "", err
   180  	}
   181  
   182  	log.Debug("Task done")
   183  	// hopefully successful!
   184  	convTaskOutput, err := ec2svc.DescribeConversionTasks(taskInput)
   185  
   186  	if err != nil {
   187  		return "", err
   188  	}
   189  
   190  	log.WithFields(log.Fields{"task": *convTaskOutput}).Debug("Convertion task result")
   191  
   192  	if len(convTaskOutput.ConversionTasks) != 1 {
   193  		return "", errors.New("Unexpected number of tasks", nil)
   194  	}
   195  	convTask := convTaskOutput.ConversionTasks[0]
   196  
   197  	if convTask.ImportVolume == nil {
   198  		return "", errors.New("No volume information", nil)
   199  	}
   200  
   201  	return *convTask.ImportVolume.Volume.Id, nil
   202  
   203  }
   204  
   205  func uploadFileToAws(s3svc *s3.S3, file string, fileSize int64, bucket, path string) error {
   206  	reader, err := os.Open(file)
   207  	if err != nil {
   208  		return nil
   209  	}
   210  	defer reader.Close()
   211  	return uploadToAws(s3svc, reader, fileSize, bucket, path)
   212  }
   213  
   214  func uploadToAws(s3svc *s3.S3, body io.ReadSeeker, size int64, bucket, path string) error {
   215  
   216  	// upload
   217  	params := &s3.PutObjectInput{
   218  		Bucket:        aws.String(bucket), // required
   219  		Key:           aws.String(path),   // required
   220  		ACL:           aws.String("private"),
   221  		Body:          body,
   222  		ContentLength: aws.Int64(size),
   223  		ContentType:   aws.String("application/octet-stream"),
   224  	}
   225  
   226  	_, err := s3svc.PutObject(params)
   227  
   228  	if err != nil {
   229  		return err
   230  	}
   231  	return nil
   232  }
   233  
   234  func toGigs(i int64) int64 {
   235  	return 1 + (i >> 30)
   236  }
   237  
   238  func createBucket(s3svc *s3.S3, bucketName string) error {
   239  
   240  	log.WithFields(log.Fields{"name": bucketName}).Debug("Creating Bucket ")
   241  
   242  	params := &s3.CreateBucketInput{
   243  		Bucket: aws.String(bucketName), // Required
   244  	}
   245  	_, err := s3svc.CreateBucket(params)
   246  
   247  	if err != nil {
   248  		return err
   249  	}
   250  
   251  	return nil
   252  }
   253  
   254  func deleteBucket(s3svc *s3.S3, bucketName string) error {
   255  	log.WithFields(log.Fields{"name": bucketName}).Debug("Deleting Bucket ")
   256  	//first delete objects
   257  	listObjectParams := &s3.ListObjectsInput{
   258  		Bucket: aws.String(bucketName),
   259  	}
   260  	objects, err := s3svc.ListObjects(listObjectParams)
   261  	if err != nil {
   262  		return err
   263  	}
   264  	for _, object := range objects.Contents {
   265  		deleteObjectParams := &s3.DeleteObjectInput{
   266  			Bucket: aws.String(bucketName),
   267  			Key:    object.Key,
   268  		}
   269  		_, err := s3svc.DeleteObject(deleteObjectParams)
   270  		if err != nil {
   271  			return err
   272  		}
   273  	}
   274  
   275  	params := &s3.DeleteBucketInput{
   276  		Bucket: aws.String(bucketName), // Required
   277  	}
   278  	_, err = s3svc.DeleteBucket(params)
   279  	return err
   280  }
   281  
   282  func deleteSnapshot(e2svc *ec2.EC2, snapshotId string) error {
   283  	param := &ec2.DeleteSnapshotInput{
   284  		SnapshotId: aws.String(snapshotId),
   285  	}
   286  	_, err := e2svc.DeleteSnapshot(param)
   287  	return err
   288  }
   289  
   290  func deleteVolume(e2svc *ec2.EC2, volumeId string) error {
   291  	param := &ec2.DeleteVolumeInput{
   292  		VolumeId: aws.String(volumeId),
   293  	}
   294  	_, err := e2svc.DeleteVolume(param)
   295  	return err
   296  }
   297  
   298  type manifest struct {
   299  	XMLName xml.Name `xml:"manifest"`
   300  
   301  	Version         string   `xml:"version"`
   302  	FileFormat      string   `xml:"file-format"`
   303  	Importer        importer `xml:"importer"`
   304  	SelfDestructUrl string   `xml:"self-destruct-url"`
   305  
   306  	ImportSpec importSpec `xml:"import"`
   307  }
   308  
   309  type importer struct {
   310  	Name    string `xml:"name"`
   311  	Version string `xml:"version"`
   312  	Release string `xml:"release"`
   313  }
   314  
   315  type importSpec struct {
   316  	Size       int64 `xml:"size"`
   317  	VolumeSize int64 `xml:"volume-size"`
   318  	Parts      parts `xml:"parts"`
   319  }
   320  type parts struct {
   321  	Count int    `xml:"count,attr"`
   322  	Parts []part `xml:"part"`
   323  }
   324  
   325  type part struct {
   326  	Index     int       `xml:"index,attr"`
   327  	ByteRange byteRange `xml:"byte-range"`
   328  	Key       string    `xml:"key"`
   329  	HeadUrl   string    `xml:"head-url"`
   330  	GetUrl    string    `xml:"get-url"`
   331  	DeleteUrl string    `xml:"delete-url"`
   332  }
   333  type byteRange struct {
   334  	Start int64 `xml:"start,attr"`
   335  	End   int64 `xml:"end,attr"`
   336  }