github.com/keybase/client/go@v0.0.0-20241007131713-f10651d043c8/citogo/s3.go (about)

     1  package main
     2  
     3  import (
     4  	"bytes"
     5  	"compress/gzip"
     6  	"crypto/hmac"
     7  	"crypto/rand"
     8  	"crypto/sha256"
     9  	"encoding/base32"
    10  	"encoding/hex"
    11  	"errors"
    12  	"fmt"
    13  	"io"
    14  	"net/http"
    15  	"os"
    16  	"strings"
    17  	"time"
    18  )
    19  
    20  func gzipSource(src io.Reader) ([]byte, error) {
    21  	buf := new(bytes.Buffer)
    22  	gzipIn := gzip.NewWriter(buf)
    23  	_, err := io.Copy(gzipIn, src)
    24  	if err != nil {
    25  		return nil, err
    26  	}
    27  	err = gzipIn.Close()
    28  	if err != nil {
    29  		return nil, err
    30  	}
    31  	return buf.Bytes(), nil
    32  }
    33  
    34  func hashToHex(b []byte) string {
    35  	h := sha256.Sum256(b)
    36  	return hex.EncodeToString(h[:])
    37  }
    38  
    39  func randString() (string, error) {
    40  	c := 20
    41  	b := make([]byte, c)
    42  	_, err := rand.Read(b)
    43  	if err != nil {
    44  		return "", err
    45  	}
    46  	return strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)), nil
    47  }
    48  
    49  func awsStringSign4(key string, date string, region string, service string, toSign string) string {
    50  	mac := func(key, payload []byte) []byte {
    51  		f := hmac.New(sha256.New, key)
    52  		_, _ = f.Write(payload)
    53  		ret := f.Sum(nil)
    54  		return ret
    55  	}
    56  	keySecret := []byte("AWS4" + key)
    57  	keyDate := mac(keySecret, []byte(date))
    58  	keyRegion := mac(keyDate, []byte(region))
    59  	keyService := mac(keyRegion, []byte(service))
    60  	keySigning := mac(keyService, []byte("aws4_request"))
    61  	signedString := mac(keySigning, []byte(toSign))
    62  	return hex.EncodeToString(signedString)
    63  
    64  }
    65  
    66  func s3put(src io.Reader, bucket string, name string) (string, error) {
    67  	buf, err := gzipSource(src)
    68  	if err != nil {
    69  		return "", err
    70  	}
    71  	randSuffix, err := randString()
    72  	if err != nil {
    73  		return "", err
    74  	}
    75  	name += "-" + randSuffix + ".gz"
    76  
    77  	_, err = awsCall("s3", bucket+".s3.amazonaws.com", name, "PUT", buf)
    78  	if err != nil {
    79  		return "", err
    80  	}
    81  	where := fmt.Sprintf("(fetch with: ```curl -s -o - https://%s.s3.amazonaws.com/%s | zcat -d | less```)", bucket, name)
    82  	return where, nil
    83  }
    84  
    85  func lambdaInvoke(functionName string, buf []byte) error {
    86  	_, err := awsCall("lambda", "lambda.us-east-1.amazonaws.com", "2015-03-31/functions/"+functionName+"/invocations", "POST", buf)
    87  	return err
    88  }
    89  
    90  // generic aws call without the dependencies, adopted from this shell script:
    91  //
    92  //	https://superuser.com/questions/279986/uploading-files-to-s3-account-from-linux-command-line
    93  func awsCall(service string, host string, path string, method string, buf []byte) (response []byte, err error) {
    94  	payloadHash := hashToHex(buf)
    95  	now := time.Now().UTC()
    96  	dateLong := now.Format("20060102T150405Z")
    97  	dateShort := now.Format("20060102")
    98  	contentType := "application/gzip"
    99  	storageClass := "STANDARD"
   100  	headerList := "content-type;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class"
   101  	canonicalRequest := strings.Join([]string{
   102  		method,
   103  		"/" + path,
   104  		"",
   105  		"content-type:" + contentType,
   106  		"host:" + host,
   107  		"x-amz-content-sha256:" + payloadHash,
   108  		"x-amz-date:" + dateLong,
   109  		"x-amz-storage-class:" + storageClass,
   110  		"",
   111  		headerList,
   112  		payloadHash,
   113  	}, "\n")
   114  
   115  	canonicalRequestHash := hashToHex([]byte(canonicalRequest))
   116  	authType := "AWS4-HMAC-SHA256"
   117  	region := "us-east-1"
   118  	req2 := strings.Join([]string{dateShort, region, service, "aws4_request"}, "/")
   119  	stringToSign := strings.Join([]string{
   120  		authType,
   121  		dateLong,
   122  		req2,
   123  		canonicalRequestHash,
   124  	}, "\n")
   125  
   126  	keyID := os.Getenv("CITOGO_AWS_ACCESS_KEY_ID")
   127  	key := os.Getenv("CITOGO_AWS_SECRET_ACCESS_KEY")
   128  	if keyID == "" || key == "" {
   129  		return response, errors.New("need CITOGO_AWS_ACCESS_KEY_ID and CITOGO_AWS_SECRET_ACCESS_KEY environment variables")
   130  	}
   131  	sig := awsStringSign4(key, dateShort, region, service, stringToSign)
   132  	authorization := authType + " " + strings.Join([]string{
   133  		"Credential=" + strings.Join([]string{keyID, dateShort, region, service, "aws4_request"}, "/"),
   134  		"SignedHeaders=" + headerList,
   135  		"Signature=" + sig,
   136  	}, ", ")
   137  
   138  	client := &http.Client{}
   139  	url := "https://" + host + "/" + path
   140  	req, err := http.NewRequest(method, url, bytes.NewReader(buf))
   141  	if err != nil {
   142  		return response, err
   143  	}
   144  	req.Header.Set("Content-type", contentType)
   145  	req.Header.Set("Host", host)
   146  	req.Header.Set("X-Amz-Content-SHA256", payloadHash)
   147  	req.Header.Set("X-Amz-Date", dateLong)
   148  	req.Header.Set("X-Amz-Storage-Class", storageClass)
   149  	req.Header.Set("Authorization", authorization)
   150  
   151  	resp, err := client.Do(req)
   152  	if err != nil {
   153  		return response, err
   154  	}
   155  	defer resp.Body.Close()
   156  	body, err := io.ReadAll(resp.Body)
   157  	if err != nil {
   158  		return response, err
   159  	}
   160  	return body, nil
   161  }