github.com/olli-ai/jx/v2@v2.0.400-0.20210921045218-14731b4dd448/pkg/cloud/buckets/buckets.go (about)

     1  package buckets
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"io/ioutil"
     8  	"net/http"
     9  	"net/url"
    10  	"strings"
    11  	"time"
    12  
    13  	jenkinsv1 "github.com/jenkins-x/jx-api/pkg/apis/jenkins.io/v1"
    14  	"github.com/olli-ai/jx/v2/pkg/cloud"
    15  	"github.com/olli-ai/jx/v2/pkg/util"
    16  	"github.com/pkg/errors"
    17  	"gocloud.dev/blob"
    18  )
    19  
    20  // CreateBucketURL creates a go-cloud URL to a bucket
    21  func CreateBucketURL(name string, kind string, settings *jenkinsv1.TeamSettings) (string, error) {
    22  	if kind == "" {
    23  		provider := settings.KubeProvider
    24  		if provider == "" {
    25  			return "", fmt.Errorf("No bucket kind provided nor is a kubernetes provider configured for this team so it could not be defaulted")
    26  		}
    27  		kind = KubeProviderToBucketScheme(provider)
    28  		if kind == "" {
    29  			return "", fmt.Errorf("No bucket kind is associated with kubernetes provider %s", provider)
    30  		}
    31  	}
    32  	return kind + "://" + name, nil
    33  }
    34  
    35  // KubeProviderToBucketScheme returns the bucket scheme for the cloud provider
    36  func KubeProviderToBucketScheme(provider string) string {
    37  	switch provider {
    38  	case cloud.AKS:
    39  		return "azblob"
    40  	case cloud.AWS, cloud.EKS:
    41  		return "s3"
    42  	case cloud.GKE:
    43  		return "gs"
    44  	default:
    45  		return ""
    46  	}
    47  }
    48  
    49  // ReadURL reads the given URL from either a http/https endpoint or a bucket URL path.
    50  // if specified the httpFn is a function which can append the user/password or token and/or add a header with the token if using a git provider
    51  func ReadURL(urlText string, timeout time.Duration, httpFn func(urlString string) (string, func(*http.Request), error)) (io.ReadCloser, error) {
    52  	u, err := url.Parse(urlText)
    53  	if err != nil {
    54  		return nil, errors.Wrapf(err, "failed to parse URL %s", urlText)
    55  	}
    56  	var headerFunc func(*http.Request)
    57  	switch u.Scheme {
    58  	case "http", "https":
    59  		if httpFn != nil {
    60  			urlText, headerFunc, err = httpFn(urlText)
    61  			if err != nil {
    62  				return nil, err
    63  			}
    64  		}
    65  		return ReadHTTPURL(urlText, headerFunc, timeout)
    66  	default:
    67  		return ReadBucketURL(u, timeout)
    68  	}
    69  }
    70  
    71  // ReadHTTPURL reads the HTTP based URL, modifying the headers as needed, and returns the data or returning an error if a 2xx status is not returned
    72  func ReadHTTPURL(u string, headerFunc func(*http.Request), timeout time.Duration) (io.ReadCloser, error) {
    73  	httpClient := util.GetClientWithTimeout(timeout)
    74  
    75  	req, err := http.NewRequest("GET", u, nil)
    76  	if err != nil {
    77  		return nil, err
    78  	}
    79  	headerFunc(req)
    80  	resp, err := httpClient.Do(req)
    81  	if err != nil {
    82  		return nil, errors.Wrapf(err, "failed to invoke GET on %s", u)
    83  	}
    84  	stream := resp.Body
    85  
    86  	if resp.StatusCode >= 400 {
    87  		_ = stream.Close()
    88  		return nil, fmt.Errorf("status %s when performing GET on %s", resp.Status, u)
    89  	}
    90  	return stream, nil
    91  }
    92  
    93  // ReadBucketURL reads the content of a bucket URL of the for 's3://bucketName/foo/bar/whatnot.txt?param=123'
    94  // where any of the query arguments are applied to the underlying Bucket URL and the path is extracted and resolved
    95  // within the bucket
    96  func ReadBucketURL(u *url.URL, timeout time.Duration) (io.ReadCloser, error) {
    97  	bucketURL, key := SplitBucketURL(u)
    98  
    99  	ctx, _ := context.WithTimeout(context.Background(), timeout)
   100  	bucket, err := blob.Open(ctx, bucketURL)
   101  	if err != nil {
   102  		return nil, errors.Wrapf(err, "failed to open bucket %s", bucketURL)
   103  	}
   104  	data, err := bucket.NewReader(ctx, key, nil)
   105  	if err != nil {
   106  		return data, errors.Wrapf(err, "failed to read key %s in bucket %s", key, bucketURL)
   107  	}
   108  	return data, nil
   109  }
   110  
   111  // WriteBucketURL writes the data to a bucket URL of the for 's3://bucketName/foo/bar/whatnot.txt?param=123'
   112  // with the given timeout
   113  func WriteBucketURL(u *url.URL, data io.Reader, timeout time.Duration) error {
   114  	bucketURL, key := SplitBucketURL(u)
   115  	return WriteBucket(bucketURL, key, data, timeout)
   116  }
   117  
   118  // WriteBucket writes the data to a bucket URL and key of the for 's3://bucketName' and key 'foo/bar/whatnot.txt'
   119  // with the given timeout
   120  func WriteBucket(bucketURL string, key string, reader io.Reader, timeout time.Duration) (err error) {
   121  	ctx, _ := context.WithTimeout(context.Background(), timeout)
   122  	bucket, err := blob.Open(ctx, bucketURL)
   123  	if err != nil {
   124  		return errors.Wrapf(err, "failed to open bucket %s", bucketURL)
   125  	}
   126  	data, err := ioutil.ReadAll(reader)
   127  	if err != nil {
   128  		return errors.Wrapf(err, "failed to read data for key %s in bucket %s", key, bucketURL)
   129  	}
   130  	err = bucket.WriteAll(ctx, key, data, nil)
   131  	if err != nil {
   132  		return errors.Wrapf(err, "failed to write key %s in bucket %s", key, bucketURL)
   133  	}
   134  	return nil
   135  }
   136  
   137  // SplitBucketURL splits the full bucket URL into the URL to open the bucket and the file name to refer to
   138  // within the bucket
   139  func SplitBucketURL(u *url.URL) (string, string) {
   140  	u2 := *u
   141  	u2.Path = ""
   142  	return u2.String(), strings.TrimPrefix(u.Path, "/")
   143  }