github.com/hairyhenderson/gomplate/v3@v3.11.7/internal/tests/integration/datasources_blob_test.go (about)

     1  package integration
     2  
     3  import (
     4  	"bytes"
     5  	"net/http/httptest"
     6  	"os"
     7  	"testing"
     8  
     9  	"github.com/johannesboyne/gofakes3"
    10  	"github.com/johannesboyne/gofakes3/backend/s3mem"
    11  	"github.com/stretchr/testify/require"
    12  )
    13  
    14  func setupDatasourcesBlobTest(t *testing.T) *httptest.Server {
    15  	backend := s3mem.New()
    16  	s3 := gofakes3.New(backend)
    17  
    18  	srv := httptest.NewServer(s3.Server())
    19  	t.Cleanup(srv.Close)
    20  
    21  	err := backend.CreateBucket("mybucket")
    22  	require.NoError(t, err)
    23  	contents := `{"value":"json", "name":"foo"}`
    24  	_, err = backend.PutObject("mybucket", "foo.json", map[string]string{"Content-Type": "application/json"}, bytes.NewBufferString(contents), int64(len(contents)))
    25  	require.NoError(t, err)
    26  
    27  	contents = `{"value":"json", "name":"bar"}`
    28  	_, err = backend.PutObject("mybucket", "bar.json", map[string]string{"Content-Type": "application/json"}, bytes.NewBufferString(contents), int64(len(contents)))
    29  	require.NoError(t, err)
    30  
    31  	contents = `hello world`
    32  	_, err = backend.PutObject("mybucket", "a/b/c/hello.txt", map[string]string{"Content-Type": "text/plain"}, bytes.NewBufferString(contents), int64(len(contents)))
    33  	require.NoError(t, err)
    34  
    35  	contents = `goodbye world`
    36  	_, err = backend.PutObject("mybucket", "a/b/c/goodbye.txt", map[string]string{"Content-Type": "text/plain"}, bytes.NewBufferString(contents), int64(len(contents)))
    37  	require.NoError(t, err)
    38  
    39  	contents = "a: foo\nb: bar\nc:\n  cc: yay for yaml\n"
    40  	_, err = backend.PutObject("mybucket", "a/b/c/d", map[string]string{"Content-Type": "application/yaml"}, bytes.NewBufferString(contents), int64(len(contents)))
    41  	require.NoError(t, err)
    42  
    43  	return srv
    44  }
    45  
    46  func TestDatasources_Blob_S3Datasource(t *testing.T) {
    47  	o, e, err := cmd(t,
    48  		"-c", "data=s3://ryft-public-sample-data/integration_test_dataset.json?region=us-east-1&type=application/array+json",
    49  		"-i", "{{ $d := index .data 0 }}{{ $d.firstName }} {{ $d.lastName }}").
    50  		withEnv("AWS_ANON", "true").
    51  		withEnv("AWS_TIMEOUT", "5000").
    52  		run()
    53  	assertSuccess(t, o, e, err, "Petra Mcintyre")
    54  
    55  	srv := setupDatasourcesBlobTest(t)
    56  
    57  	o, e, err = cmd(t,
    58  		"-c", "data=s3://mybucket/foo.json?"+
    59  			"region=us-east-1&"+
    60  			"disableSSL=true&"+
    61  			"endpoint="+srv.Listener.Addr().String()+"&"+
    62  			"s3ForcePathStyle=true",
    63  		"-i", "{{ .data.value }}").
    64  		withEnv("AWS_ACCESS_KEY_ID", "YOUR-ACCESSKEYID").
    65  		withEnv("AWS_SECRET_ACCESS_KEY", "YOUR-SECRETACCESSKEY").
    66  		run()
    67  	assertSuccess(t, o, e, err, "json")
    68  
    69  	o, e, err = cmd(t,
    70  		"-c", "data=s3://mybucket/foo.json?"+
    71  			"region=us-east-1&"+
    72  			"disableSSL=true&"+
    73  			"s3ForcePathStyle=true",
    74  		"-i", "{{ .data.value }}").
    75  		withEnv("AWS_ANON", "true").
    76  		withEnv("AWS_S3_ENDPOINT", srv.Listener.Addr().String()).
    77  		run()
    78  	assertSuccess(t, o, e, err, "json")
    79  }
    80  
    81  func TestDatasources_Blob_S3Directory(t *testing.T) {
    82  	o, e, err := cmd(t, "-c", "data=s3://ryft-public-sample-data/?region=us-east-1",
    83  		"-i", "{{ index .data 0 }}").
    84  		withEnv("AWS_ANON", "true").
    85  		withEnv("AWS_TIMEOUT", "15000").
    86  		run()
    87  	assertSuccess(t, o, e, err, "AWS-x86-AMI-queries.json")
    88  
    89  	srv := setupDatasourcesBlobTest(t)
    90  
    91  	o, e, err = cmd(t, "-c", "data=s3://mybucket/a/b/c/?"+
    92  		"region=us-east-1&"+
    93  		"disableSSL=true&"+
    94  		"endpoint="+srv.Listener.Addr().String()+"&"+
    95  		"s3ForcePathStyle=true",
    96  		"-i", "{{ .data }}").
    97  		withEnv("AWS_ACCESS_KEY_ID", "YOUR-ACCESSKEYID").
    98  		withEnv("AWS_SECRET_ACCESS_KEY", "YOUR-SECRETACCESSKEY").
    99  		run()
   100  	assertSuccess(t, o, e, err, "[d goodbye.txt hello.txt]")
   101  }
   102  
   103  func TestDatasources_Blob_S3MIMETypes(t *testing.T) {
   104  	srv := setupDatasourcesBlobTest(t)
   105  	o, e, err := cmd(t, "-c", "data=s3://mybucket/a/b/c/d?"+
   106  		"region=us-east-1&"+
   107  		"disableSSL=true&"+
   108  		"endpoint="+srv.Listener.Addr().String()+"&"+
   109  		"s3ForcePathStyle=true",
   110  		"-i", "{{ .data.c.cc }}").
   111  		withEnv("AWS_ANON", "true").run()
   112  	assertSuccess(t, o, e, err, "yay for yaml")
   113  }
   114  
   115  func TestDatasources_Blob_GCSDatasource(t *testing.T) {
   116  	// this only works if we're authed with GCS
   117  	if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
   118  		t.Skip("Not configured to authenticate with Google Cloud - skipping")
   119  		return
   120  	}
   121  
   122  	o, e, err := cmd(t, "-c", "data=gs://gcp-public-data-landsat/LT08/01/015/013/LT08_L1GT_015013_20130315_20170310_01_T2/LT08_L1GT_015013_20130315_20170310_01_T2_MTL.txt?type=text/plain",
   123  		"-i", "{{ len .data }}").
   124  		withEnv("GOOGLE_APPLICATION_CREDENTIALS",
   125  			os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")).run()
   126  	assertSuccess(t, o, e, err, "3672")
   127  }
   128  
   129  func TestDatasources_Blob_GCSDirectory(t *testing.T) {
   130  	// this only works if we're likely to be authed with GCS
   131  	if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
   132  		t.Skip("Not configured to authenticate with Google Cloud - skipping")
   133  		return
   134  	}
   135  
   136  	o, e, err := cmd(t, "-c", "data=gs://gcp-public-data-landsat/",
   137  		"-i", "{{ coll.Has .data `index.csv.gz` }}").
   138  		withEnv("GOOGLE_APPLICATION_CREDENTIALS",
   139  			os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")).run()
   140  	assertSuccess(t, o, e, err, "true")
   141  }