github.com/hairyhenderson/gomplate/v3@v3.11.7/internal/tests/integration/datasources_blob_test.go (about) 1 package integration 2 3 import ( 4 "bytes" 5 "net/http/httptest" 6 "os" 7 "testing" 8 9 "github.com/johannesboyne/gofakes3" 10 "github.com/johannesboyne/gofakes3/backend/s3mem" 11 "github.com/stretchr/testify/require" 12 ) 13 14 func setupDatasourcesBlobTest(t *testing.T) *httptest.Server { 15 backend := s3mem.New() 16 s3 := gofakes3.New(backend) 17 18 srv := httptest.NewServer(s3.Server()) 19 t.Cleanup(srv.Close) 20 21 err := backend.CreateBucket("mybucket") 22 require.NoError(t, err) 23 contents := `{"value":"json", "name":"foo"}` 24 _, err = backend.PutObject("mybucket", "foo.json", map[string]string{"Content-Type": "application/json"}, bytes.NewBufferString(contents), int64(len(contents))) 25 require.NoError(t, err) 26 27 contents = `{"value":"json", "name":"bar"}` 28 _, err = backend.PutObject("mybucket", "bar.json", map[string]string{"Content-Type": "application/json"}, bytes.NewBufferString(contents), int64(len(contents))) 29 require.NoError(t, err) 30 31 contents = `hello world` 32 _, err = backend.PutObject("mybucket", "a/b/c/hello.txt", map[string]string{"Content-Type": "text/plain"}, bytes.NewBufferString(contents), int64(len(contents))) 33 require.NoError(t, err) 34 35 contents = `goodbye world` 36 _, err = backend.PutObject("mybucket", "a/b/c/goodbye.txt", map[string]string{"Content-Type": "text/plain"}, bytes.NewBufferString(contents), int64(len(contents))) 37 require.NoError(t, err) 38 39 contents = "a: foo\nb: bar\nc:\n cc: yay for yaml\n" 40 _, err = backend.PutObject("mybucket", "a/b/c/d", map[string]string{"Content-Type": "application/yaml"}, bytes.NewBufferString(contents), int64(len(contents))) 41 require.NoError(t, err) 42 43 return srv 44 } 45 46 func TestDatasources_Blob_S3Datasource(t *testing.T) { 47 o, e, err := cmd(t, 48 "-c", "data=s3://ryft-public-sample-data/integration_test_dataset.json?region=us-east-1&type=application/array+json", 49 "-i", "{{ $d := index .data 0 }}{{ $d.firstName }} {{ $d.lastName }}"). 50 withEnv("AWS_ANON", "true"). 51 withEnv("AWS_TIMEOUT", "5000"). 52 run() 53 assertSuccess(t, o, e, err, "Petra Mcintyre") 54 55 srv := setupDatasourcesBlobTest(t) 56 57 o, e, err = cmd(t, 58 "-c", "data=s3://mybucket/foo.json?"+ 59 "region=us-east-1&"+ 60 "disableSSL=true&"+ 61 "endpoint="+srv.Listener.Addr().String()+"&"+ 62 "s3ForcePathStyle=true", 63 "-i", "{{ .data.value }}"). 64 withEnv("AWS_ACCESS_KEY_ID", "YOUR-ACCESSKEYID"). 65 withEnv("AWS_SECRET_ACCESS_KEY", "YOUR-SECRETACCESSKEY"). 66 run() 67 assertSuccess(t, o, e, err, "json") 68 69 o, e, err = cmd(t, 70 "-c", "data=s3://mybucket/foo.json?"+ 71 "region=us-east-1&"+ 72 "disableSSL=true&"+ 73 "s3ForcePathStyle=true", 74 "-i", "{{ .data.value }}"). 75 withEnv("AWS_ANON", "true"). 76 withEnv("AWS_S3_ENDPOINT", srv.Listener.Addr().String()). 77 run() 78 assertSuccess(t, o, e, err, "json") 79 } 80 81 func TestDatasources_Blob_S3Directory(t *testing.T) { 82 o, e, err := cmd(t, "-c", "data=s3://ryft-public-sample-data/?region=us-east-1", 83 "-i", "{{ index .data 0 }}"). 84 withEnv("AWS_ANON", "true"). 85 withEnv("AWS_TIMEOUT", "15000"). 86 run() 87 assertSuccess(t, o, e, err, "AWS-x86-AMI-queries.json") 88 89 srv := setupDatasourcesBlobTest(t) 90 91 o, e, err = cmd(t, "-c", "data=s3://mybucket/a/b/c/?"+ 92 "region=us-east-1&"+ 93 "disableSSL=true&"+ 94 "endpoint="+srv.Listener.Addr().String()+"&"+ 95 "s3ForcePathStyle=true", 96 "-i", "{{ .data }}"). 97 withEnv("AWS_ACCESS_KEY_ID", "YOUR-ACCESSKEYID"). 98 withEnv("AWS_SECRET_ACCESS_KEY", "YOUR-SECRETACCESSKEY"). 99 run() 100 assertSuccess(t, o, e, err, "[d goodbye.txt hello.txt]") 101 } 102 103 func TestDatasources_Blob_S3MIMETypes(t *testing.T) { 104 srv := setupDatasourcesBlobTest(t) 105 o, e, err := cmd(t, "-c", "data=s3://mybucket/a/b/c/d?"+ 106 "region=us-east-1&"+ 107 "disableSSL=true&"+ 108 "endpoint="+srv.Listener.Addr().String()+"&"+ 109 "s3ForcePathStyle=true", 110 "-i", "{{ .data.c.cc }}"). 111 withEnv("AWS_ANON", "true").run() 112 assertSuccess(t, o, e, err, "yay for yaml") 113 } 114 115 func TestDatasources_Blob_GCSDatasource(t *testing.T) { 116 // this only works if we're authed with GCS 117 if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { 118 t.Skip("Not configured to authenticate with Google Cloud - skipping") 119 return 120 } 121 122 o, e, err := cmd(t, "-c", "data=gs://gcp-public-data-landsat/LT08/01/015/013/LT08_L1GT_015013_20130315_20170310_01_T2/LT08_L1GT_015013_20130315_20170310_01_T2_MTL.txt?type=text/plain", 123 "-i", "{{ len .data }}"). 124 withEnv("GOOGLE_APPLICATION_CREDENTIALS", 125 os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")).run() 126 assertSuccess(t, o, e, err, "3672") 127 } 128 129 func TestDatasources_Blob_GCSDirectory(t *testing.T) { 130 // this only works if we're likely to be authed with GCS 131 if os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" { 132 t.Skip("Not configured to authenticate with Google Cloud - skipping") 133 return 134 } 135 136 o, e, err := cmd(t, "-c", "data=gs://gcp-public-data-landsat/", 137 "-i", "{{ coll.Has .data `index.csv.gz` }}"). 138 withEnv("GOOGLE_APPLICATION_CREDENTIALS", 139 os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")).run() 140 assertSuccess(t, o, e, err, "true") 141 }