github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/storage/cloud/s3_storage_test.go (about)

     1  // Copyright 2019 The Cockroach Authors.
     2  //
     3  // Use of this software is governed by the Business Source License
     4  // included in the file licenses/BSL.txt.
     5  //
     6  // As of the Change Date specified in that file, in accordance with
     7  // the Business Source License, use of this software will be governed
     8  // by the Apache License, Version 2.0, included in the file
     9  // licenses/APL.txt.
    10  
    11  package cloud
    12  
    13  import (
    14  	"context"
    15  	"fmt"
    16  	"net/url"
    17  	"os"
    18  	"strings"
    19  	"testing"
    20  
    21  	"github.com/aws/aws-sdk-go/aws/credentials"
    22  	"github.com/cockroachdb/cockroach/pkg/base"
    23  	"github.com/cockroachdb/cockroach/pkg/blobs"
    24  	"github.com/cockroachdb/cockroach/pkg/roachpb"
    25  	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
    26  	"github.com/stretchr/testify/require"
    27  )
    28  
    29  func TestPutS3(t *testing.T) {
    30  	defer leaktest.AfterTest(t)()
    31  
    32  	// If environment credentials are not present, we want to
    33  	// skip all S3 tests, including auth-implicit, even though
    34  	// it is not used in auth-implicit.
    35  	creds, err := credentials.NewEnvCredentials().Get()
    36  	if err != nil {
    37  		t.Skip("No AWS credentials")
    38  	}
    39  	bucket := os.Getenv("AWS_S3_BUCKET")
    40  	if bucket == "" {
    41  		t.Skip("AWS_S3_BUCKET env var must be set")
    42  	}
    43  
    44  	ctx := context.Background()
    45  	t.Run("auth-empty-no-cred", func(t *testing.T) {
    46  		_, err := ExternalStorageFromURI(
    47  			ctx, fmt.Sprintf("s3://%s/%s", bucket, "backup-test-default"),
    48  			base.ExternalIODirConfig{}, testSettings, blobs.TestEmptyBlobClientFactory,
    49  		)
    50  		require.EqualError(t, err, fmt.Sprintf(
    51  			`%s is set to '%s', but %s is not set`,
    52  			AuthParam,
    53  			authParamSpecified,
    54  			S3AccessKeyParam,
    55  		))
    56  	})
    57  	t.Run("auth-implicit", func(t *testing.T) {
    58  		// You can create an IAM that can access S3
    59  		// in the AWS console, then set it up locally.
    60  		// https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-role.html
    61  		// We only run this test if default role exists.
    62  		credentialsProvider := credentials.SharedCredentialsProvider{}
    63  		_, err := credentialsProvider.Retrieve()
    64  		if err != nil {
    65  			t.Skip(err)
    66  		}
    67  
    68  		testExportStore(
    69  			t,
    70  			fmt.Sprintf(
    71  				"s3://%s/%s?%s=%s",
    72  				bucket, "backup-test-default",
    73  				AuthParam, authParamImplicit,
    74  			),
    75  			false,
    76  		)
    77  	})
    78  
    79  	t.Run("auth-specified", func(t *testing.T) {
    80  		testExportStore(t,
    81  			fmt.Sprintf(
    82  				"s3://%s/%s?%s=%s&%s=%s",
    83  				bucket, "backup-test",
    84  				S3AccessKeyParam, url.QueryEscape(creds.AccessKeyID),
    85  				S3SecretParam, url.QueryEscape(creds.SecretAccessKey),
    86  			),
    87  			false,
    88  		)
    89  		testListFiles(t,
    90  			fmt.Sprintf(
    91  				"s3://%s/%s?%s=%s&%s=%s",
    92  				bucket, "listing-test",
    93  				S3AccessKeyParam, url.QueryEscape(creds.AccessKeyID),
    94  				S3SecretParam, url.QueryEscape(creds.SecretAccessKey),
    95  			),
    96  		)
    97  	})
    98  }
    99  
   100  func TestPutS3Endpoint(t *testing.T) {
   101  	defer leaktest.AfterTest(t)()
   102  
   103  	q := make(url.Values)
   104  	expect := map[string]string{
   105  		"AWS_S3_ENDPOINT":        S3EndpointParam,
   106  		"AWS_S3_ENDPOINT_KEY":    S3AccessKeyParam,
   107  		"AWS_S3_ENDPOINT_REGION": S3RegionParam,
   108  		"AWS_S3_ENDPOINT_SECRET": S3SecretParam,
   109  	}
   110  	for env, param := range expect {
   111  		v := os.Getenv(env)
   112  		if v == "" {
   113  			t.Skipf("%s env var must be set", env)
   114  		}
   115  		q.Add(param, v)
   116  	}
   117  
   118  	bucket := os.Getenv("AWS_S3_ENDPOINT_BUCKET")
   119  	if bucket == "" {
   120  		t.Skip("AWS_S3_ENDPOINT_BUCKET env var must be set")
   121  	}
   122  
   123  	u := url.URL{
   124  		Scheme:   "s3",
   125  		Host:     bucket,
   126  		Path:     "backup-test",
   127  		RawQuery: q.Encode(),
   128  	}
   129  
   130  	testExportStore(t, u.String(), false)
   131  }
   132  
   133  func TestS3DisallowCustomEndpoints(t *testing.T) {
   134  	s3, err := makeS3Storage(context.Background(),
   135  		base.ExternalIODirConfig{DisableHTTP: true},
   136  		&roachpb.ExternalStorage_S3{Endpoint: "http://do.not.go.there/"}, nil,
   137  	)
   138  	require.Nil(t, s3)
   139  	require.Error(t, err)
   140  }
   141  
   142  func TestS3DisallowImplicitCredentials(t *testing.T) {
   143  	defer leaktest.AfterTest(t)()
   144  	s3, err := makeS3Storage(context.Background(),
   145  		base.ExternalIODirConfig{DisableImplicitCredentials: true},
   146  		&roachpb.ExternalStorage_S3{
   147  			Endpoint: "http://do-not-go-there",
   148  			Auth:     authParamImplicit,
   149  		}, testSettings,
   150  	)
   151  	require.Nil(t, s3)
   152  	require.Error(t, err)
   153  	require.True(t, strings.Contains(err.Error(), "implicit"))
   154  }