github.com/cockroachdb/cockroach@v20.2.0-alpha.1+incompatible/pkg/ccl/backupccl/backup_cloud_test.go (about)

     1  // Copyright 2017 The Cockroach Authors.
     2  //
     3  // Licensed as a CockroachDB Enterprise file under the Cockroach Community
     4  // License (the "License"); you may not use this file except in compliance with
     5  // the License. You may obtain a copy of the License at
     6  //
     7  //     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
     8  
     9  package backupccl_test
    10  
    11  import (
    12  	"fmt"
    13  	"net/url"
    14  	"os"
    15  	"testing"
    16  
    17  	"github.com/aws/aws-sdk-go/aws/credentials"
    18  	"github.com/cockroachdb/cockroach/pkg/sql/catalog/lease"
    19  	"github.com/cockroachdb/cockroach/pkg/storage/cloud"
    20  	"github.com/cockroachdb/cockroach/pkg/testutils/testcluster"
    21  	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
    22  	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
    23  )
    24  
    25  func initNone(_ *testcluster.TestCluster) {}
    26  
    27  // The tests in this file talk to remote APIs which require credentials.
    28  // To run these tests, you need to supply credentials via env vars (the tests
    29  // skip themselves if they are not set). Obtain these credentials from the
    30  // admin consoles of the various cloud providers.
    31  // customenv.mak (gitignored) may be a useful place to record these.
    32  // Cockroach Labs Employees: symlink customenv.mk to copy in `production`.
    33  
    34  // TestBackupRestoreS3 hits the real S3 and so could occasionally be flaky. It's
    35  // only run if the AWS_S3_BUCKET environment var is set.
    36  func TestCloudBackupRestoreS3(t *testing.T) {
    37  	defer leaktest.AfterTest(t)()
    38  	creds, err := credentials.NewEnvCredentials().Get()
    39  	if err != nil {
    40  		t.Skipf("No AWS env keys (%v)", err)
    41  	}
    42  	bucket := os.Getenv("AWS_S3_BUCKET")
    43  	if bucket == "" {
    44  		t.Skip("AWS_S3_BUCKET env var must be set")
    45  	}
    46  
    47  	// TODO(dan): Actually invalidate the descriptor cache and delete this line.
    48  	defer lease.TestingDisableTableLeases()()
    49  	const numAccounts = 1000
    50  
    51  	ctx, tc, _, _, cleanupFn := backupRestoreTestSetup(t, 1, numAccounts, initNone)
    52  	defer cleanupFn()
    53  	prefix := fmt.Sprintf("TestBackupRestoreS3-%d", timeutil.Now().UnixNano())
    54  	uri := url.URL{Scheme: "s3", Host: bucket, Path: prefix}
    55  	values := uri.Query()
    56  	values.Add(cloud.S3AccessKeyParam, creds.AccessKeyID)
    57  	values.Add(cloud.S3SecretParam, creds.SecretAccessKey)
    58  	uri.RawQuery = values.Encode()
    59  
    60  	backupAndRestore(ctx, t, tc, []string{uri.String()}, []string{uri.String()}, numAccounts)
    61  }
    62  
    63  // TestBackupRestoreGoogleCloudStorage hits the real GCS and so could
    64  // occasionally be flaky. It's only run if the GS_BUCKET environment var is set.
    65  func TestCloudBackupRestoreGoogleCloudStorage(t *testing.T) {
    66  	defer leaktest.AfterTest(t)()
    67  	bucket := os.Getenv("GS_BUCKET")
    68  	if bucket == "" {
    69  		t.Skip("GS_BUCKET env var must be set")
    70  	}
    71  
    72  	// TODO(dan): Actually invalidate the descriptor cache and delete this line.
    73  	defer lease.TestingDisableTableLeases()()
    74  	const numAccounts = 1000
    75  
    76  	ctx, tc, _, _, cleanupFn := backupRestoreTestSetup(t, 1, numAccounts, initNone)
    77  	defer cleanupFn()
    78  	prefix := fmt.Sprintf("TestBackupRestoreGoogleCloudStorage-%d", timeutil.Now().UnixNano())
    79  	uri := url.URL{Scheme: "gs", Host: bucket, Path: prefix}
    80  	backupAndRestore(ctx, t, tc, []string{uri.String()}, []string{uri.String()}, numAccounts)
    81  }
    82  
    83  // TestBackupRestoreAzure hits the real Azure Blob Storage and so could
    84  // occasionally be flaky. It's only run if the AZURE_ACCOUNT_NAME and
    85  // AZURE_ACCOUNT_KEY environment vars are set.
    86  func TestCloudBackupRestoreAzure(t *testing.T) {
    87  	defer leaktest.AfterTest(t)()
    88  	accountName := os.Getenv("AZURE_ACCOUNT_NAME")
    89  	accountKey := os.Getenv("AZURE_ACCOUNT_KEY")
    90  	if accountName == "" || accountKey == "" {
    91  		t.Skip("AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY env vars must be set")
    92  	}
    93  	bucket := os.Getenv("AZURE_CONTAINER")
    94  	if bucket == "" {
    95  		t.Skip("AZURE_CONTAINER env var must be set")
    96  	}
    97  
    98  	// TODO(dan): Actually invalidate the descriptor cache and delete this line.
    99  	defer lease.TestingDisableTableLeases()()
   100  	const numAccounts = 1000
   101  
   102  	ctx, tc, _, _, cleanupFn := backupRestoreTestSetup(t, 1, numAccounts, initNone)
   103  	defer cleanupFn()
   104  	prefix := fmt.Sprintf("TestBackupRestoreAzure-%d", timeutil.Now().UnixNano())
   105  	uri := url.URL{Scheme: "azure", Host: bucket, Path: prefix}
   106  	values := uri.Query()
   107  	values.Add(cloud.AzureAccountNameParam, accountName)
   108  	values.Add(cloud.AzureAccountKeyParam, accountKey)
   109  	uri.RawQuery = values.Encode()
   110  
   111  	backupAndRestore(ctx, t, tc, []string{uri.String()}, []string{uri.String()}, numAccounts)
   112  }