github.com/terramate-io/tf@v0.0.0-20230830114523-fce866b4dfcd/backend/remote-state/s3/client_test.go (about)

     1  // Copyright (c) HashiCorp, Inc.
     2  // SPDX-License-Identifier: MPL-2.0
     3  
     4  package s3
     5  
     6  import (
     7  	"bytes"
     8  	"crypto/md5"
     9  	"fmt"
    10  	"strings"
    11  	"testing"
    12  	"time"
    13  
    14  	"github.com/terramate-io/tf/backend"
    15  	"github.com/terramate-io/tf/states/remote"
    16  	"github.com/terramate-io/tf/states/statefile"
    17  	"github.com/terramate-io/tf/states/statemgr"
    18  )
    19  
    20  func TestRemoteClient_impl(t *testing.T) {
    21  	var _ remote.Client = new(RemoteClient)
    22  	var _ remote.ClientLocker = new(RemoteClient)
    23  }
    24  
    25  func TestRemoteClient(t *testing.T) {
    26  	testACC(t)
    27  	bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix())
    28  	keyName := "testState"
    29  
    30  	b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
    31  		"bucket":  bucketName,
    32  		"key":     keyName,
    33  		"encrypt": true,
    34  	})).(*Backend)
    35  
    36  	createS3Bucket(t, b.s3Client, bucketName)
    37  	defer deleteS3Bucket(t, b.s3Client, bucketName)
    38  
    39  	state, err := b.StateMgr(backend.DefaultStateName)
    40  	if err != nil {
    41  		t.Fatal(err)
    42  	}
    43  
    44  	remote.TestClient(t, state.(*remote.State).Client)
    45  }
    46  
    47  func TestRemoteClientLocks(t *testing.T) {
    48  	testACC(t)
    49  	bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix())
    50  	keyName := "testState"
    51  
    52  	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
    53  		"bucket":         bucketName,
    54  		"key":            keyName,
    55  		"encrypt":        true,
    56  		"dynamodb_table": bucketName,
    57  	})).(*Backend)
    58  
    59  	b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
    60  		"bucket":         bucketName,
    61  		"key":            keyName,
    62  		"encrypt":        true,
    63  		"dynamodb_table": bucketName,
    64  	})).(*Backend)
    65  
    66  	createS3Bucket(t, b1.s3Client, bucketName)
    67  	defer deleteS3Bucket(t, b1.s3Client, bucketName)
    68  	createDynamoDBTable(t, b1.dynClient, bucketName)
    69  	defer deleteDynamoDBTable(t, b1.dynClient, bucketName)
    70  
    71  	s1, err := b1.StateMgr(backend.DefaultStateName)
    72  	if err != nil {
    73  		t.Fatal(err)
    74  	}
    75  
    76  	s2, err := b2.StateMgr(backend.DefaultStateName)
    77  	if err != nil {
    78  		t.Fatal(err)
    79  	}
    80  
    81  	remote.TestRemoteLocks(t, s1.(*remote.State).Client, s2.(*remote.State).Client)
    82  }
    83  
    84  // verify that we can unlock a state with an existing lock
    85  func TestForceUnlock(t *testing.T) {
    86  	testACC(t)
    87  	bucketName := fmt.Sprintf("terraform-remote-s3-test-force-%x", time.Now().Unix())
    88  	keyName := "testState"
    89  
    90  	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
    91  		"bucket":         bucketName,
    92  		"key":            keyName,
    93  		"encrypt":        true,
    94  		"dynamodb_table": bucketName,
    95  	})).(*Backend)
    96  
    97  	b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
    98  		"bucket":         bucketName,
    99  		"key":            keyName,
   100  		"encrypt":        true,
   101  		"dynamodb_table": bucketName,
   102  	})).(*Backend)
   103  
   104  	createS3Bucket(t, b1.s3Client, bucketName)
   105  	defer deleteS3Bucket(t, b1.s3Client, bucketName)
   106  	createDynamoDBTable(t, b1.dynClient, bucketName)
   107  	defer deleteDynamoDBTable(t, b1.dynClient, bucketName)
   108  
   109  	// first test with default
   110  	s1, err := b1.StateMgr(backend.DefaultStateName)
   111  	if err != nil {
   112  		t.Fatal(err)
   113  	}
   114  
   115  	info := statemgr.NewLockInfo()
   116  	info.Operation = "test"
   117  	info.Who = "clientA"
   118  
   119  	lockID, err := s1.Lock(info)
   120  	if err != nil {
   121  		t.Fatal("unable to get initial lock:", err)
   122  	}
   123  
   124  	// s1 is now locked, get the same state through s2 and unlock it
   125  	s2, err := b2.StateMgr(backend.DefaultStateName)
   126  	if err != nil {
   127  		t.Fatal("failed to get default state to force unlock:", err)
   128  	}
   129  
   130  	if err := s2.Unlock(lockID); err != nil {
   131  		t.Fatal("failed to force-unlock default state")
   132  	}
   133  
   134  	// now try the same thing with a named state
   135  	// first test with default
   136  	s1, err = b1.StateMgr("test")
   137  	if err != nil {
   138  		t.Fatal(err)
   139  	}
   140  
   141  	info = statemgr.NewLockInfo()
   142  	info.Operation = "test"
   143  	info.Who = "clientA"
   144  
   145  	lockID, err = s1.Lock(info)
   146  	if err != nil {
   147  		t.Fatal("unable to get initial lock:", err)
   148  	}
   149  
   150  	// s1 is now locked, get the same state through s2 and unlock it
   151  	s2, err = b2.StateMgr("test")
   152  	if err != nil {
   153  		t.Fatal("failed to get named state to force unlock:", err)
   154  	}
   155  
   156  	if err = s2.Unlock(lockID); err != nil {
   157  		t.Fatal("failed to force-unlock named state")
   158  	}
   159  }
   160  
   161  func TestRemoteClient_clientMD5(t *testing.T) {
   162  	testACC(t)
   163  
   164  	bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix())
   165  	keyName := "testState"
   166  
   167  	b := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
   168  		"bucket":         bucketName,
   169  		"key":            keyName,
   170  		"dynamodb_table": bucketName,
   171  	})).(*Backend)
   172  
   173  	createS3Bucket(t, b.s3Client, bucketName)
   174  	defer deleteS3Bucket(t, b.s3Client, bucketName)
   175  	createDynamoDBTable(t, b.dynClient, bucketName)
   176  	defer deleteDynamoDBTable(t, b.dynClient, bucketName)
   177  
   178  	s, err := b.StateMgr(backend.DefaultStateName)
   179  	if err != nil {
   180  		t.Fatal(err)
   181  	}
   182  	client := s.(*remote.State).Client.(*RemoteClient)
   183  
   184  	sum := md5.Sum([]byte("test"))
   185  
   186  	if err := client.putMD5(sum[:]); err != nil {
   187  		t.Fatal(err)
   188  	}
   189  
   190  	getSum, err := client.getMD5()
   191  	if err != nil {
   192  		t.Fatal(err)
   193  	}
   194  
   195  	if !bytes.Equal(getSum, sum[:]) {
   196  		t.Fatalf("getMD5 returned the wrong checksum: expected %x, got %x", sum[:], getSum)
   197  	}
   198  
   199  	if err := client.deleteMD5(); err != nil {
   200  		t.Fatal(err)
   201  	}
   202  
   203  	if getSum, err := client.getMD5(); err == nil {
   204  		t.Fatalf("expected getMD5 error, got none. checksum: %x", getSum)
   205  	}
   206  }
   207  
   208  // verify that a client won't return a state with an incorrect checksum.
   209  func TestRemoteClient_stateChecksum(t *testing.T) {
   210  	testACC(t)
   211  
   212  	bucketName := fmt.Sprintf("terraform-remote-s3-test-%x", time.Now().Unix())
   213  	keyName := "testState"
   214  
   215  	b1 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
   216  		"bucket":         bucketName,
   217  		"key":            keyName,
   218  		"dynamodb_table": bucketName,
   219  	})).(*Backend)
   220  
   221  	createS3Bucket(t, b1.s3Client, bucketName)
   222  	defer deleteS3Bucket(t, b1.s3Client, bucketName)
   223  	createDynamoDBTable(t, b1.dynClient, bucketName)
   224  	defer deleteDynamoDBTable(t, b1.dynClient, bucketName)
   225  
   226  	s1, err := b1.StateMgr(backend.DefaultStateName)
   227  	if err != nil {
   228  		t.Fatal(err)
   229  	}
   230  	client1 := s1.(*remote.State).Client
   231  
   232  	// create an old and new state version to persist
   233  	s := statemgr.TestFullInitialState()
   234  	sf := &statefile.File{State: s}
   235  	var oldState bytes.Buffer
   236  	if err := statefile.Write(sf, &oldState); err != nil {
   237  		t.Fatal(err)
   238  	}
   239  	sf.Serial++
   240  	var newState bytes.Buffer
   241  	if err := statefile.Write(sf, &newState); err != nil {
   242  		t.Fatal(err)
   243  	}
   244  
   245  	// Use b2 without a dynamodb_table to bypass the lock table to write the state directly.
   246  	// client2 will write the "incorrect" state, simulating s3 eventually consistency delays
   247  	b2 := backend.TestBackendConfig(t, New(), backend.TestWrapConfig(map[string]interface{}{
   248  		"bucket": bucketName,
   249  		"key":    keyName,
   250  	})).(*Backend)
   251  	s2, err := b2.StateMgr(backend.DefaultStateName)
   252  	if err != nil {
   253  		t.Fatal(err)
   254  	}
   255  	client2 := s2.(*remote.State).Client
   256  
   257  	// write the new state through client2 so that there is no checksum yet
   258  	if err := client2.Put(newState.Bytes()); err != nil {
   259  		t.Fatal(err)
   260  	}
   261  
   262  	// verify that we can pull a state without a checksum
   263  	if _, err := client1.Get(); err != nil {
   264  		t.Fatal(err)
   265  	}
   266  
   267  	// write the new state back with its checksum
   268  	if err := client1.Put(newState.Bytes()); err != nil {
   269  		t.Fatal(err)
   270  	}
   271  
   272  	// put an empty state in place to check for panics during get
   273  	if err := client2.Put([]byte{}); err != nil {
   274  		t.Fatal(err)
   275  	}
   276  
   277  	// remove the timeouts so we can fail immediately
   278  	origTimeout := consistencyRetryTimeout
   279  	origInterval := consistencyRetryPollInterval
   280  	defer func() {
   281  		consistencyRetryTimeout = origTimeout
   282  		consistencyRetryPollInterval = origInterval
   283  	}()
   284  	consistencyRetryTimeout = 0
   285  	consistencyRetryPollInterval = 0
   286  
   287  	// fetching an empty state through client1 should now error out due to a
   288  	// mismatched checksum.
   289  	if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) {
   290  		t.Fatalf("expected state checksum error: got %s", err)
   291  	}
   292  
   293  	// put the old state in place of the new, without updating the checksum
   294  	if err := client2.Put(oldState.Bytes()); err != nil {
   295  		t.Fatal(err)
   296  	}
   297  
   298  	// fetching the wrong state through client1 should now error out due to a
   299  	// mismatched checksum.
   300  	if _, err := client1.Get(); !strings.HasPrefix(err.Error(), errBadChecksumFmt[:80]) {
   301  		t.Fatalf("expected state checksum error: got %s", err)
   302  	}
   303  
   304  	// update the state with the correct one after we Get again
   305  	testChecksumHook = func() {
   306  		if err := client2.Put(newState.Bytes()); err != nil {
   307  			t.Fatal(err)
   308  		}
   309  		testChecksumHook = nil
   310  	}
   311  
   312  	consistencyRetryTimeout = origTimeout
   313  
   314  	// this final Get will fail to fail the checksum verification, the above
   315  	// callback will update the state with the correct version, and Get should
   316  	// retry automatically.
   317  	if _, err := client1.Get(); err != nil {
   318  		t.Fatal(err)
   319  	}
   320  }