github.com/opentofu/opentofu@v1.7.1/internal/backend/remote-state/consul/client_test.go (about)

     1  // Copyright (c) The OpenTofu Authors
     2  // SPDX-License-Identifier: MPL-2.0
     3  // Copyright (c) 2023 HashiCorp, Inc.
     4  // SPDX-License-Identifier: MPL-2.0
     5  
     6  package consul
     7  
     8  import (
     9  	"bytes"
    10  	"context"
    11  	"encoding/json"
    12  	"fmt"
    13  	"math/rand"
    14  	"net"
    15  	"reflect"
    16  	"strings"
    17  	"sync"
    18  	"testing"
    19  	"time"
    20  
    21  	"github.com/opentofu/opentofu/internal/backend"
    22  	"github.com/opentofu/opentofu/internal/encryption"
    23  	"github.com/opentofu/opentofu/internal/states/remote"
    24  	"github.com/opentofu/opentofu/internal/states/statemgr"
    25  )
    26  
    27  func TestRemoteClient_impl(t *testing.T) {
    28  	var _ remote.Client = new(RemoteClient)
    29  	var _ remote.ClientLocker = new(RemoteClient)
    30  }
    31  
    32  func TestRemoteClient(t *testing.T) {
    33  	srv := newConsulTestServer(t)
    34  	defer func() { _ = srv.Stop() }()
    35  
    36  	testCases := []string{
    37  		fmt.Sprintf("tf-unit/%s", time.Now().String()),
    38  		fmt.Sprintf("tf-unit/%s/", time.Now().String()),
    39  	}
    40  
    41  	for _, path := range testCases {
    42  		t.Run(path, func(*testing.T) {
    43  			// Get the backend
    44  			b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
    45  				"address": srv.HTTPAddr,
    46  				"path":    path,
    47  			}))
    48  
    49  			// Grab the client
    50  			state, err := b.StateMgr(backend.DefaultStateName)
    51  			if err != nil {
    52  				t.Fatalf("err: %s", err)
    53  			}
    54  
    55  			// Test
    56  			remote.TestClient(t, state.(*remote.State).Client)
    57  		})
    58  	}
    59  }
    60  
    61  // test the gzip functionality of the client
    62  func TestRemoteClient_gzipUpgrade(t *testing.T) {
    63  	srv := newConsulTestServer(t)
    64  	defer func() { _ = srv.Stop() }()
    65  
    66  	statePath := fmt.Sprintf("tf-unit/%s", time.Now().String())
    67  
    68  	// Get the backend
    69  	b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
    70  		"address": srv.HTTPAddr,
    71  		"path":    statePath,
    72  	}))
    73  
    74  	// Grab the client
    75  	state, err := b.StateMgr(backend.DefaultStateName)
    76  	if err != nil {
    77  		t.Fatalf("err: %s", err)
    78  	}
    79  
    80  	// Test
    81  	remote.TestClient(t, state.(*remote.State).Client)
    82  
    83  	// create a new backend with gzip
    84  	b = backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
    85  		"address": srv.HTTPAddr,
    86  		"path":    statePath,
    87  		"gzip":    true,
    88  	}))
    89  
    90  	// Grab the client
    91  	state, err = b.StateMgr(backend.DefaultStateName)
    92  	if err != nil {
    93  		t.Fatalf("err: %s", err)
    94  	}
    95  
    96  	// Test
    97  	remote.TestClient(t, state.(*remote.State).Client)
    98  }
    99  
   100  // TestConsul_largeState tries to write a large payload using the Consul state
   101  // manager, as there is a limit to the size of the values in the KV store it
   102  // will need to be split up before being saved and put back together when read.
   103  func TestConsul_largeState(t *testing.T) {
   104  	srv := newConsulTestServer(t)
   105  	defer func() { _ = srv.Stop() }()
   106  
   107  	path := "tf-unit/test-large-state"
   108  
   109  	b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   110  		"address": srv.HTTPAddr,
   111  		"path":    path,
   112  	}))
   113  
   114  	s, err := b.StateMgr(backend.DefaultStateName)
   115  	if err != nil {
   116  		t.Fatal(err)
   117  	}
   118  
   119  	c := s.(*remote.State).Client.(*RemoteClient)
   120  	c.Path = path
   121  
   122  	// testPaths fails the test if the keys found at the prefix don't match
   123  	// what is expected
   124  	testPaths := func(t *testing.T, expected []string) {
   125  		kv := c.Client.KV()
   126  		pairs, _, err := kv.List(c.Path, nil)
   127  		if err != nil {
   128  			t.Fatal(err)
   129  		}
   130  		res := make([]string, 0)
   131  		for _, p := range pairs {
   132  			res = append(res, p.Key)
   133  		}
   134  		if !reflect.DeepEqual(res, expected) {
   135  			t.Fatalf("Wrong keys: %#v", res)
   136  		}
   137  	}
   138  
   139  	testPayload := func(t *testing.T, data map[string]string, keys []string) {
   140  		payload, err := json.Marshal(data)
   141  		if err != nil {
   142  			t.Fatal(err)
   143  		}
   144  		err = c.Put(payload)
   145  		if err != nil {
   146  			t.Fatal("could not put payload", err)
   147  		}
   148  
   149  		remote, err := c.Get()
   150  		if err != nil {
   151  			t.Fatal(err)
   152  		}
   153  
   154  		if !bytes.Equal(payload, remote.Data) {
   155  			t.Fatal("the data do not match")
   156  		}
   157  
   158  		testPaths(t, keys)
   159  	}
   160  
   161  	// The default limit for the size of the value in Consul is 524288 bytes
   162  	testPayload(
   163  		t,
   164  		map[string]string{
   165  			"foo": strings.Repeat("a", 524288+2),
   166  		},
   167  		[]string{
   168  			"tf-unit/test-large-state",
   169  			"tf-unit/test-large-state/tfstate.2cb96f52c9fff8e0b56cb786ec4d2bed/0",
   170  			"tf-unit/test-large-state/tfstate.2cb96f52c9fff8e0b56cb786ec4d2bed/1",
   171  		},
   172  	)
   173  
   174  	// This payload is just short enough to be stored but will be bigger when
   175  	// going through the Transaction API as it will be base64 encoded
   176  	testPayload(
   177  		t,
   178  		map[string]string{
   179  			"foo": strings.Repeat("a", 524288-10),
   180  		},
   181  		[]string{
   182  			"tf-unit/test-large-state",
   183  			"tf-unit/test-large-state/tfstate.4f407ace136a86521fd0d366972fe5c7/0",
   184  		},
   185  	)
   186  
   187  	// We try to replace the payload with a small one, the old chunks should be removed
   188  	testPayload(
   189  		t,
   190  		map[string]string{"var": "a"},
   191  		[]string{"tf-unit/test-large-state"},
   192  	)
   193  
   194  	// Test with gzip and chunks
   195  	b = backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   196  		"address": srv.HTTPAddr,
   197  		"path":    path,
   198  		"gzip":    true,
   199  	}))
   200  
   201  	s, err = b.StateMgr(backend.DefaultStateName)
   202  	if err != nil {
   203  		t.Fatal(err)
   204  	}
   205  
   206  	c = s.(*remote.State).Client.(*RemoteClient)
   207  	c.Path = path
   208  
   209  	// We need a long random string so it results in multiple chunks even after
   210  	// being gziped
   211  
   212  	// We use a fixed seed so the test can be reproductible
   213  	randomizer := rand.New(rand.NewSource(1234))
   214  	RandStringRunes := func(n int) string {
   215  		var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
   216  		b := make([]rune, n)
   217  		for i := range b {
   218  			b[i] = letterRunes[randomizer.Intn(len(letterRunes))]
   219  		}
   220  		return string(b)
   221  	}
   222  
   223  	testPayload(
   224  		t,
   225  		map[string]string{
   226  			"bar": RandStringRunes(5 * (524288 + 2)),
   227  		},
   228  		[]string{
   229  			"tf-unit/test-large-state",
   230  			"tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/0",
   231  			"tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/1",
   232  			"tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/2",
   233  			"tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/3",
   234  		},
   235  	)
   236  
   237  	// Deleting the state should remove all chunks
   238  	err = c.Delete()
   239  	if err != nil {
   240  		t.Fatal(err)
   241  	}
   242  	testPaths(t, []string{})
   243  }
   244  
   245  func TestConsul_stateLock(t *testing.T) {
   246  	srv := newConsulTestServer(t)
   247  	defer func() { _ = srv.Stop() }()
   248  
   249  	testCases := []string{
   250  		fmt.Sprintf("tf-unit/%s", time.Now().String()),
   251  		fmt.Sprintf("tf-unit/%s/", time.Now().String()),
   252  	}
   253  
   254  	for _, path := range testCases {
   255  		t.Run(path, func(*testing.T) {
   256  			// create 2 instances to get 2 remote.Clients
   257  			sA, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   258  				"address": srv.HTTPAddr,
   259  				"path":    path,
   260  			})).StateMgr(backend.DefaultStateName)
   261  			if err != nil {
   262  				t.Fatal(err)
   263  			}
   264  
   265  			sB, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   266  				"address": srv.HTTPAddr,
   267  				"path":    path,
   268  			})).StateMgr(backend.DefaultStateName)
   269  			if err != nil {
   270  				t.Fatal(err)
   271  			}
   272  
   273  			remote.TestRemoteLocks(t, sA.(*remote.State).Client, sB.(*remote.State).Client)
   274  		})
   275  	}
   276  }
   277  
   278  func TestConsul_destroyLock(t *testing.T) {
   279  	srv := newConsulTestServer(t)
   280  	defer func() { _ = srv.Stop() }()
   281  
   282  	testCases := []string{
   283  		fmt.Sprintf("tf-unit/%s", time.Now().String()),
   284  		fmt.Sprintf("tf-unit/%s/", time.Now().String()),
   285  	}
   286  
   287  	testLock := func(client *RemoteClient, lockPath string) {
   288  		// get the lock val
   289  		pair, _, err := client.Client.KV().Get(lockPath, nil)
   290  		if err != nil {
   291  			t.Fatal(err)
   292  		}
   293  		if pair != nil {
   294  			t.Fatalf("lock key not cleaned up at: %s", pair.Key)
   295  		}
   296  	}
   297  
   298  	for _, path := range testCases {
   299  		t.Run(path, func(*testing.T) {
   300  			// Get the backend
   301  			b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   302  				"address": srv.HTTPAddr,
   303  				"path":    path,
   304  			}))
   305  
   306  			// Grab the client
   307  			s, err := b.StateMgr(backend.DefaultStateName)
   308  			if err != nil {
   309  				t.Fatalf("err: %s", err)
   310  			}
   311  
   312  			clientA := s.(*remote.State).Client.(*RemoteClient)
   313  
   314  			info := statemgr.NewLockInfo()
   315  			id, err := clientA.Lock(info)
   316  			if err != nil {
   317  				t.Fatal(err)
   318  			}
   319  
   320  			lockPath := clientA.Path + lockSuffix
   321  
   322  			if err := clientA.Unlock(id); err != nil {
   323  				t.Fatal(err)
   324  			}
   325  
   326  			testLock(clientA, lockPath)
   327  
   328  			// The release the lock from a second client to test the
   329  			// `tofu force-unlock <lock_id>` functionality
   330  			s, err = b.StateMgr(backend.DefaultStateName)
   331  			if err != nil {
   332  				t.Fatalf("err: %s", err)
   333  			}
   334  
   335  			clientB := s.(*remote.State).Client.(*RemoteClient)
   336  
   337  			info = statemgr.NewLockInfo()
   338  			id, err = clientA.Lock(info)
   339  			if err != nil {
   340  				t.Fatal(err)
   341  			}
   342  
   343  			if err := clientB.Unlock(id); err != nil {
   344  				t.Fatal(err)
   345  			}
   346  
   347  			testLock(clientA, lockPath)
   348  
   349  			err = clientA.Unlock(id)
   350  
   351  			if err == nil {
   352  				t.Fatal("consul lock should have been lost")
   353  			}
   354  			if err.Error() != "consul lock was lost" {
   355  				t.Fatal("got wrong error", err)
   356  			}
   357  		})
   358  	}
   359  }
   360  
   361  func TestConsul_lostLock(t *testing.T) {
   362  	srv := newConsulTestServer(t)
   363  	defer func() { _ = srv.Stop() }()
   364  
   365  	path := fmt.Sprintf("tf-unit/%s", time.Now().String())
   366  
   367  	// create 2 instances to get 2 remote.Clients
   368  	sA, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   369  		"address": srv.HTTPAddr,
   370  		"path":    path,
   371  	})).StateMgr(backend.DefaultStateName)
   372  	if err != nil {
   373  		t.Fatal(err)
   374  	}
   375  
   376  	sB, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   377  		"address": srv.HTTPAddr,
   378  		"path":    path + "-not-used",
   379  	})).StateMgr(backend.DefaultStateName)
   380  	if err != nil {
   381  		t.Fatal(err)
   382  	}
   383  
   384  	info := statemgr.NewLockInfo()
   385  	info.Operation = "test-lost-lock"
   386  	id, err := sA.Lock(info)
   387  	if err != nil {
   388  		t.Fatal(err)
   389  	}
   390  
   391  	reLocked := make(chan struct{})
   392  	testLockHook = func() {
   393  		close(reLocked)
   394  		testLockHook = nil
   395  	}
   396  
   397  	// now we use the second client to break the lock
   398  	kv := sB.(*remote.State).Client.(*RemoteClient).Client.KV()
   399  	_, err = kv.Delete(path+lockSuffix, nil)
   400  	if err != nil {
   401  		t.Fatal(err)
   402  	}
   403  
   404  	<-reLocked
   405  
   406  	if err := sA.Unlock(id); err != nil {
   407  		t.Fatal(err)
   408  	}
   409  }
   410  
   411  func TestConsul_lostLockConnection(t *testing.T) {
   412  	srv := newConsulTestServer(t)
   413  	defer func() { _ = srv.Stop() }()
   414  
   415  	// create an "unreliable" network by closing all the consul client's
   416  	// network connections
   417  	conns := &unreliableConns{}
   418  	origDialFn := dialContext
   419  	defer func() {
   420  		dialContext = origDialFn
   421  	}()
   422  	dialContext = conns.DialContext
   423  
   424  	path := fmt.Sprintf("tf-unit/%s", time.Now().String())
   425  
   426  	b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{
   427  		"address": srv.HTTPAddr,
   428  		"path":    path,
   429  	}))
   430  
   431  	s, err := b.StateMgr(backend.DefaultStateName)
   432  	if err != nil {
   433  		t.Fatal(err)
   434  	}
   435  
   436  	info := statemgr.NewLockInfo()
   437  	info.Operation = "test-lost-lock-connection"
   438  	id, err := s.Lock(info)
   439  	if err != nil {
   440  		t.Fatal(err)
   441  	}
   442  
   443  	// kill the connection a few times
   444  	for i := 0; i < 3; i++ {
   445  		dialed := conns.dialedDone()
   446  		// kill any open connections
   447  		conns.Kill()
   448  		// wait for a new connection to be dialed, and kill it again
   449  		<-dialed
   450  	}
   451  
   452  	if err := s.Unlock(id); err != nil {
   453  		t.Fatal("unlock error:", err)
   454  	}
   455  }
   456  
   457  type unreliableConns struct {
   458  	sync.Mutex
   459  	conns        []net.Conn
   460  	dialCallback func()
   461  }
   462  
   463  func (u *unreliableConns) DialContext(ctx context.Context, netw, addr string) (net.Conn, error) {
   464  	u.Lock()
   465  	defer u.Unlock()
   466  
   467  	dialer := &net.Dialer{}
   468  	conn, err := dialer.DialContext(ctx, netw, addr)
   469  	if err != nil {
   470  		return nil, err
   471  	}
   472  
   473  	u.conns = append(u.conns, conn)
   474  
   475  	if u.dialCallback != nil {
   476  		u.dialCallback()
   477  	}
   478  
   479  	return conn, nil
   480  }
   481  
   482  func (u *unreliableConns) dialedDone() chan struct{} {
   483  	u.Lock()
   484  	defer u.Unlock()
   485  	dialed := make(chan struct{})
   486  	u.dialCallback = func() {
   487  		defer close(dialed)
   488  		u.dialCallback = nil
   489  	}
   490  
   491  	return dialed
   492  }
   493  
   494  // Kill these with a deadline, just to make sure we don't end up with any EOFs
   495  // that get ignored.
   496  func (u *unreliableConns) Kill() {
   497  	u.Lock()
   498  	defer u.Unlock()
   499  
   500  	for _, conn := range u.conns {
   501  		conn.(*net.TCPConn).SetDeadline(time.Now())
   502  	}
   503  	u.conns = nil
   504  }