github.com/opentofu/opentofu@v1.7.1/internal/backend/remote-state/consul/client_test.go (about) 1 // Copyright (c) The OpenTofu Authors 2 // SPDX-License-Identifier: MPL-2.0 3 // Copyright (c) 2023 HashiCorp, Inc. 4 // SPDX-License-Identifier: MPL-2.0 5 6 package consul 7 8 import ( 9 "bytes" 10 "context" 11 "encoding/json" 12 "fmt" 13 "math/rand" 14 "net" 15 "reflect" 16 "strings" 17 "sync" 18 "testing" 19 "time" 20 21 "github.com/opentofu/opentofu/internal/backend" 22 "github.com/opentofu/opentofu/internal/encryption" 23 "github.com/opentofu/opentofu/internal/states/remote" 24 "github.com/opentofu/opentofu/internal/states/statemgr" 25 ) 26 27 func TestRemoteClient_impl(t *testing.T) { 28 var _ remote.Client = new(RemoteClient) 29 var _ remote.ClientLocker = new(RemoteClient) 30 } 31 32 func TestRemoteClient(t *testing.T) { 33 srv := newConsulTestServer(t) 34 defer func() { _ = srv.Stop() }() 35 36 testCases := []string{ 37 fmt.Sprintf("tf-unit/%s", time.Now().String()), 38 fmt.Sprintf("tf-unit/%s/", time.Now().String()), 39 } 40 41 for _, path := range testCases { 42 t.Run(path, func(*testing.T) { 43 // Get the backend 44 b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 45 "address": srv.HTTPAddr, 46 "path": path, 47 })) 48 49 // Grab the client 50 state, err := b.StateMgr(backend.DefaultStateName) 51 if err != nil { 52 t.Fatalf("err: %s", err) 53 } 54 55 // Test 56 remote.TestClient(t, state.(*remote.State).Client) 57 }) 58 } 59 } 60 61 // test the gzip functionality of the client 62 func TestRemoteClient_gzipUpgrade(t *testing.T) { 63 srv := newConsulTestServer(t) 64 defer func() { _ = srv.Stop() }() 65 66 statePath := fmt.Sprintf("tf-unit/%s", time.Now().String()) 67 68 // Get the backend 69 b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 70 "address": srv.HTTPAddr, 71 "path": statePath, 72 })) 73 74 // Grab the client 75 state, err := b.StateMgr(backend.DefaultStateName) 76 if err != nil { 77 t.Fatalf("err: %s", err) 78 } 79 80 // Test 81 remote.TestClient(t, state.(*remote.State).Client) 82 83 // create a new backend with gzip 84 b = backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 85 "address": srv.HTTPAddr, 86 "path": statePath, 87 "gzip": true, 88 })) 89 90 // Grab the client 91 state, err = b.StateMgr(backend.DefaultStateName) 92 if err != nil { 93 t.Fatalf("err: %s", err) 94 } 95 96 // Test 97 remote.TestClient(t, state.(*remote.State).Client) 98 } 99 100 // TestConsul_largeState tries to write a large payload using the Consul state 101 // manager, as there is a limit to the size of the values in the KV store it 102 // will need to be split up before being saved and put back together when read. 103 func TestConsul_largeState(t *testing.T) { 104 srv := newConsulTestServer(t) 105 defer func() { _ = srv.Stop() }() 106 107 path := "tf-unit/test-large-state" 108 109 b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 110 "address": srv.HTTPAddr, 111 "path": path, 112 })) 113 114 s, err := b.StateMgr(backend.DefaultStateName) 115 if err != nil { 116 t.Fatal(err) 117 } 118 119 c := s.(*remote.State).Client.(*RemoteClient) 120 c.Path = path 121 122 // testPaths fails the test if the keys found at the prefix don't match 123 // what is expected 124 testPaths := func(t *testing.T, expected []string) { 125 kv := c.Client.KV() 126 pairs, _, err := kv.List(c.Path, nil) 127 if err != nil { 128 t.Fatal(err) 129 } 130 res := make([]string, 0) 131 for _, p := range pairs { 132 res = append(res, p.Key) 133 } 134 if !reflect.DeepEqual(res, expected) { 135 t.Fatalf("Wrong keys: %#v", res) 136 } 137 } 138 139 testPayload := func(t *testing.T, data map[string]string, keys []string) { 140 payload, err := json.Marshal(data) 141 if err != nil { 142 t.Fatal(err) 143 } 144 err = c.Put(payload) 145 if err != nil { 146 t.Fatal("could not put payload", err) 147 } 148 149 remote, err := c.Get() 150 if err != nil { 151 t.Fatal(err) 152 } 153 154 if !bytes.Equal(payload, remote.Data) { 155 t.Fatal("the data do not match") 156 } 157 158 testPaths(t, keys) 159 } 160 161 // The default limit for the size of the value in Consul is 524288 bytes 162 testPayload( 163 t, 164 map[string]string{ 165 "foo": strings.Repeat("a", 524288+2), 166 }, 167 []string{ 168 "tf-unit/test-large-state", 169 "tf-unit/test-large-state/tfstate.2cb96f52c9fff8e0b56cb786ec4d2bed/0", 170 "tf-unit/test-large-state/tfstate.2cb96f52c9fff8e0b56cb786ec4d2bed/1", 171 }, 172 ) 173 174 // This payload is just short enough to be stored but will be bigger when 175 // going through the Transaction API as it will be base64 encoded 176 testPayload( 177 t, 178 map[string]string{ 179 "foo": strings.Repeat("a", 524288-10), 180 }, 181 []string{ 182 "tf-unit/test-large-state", 183 "tf-unit/test-large-state/tfstate.4f407ace136a86521fd0d366972fe5c7/0", 184 }, 185 ) 186 187 // We try to replace the payload with a small one, the old chunks should be removed 188 testPayload( 189 t, 190 map[string]string{"var": "a"}, 191 []string{"tf-unit/test-large-state"}, 192 ) 193 194 // Test with gzip and chunks 195 b = backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 196 "address": srv.HTTPAddr, 197 "path": path, 198 "gzip": true, 199 })) 200 201 s, err = b.StateMgr(backend.DefaultStateName) 202 if err != nil { 203 t.Fatal(err) 204 } 205 206 c = s.(*remote.State).Client.(*RemoteClient) 207 c.Path = path 208 209 // We need a long random string so it results in multiple chunks even after 210 // being gziped 211 212 // We use a fixed seed so the test can be reproductible 213 randomizer := rand.New(rand.NewSource(1234)) 214 RandStringRunes := func(n int) string { 215 var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") 216 b := make([]rune, n) 217 for i := range b { 218 b[i] = letterRunes[randomizer.Intn(len(letterRunes))] 219 } 220 return string(b) 221 } 222 223 testPayload( 224 t, 225 map[string]string{ 226 "bar": RandStringRunes(5 * (524288 + 2)), 227 }, 228 []string{ 229 "tf-unit/test-large-state", 230 "tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/0", 231 "tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/1", 232 "tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/2", 233 "tf-unit/test-large-state/tfstate.58e8160335864b520b1cc7f2222a4019/3", 234 }, 235 ) 236 237 // Deleting the state should remove all chunks 238 err = c.Delete() 239 if err != nil { 240 t.Fatal(err) 241 } 242 testPaths(t, []string{}) 243 } 244 245 func TestConsul_stateLock(t *testing.T) { 246 srv := newConsulTestServer(t) 247 defer func() { _ = srv.Stop() }() 248 249 testCases := []string{ 250 fmt.Sprintf("tf-unit/%s", time.Now().String()), 251 fmt.Sprintf("tf-unit/%s/", time.Now().String()), 252 } 253 254 for _, path := range testCases { 255 t.Run(path, func(*testing.T) { 256 // create 2 instances to get 2 remote.Clients 257 sA, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 258 "address": srv.HTTPAddr, 259 "path": path, 260 })).StateMgr(backend.DefaultStateName) 261 if err != nil { 262 t.Fatal(err) 263 } 264 265 sB, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 266 "address": srv.HTTPAddr, 267 "path": path, 268 })).StateMgr(backend.DefaultStateName) 269 if err != nil { 270 t.Fatal(err) 271 } 272 273 remote.TestRemoteLocks(t, sA.(*remote.State).Client, sB.(*remote.State).Client) 274 }) 275 } 276 } 277 278 func TestConsul_destroyLock(t *testing.T) { 279 srv := newConsulTestServer(t) 280 defer func() { _ = srv.Stop() }() 281 282 testCases := []string{ 283 fmt.Sprintf("tf-unit/%s", time.Now().String()), 284 fmt.Sprintf("tf-unit/%s/", time.Now().String()), 285 } 286 287 testLock := func(client *RemoteClient, lockPath string) { 288 // get the lock val 289 pair, _, err := client.Client.KV().Get(lockPath, nil) 290 if err != nil { 291 t.Fatal(err) 292 } 293 if pair != nil { 294 t.Fatalf("lock key not cleaned up at: %s", pair.Key) 295 } 296 } 297 298 for _, path := range testCases { 299 t.Run(path, func(*testing.T) { 300 // Get the backend 301 b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 302 "address": srv.HTTPAddr, 303 "path": path, 304 })) 305 306 // Grab the client 307 s, err := b.StateMgr(backend.DefaultStateName) 308 if err != nil { 309 t.Fatalf("err: %s", err) 310 } 311 312 clientA := s.(*remote.State).Client.(*RemoteClient) 313 314 info := statemgr.NewLockInfo() 315 id, err := clientA.Lock(info) 316 if err != nil { 317 t.Fatal(err) 318 } 319 320 lockPath := clientA.Path + lockSuffix 321 322 if err := clientA.Unlock(id); err != nil { 323 t.Fatal(err) 324 } 325 326 testLock(clientA, lockPath) 327 328 // The release the lock from a second client to test the 329 // `tofu force-unlock <lock_id>` functionality 330 s, err = b.StateMgr(backend.DefaultStateName) 331 if err != nil { 332 t.Fatalf("err: %s", err) 333 } 334 335 clientB := s.(*remote.State).Client.(*RemoteClient) 336 337 info = statemgr.NewLockInfo() 338 id, err = clientA.Lock(info) 339 if err != nil { 340 t.Fatal(err) 341 } 342 343 if err := clientB.Unlock(id); err != nil { 344 t.Fatal(err) 345 } 346 347 testLock(clientA, lockPath) 348 349 err = clientA.Unlock(id) 350 351 if err == nil { 352 t.Fatal("consul lock should have been lost") 353 } 354 if err.Error() != "consul lock was lost" { 355 t.Fatal("got wrong error", err) 356 } 357 }) 358 } 359 } 360 361 func TestConsul_lostLock(t *testing.T) { 362 srv := newConsulTestServer(t) 363 defer func() { _ = srv.Stop() }() 364 365 path := fmt.Sprintf("tf-unit/%s", time.Now().String()) 366 367 // create 2 instances to get 2 remote.Clients 368 sA, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 369 "address": srv.HTTPAddr, 370 "path": path, 371 })).StateMgr(backend.DefaultStateName) 372 if err != nil { 373 t.Fatal(err) 374 } 375 376 sB, err := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 377 "address": srv.HTTPAddr, 378 "path": path + "-not-used", 379 })).StateMgr(backend.DefaultStateName) 380 if err != nil { 381 t.Fatal(err) 382 } 383 384 info := statemgr.NewLockInfo() 385 info.Operation = "test-lost-lock" 386 id, err := sA.Lock(info) 387 if err != nil { 388 t.Fatal(err) 389 } 390 391 reLocked := make(chan struct{}) 392 testLockHook = func() { 393 close(reLocked) 394 testLockHook = nil 395 } 396 397 // now we use the second client to break the lock 398 kv := sB.(*remote.State).Client.(*RemoteClient).Client.KV() 399 _, err = kv.Delete(path+lockSuffix, nil) 400 if err != nil { 401 t.Fatal(err) 402 } 403 404 <-reLocked 405 406 if err := sA.Unlock(id); err != nil { 407 t.Fatal(err) 408 } 409 } 410 411 func TestConsul_lostLockConnection(t *testing.T) { 412 srv := newConsulTestServer(t) 413 defer func() { _ = srv.Stop() }() 414 415 // create an "unreliable" network by closing all the consul client's 416 // network connections 417 conns := &unreliableConns{} 418 origDialFn := dialContext 419 defer func() { 420 dialContext = origDialFn 421 }() 422 dialContext = conns.DialContext 423 424 path := fmt.Sprintf("tf-unit/%s", time.Now().String()) 425 426 b := backend.TestBackendConfig(t, New(encryption.StateEncryptionDisabled()), backend.TestWrapConfig(map[string]interface{}{ 427 "address": srv.HTTPAddr, 428 "path": path, 429 })) 430 431 s, err := b.StateMgr(backend.DefaultStateName) 432 if err != nil { 433 t.Fatal(err) 434 } 435 436 info := statemgr.NewLockInfo() 437 info.Operation = "test-lost-lock-connection" 438 id, err := s.Lock(info) 439 if err != nil { 440 t.Fatal(err) 441 } 442 443 // kill the connection a few times 444 for i := 0; i < 3; i++ { 445 dialed := conns.dialedDone() 446 // kill any open connections 447 conns.Kill() 448 // wait for a new connection to be dialed, and kill it again 449 <-dialed 450 } 451 452 if err := s.Unlock(id); err != nil { 453 t.Fatal("unlock error:", err) 454 } 455 } 456 457 type unreliableConns struct { 458 sync.Mutex 459 conns []net.Conn 460 dialCallback func() 461 } 462 463 func (u *unreliableConns) DialContext(ctx context.Context, netw, addr string) (net.Conn, error) { 464 u.Lock() 465 defer u.Unlock() 466 467 dialer := &net.Dialer{} 468 conn, err := dialer.DialContext(ctx, netw, addr) 469 if err != nil { 470 return nil, err 471 } 472 473 u.conns = append(u.conns, conn) 474 475 if u.dialCallback != nil { 476 u.dialCallback() 477 } 478 479 return conn, nil 480 } 481 482 func (u *unreliableConns) dialedDone() chan struct{} { 483 u.Lock() 484 defer u.Unlock() 485 dialed := make(chan struct{}) 486 u.dialCallback = func() { 487 defer close(dialed) 488 u.dialCallback = nil 489 } 490 491 return dialed 492 } 493 494 // Kill these with a deadline, just to make sure we don't end up with any EOFs 495 // that get ignored. 496 func (u *unreliableConns) Kill() { 497 u.Lock() 498 defer u.Unlock() 499 500 for _, conn := range u.conns { 501 conn.(*net.TCPConn).SetDeadline(time.Now()) 502 } 503 u.conns = nil 504 }