github.com/outbrain/consul@v1.4.5/api/lock_test.go (about) 1 package api 2 3 import ( 4 "log" 5 "net/http" 6 "net/http/httptest" 7 "net/http/httputil" 8 "strings" 9 "sync" 10 "testing" 11 "time" 12 13 "github.com/hashicorp/consul/testutil/retry" 14 ) 15 16 func createTestLock(t *testing.T, c *Client, key string) (*Lock, *Session) { 17 t.Helper() 18 session := c.Session() 19 20 se := &SessionEntry{ 21 Name: DefaultLockSessionName, 22 TTL: DefaultLockSessionTTL, 23 Behavior: SessionBehaviorDelete, 24 } 25 id, _, err := session.CreateNoChecks(se, nil) 26 if err != nil { 27 t.Fatalf("err: %v", err) 28 } 29 30 opts := &LockOptions{ 31 Key: key, 32 Session: id, 33 SessionName: se.Name, 34 SessionTTL: se.TTL, 35 } 36 lock, err := c.LockOpts(opts) 37 if err != nil { 38 t.Fatalf("err: %v", err) 39 } 40 41 return lock, session 42 } 43 44 func TestAPI_LockLockUnlock(t *testing.T) { 45 t.Parallel() 46 c, s := makeClientWithoutConnect(t) 47 defer s.Stop() 48 49 lock, session := createTestLock(t, c, "test/lock") 50 defer session.Destroy(lock.opts.Session, nil) 51 52 // Initial unlock should fail 53 err := lock.Unlock() 54 if err != ErrLockNotHeld { 55 t.Fatalf("err: %v", err) 56 } 57 58 // Should work 59 leaderCh, err := lock.Lock(nil) 60 if err != nil { 61 t.Fatalf("err: %v", err) 62 } 63 if leaderCh == nil { 64 t.Fatalf("not leader") 65 } 66 67 // Double lock should fail 68 _, err = lock.Lock(nil) 69 if err != ErrLockHeld { 70 t.Fatalf("err: %v", err) 71 } 72 73 // Should be leader 74 select { 75 case <-leaderCh: 76 t.Fatalf("should be leader") 77 default: 78 } 79 80 // Initial unlock should work 81 err = lock.Unlock() 82 if err != nil { 83 t.Fatalf("err: %v", err) 84 } 85 86 // Double unlock should fail 87 err = lock.Unlock() 88 if err != ErrLockNotHeld { 89 t.Fatalf("err: %v", err) 90 } 91 92 // Should lose leadership 93 select { 94 case <-leaderCh: 95 case <-time.After(time.Second): 96 t.Fatalf("should not be leader") 97 } 98 } 99 100 func TestAPI_LockForceInvalidate(t *testing.T) { 101 t.Parallel() 102 c, s := makeClientWithoutConnect(t) 103 defer s.Stop() 104 105 retry.Run(t, func(r *retry.R) { 106 lock, session := createTestLock(t, c, "test/lock") 107 defer session.Destroy(lock.opts.Session, nil) 108 109 // Should work 110 leaderCh, err := lock.Lock(nil) 111 if err != nil { 112 t.Fatalf("err: %v", err) 113 } 114 if leaderCh == nil { 115 t.Fatalf("not leader") 116 } 117 defer lock.Unlock() 118 119 go func() { 120 // Nuke the session, simulator an operator invalidation 121 // or a health check failure 122 session := c.Session() 123 session.Destroy(lock.lockSession, nil) 124 }() 125 126 // Should loose leadership 127 select { 128 case <-leaderCh: 129 case <-time.After(time.Second): 130 t.Fatalf("should not be leader") 131 } 132 }) 133 } 134 135 func TestAPI_LockDeleteKey(t *testing.T) { 136 t.Parallel() 137 c, s := makeClientWithoutConnect(t) 138 defer s.Stop() 139 140 // This uncovered some issues around special-case handling of low index 141 // numbers where it would work with a low number but fail for higher 142 // ones, so we loop this a bit to sweep the index up out of that 143 // territory. 144 for i := 0; i < 10; i++ { 145 func() { 146 lock, session := createTestLock(t, c, "test/lock") 147 defer session.Destroy(lock.opts.Session, nil) 148 149 // Should work 150 leaderCh, err := lock.Lock(nil) 151 if err != nil { 152 t.Fatalf("err: %v", err) 153 } 154 if leaderCh == nil { 155 t.Fatalf("not leader") 156 } 157 defer lock.Unlock() 158 159 go func() { 160 // Nuke the key, simulate an operator intervention 161 kv := c.KV() 162 kv.Delete("test/lock", nil) 163 }() 164 165 // Should loose leadership 166 select { 167 case <-leaderCh: 168 case <-time.After(10 * time.Second): 169 t.Fatalf("should not be leader") 170 } 171 }() 172 } 173 } 174 175 func TestAPI_LockContend(t *testing.T) { 176 t.Parallel() 177 c, s := makeClientWithoutConnect(t) 178 defer s.Stop() 179 180 wg := &sync.WaitGroup{} 181 acquired := make([]bool, 3) 182 for idx := range acquired { 183 wg.Add(1) 184 go func(idx int) { 185 defer wg.Done() 186 lock, session := createTestLock(t, c, "test/lock") 187 defer session.Destroy(lock.opts.Session, nil) 188 189 // Should work eventually, will contend 190 leaderCh, err := lock.Lock(nil) 191 if err != nil { 192 t.Fatalf("err: %v", err) 193 } 194 if leaderCh == nil { 195 t.Fatalf("not leader") 196 } 197 defer lock.Unlock() 198 log.Printf("Contender %d acquired", idx) 199 200 // Set acquired and then leave 201 acquired[idx] = true 202 }(idx) 203 } 204 205 // Wait for termination 206 doneCh := make(chan struct{}) 207 go func() { 208 wg.Wait() 209 close(doneCh) 210 }() 211 212 // Wait for everybody to get a turn 213 select { 214 case <-doneCh: 215 case <-time.After(3 * DefaultLockRetryTime): 216 t.Fatalf("timeout") 217 } 218 219 for idx, did := range acquired { 220 if !did { 221 t.Fatalf("contender %d never acquired", idx) 222 } 223 } 224 } 225 226 func TestAPI_LockDestroy(t *testing.T) { 227 t.Parallel() 228 c, s := makeClientWithoutConnect(t) 229 defer s.Stop() 230 231 lock, session := createTestLock(t, c, "test/lock") 232 defer session.Destroy(lock.opts.Session, nil) 233 234 // Should work 235 leaderCh, err := lock.Lock(nil) 236 if err != nil { 237 t.Fatalf("err: %v", err) 238 } 239 if leaderCh == nil { 240 t.Fatalf("not leader") 241 } 242 243 // Destroy should fail 244 if err := lock.Destroy(); err != ErrLockHeld { 245 t.Fatalf("err: %v", err) 246 } 247 248 // Should be able to release 249 err = lock.Unlock() 250 if err != nil { 251 t.Fatalf("err: %v", err) 252 } 253 254 // Acquire with a different lock 255 l2, session := createTestLock(t, c, "test/lock") 256 defer session.Destroy(lock.opts.Session, nil) 257 258 // Should work 259 leaderCh, err = l2.Lock(nil) 260 if err != nil { 261 t.Fatalf("err: %v", err) 262 } 263 if leaderCh == nil { 264 t.Fatalf("not leader") 265 } 266 267 // Destroy should still fail 268 if err := lock.Destroy(); err != ErrLockInUse { 269 t.Fatalf("err: %v", err) 270 } 271 272 // Should release 273 err = l2.Unlock() 274 if err != nil { 275 t.Fatalf("err: %v", err) 276 } 277 278 // Destroy should work 279 err = lock.Destroy() 280 if err != nil { 281 t.Fatalf("err: %v", err) 282 } 283 284 // Double destroy should work 285 err = l2.Destroy() 286 if err != nil { 287 t.Fatalf("err: %v", err) 288 } 289 } 290 291 func TestAPI_LockConflict(t *testing.T) { 292 t.Parallel() 293 c, s := makeClientWithoutConnect(t) 294 defer s.Stop() 295 296 sema, session := createTestSemaphore(t, c, "test/lock/", 2) 297 defer session.Destroy(sema.opts.Session, nil) 298 299 // Should work 300 lockCh, err := sema.Acquire(nil) 301 if err != nil { 302 t.Fatalf("err: %v", err) 303 } 304 if lockCh == nil { 305 t.Fatalf("not hold") 306 } 307 defer sema.Release() 308 309 lock, session := createTestLock(t, c, "test/lock/.lock") 310 defer session.Destroy(lock.opts.Session, nil) 311 312 // Should conflict with semaphore 313 _, err = lock.Lock(nil) 314 if err != ErrLockConflict { 315 t.Fatalf("err: %v", err) 316 } 317 318 // Should conflict with semaphore 319 err = lock.Destroy() 320 if err != ErrLockConflict { 321 t.Fatalf("err: %v", err) 322 } 323 } 324 325 func TestAPI_LockReclaimLock(t *testing.T) { 326 t.Parallel() 327 c, s := makeClientWithoutConnect(t) 328 defer s.Stop() 329 330 s.WaitForSerfCheck(t) 331 332 session, _, err := c.Session().Create(&SessionEntry{}, nil) 333 if err != nil { 334 t.Fatalf("err: %v", err) 335 } 336 337 lock, err := c.LockOpts(&LockOptions{Key: "test/lock", Session: session}) 338 if err != nil { 339 t.Fatalf("err: %v", err) 340 } 341 342 // Should work 343 leaderCh, err := lock.Lock(nil) 344 if err != nil { 345 t.Fatalf("err: %v", err) 346 } 347 if leaderCh == nil { 348 t.Fatalf("not leader") 349 } 350 defer lock.Unlock() 351 352 l2, err := c.LockOpts(&LockOptions{Key: "test/lock", Session: session}) 353 if err != nil { 354 t.Fatalf("err: %v", err) 355 } 356 357 reclaimed := make(chan (<-chan struct{}), 1) 358 go func() { 359 l2Ch, err := l2.Lock(nil) 360 if err != nil { 361 t.Fatalf("not locked: %v", err) 362 } 363 reclaimed <- l2Ch 364 }() 365 366 // Should reclaim the lock 367 var leader2Ch <-chan struct{} 368 369 select { 370 case leader2Ch = <-reclaimed: 371 case <-time.After(time.Second): 372 t.Fatalf("should have locked") 373 } 374 375 // unlock should work 376 err = l2.Unlock() 377 if err != nil { 378 t.Fatalf("err: %v", err) 379 } 380 381 //Both locks should see the unlock 382 select { 383 case <-leader2Ch: 384 case <-time.After(time.Second): 385 t.Fatalf("should not be leader") 386 } 387 388 select { 389 case <-leaderCh: 390 case <-time.After(time.Second): 391 t.Fatalf("should not be leader") 392 } 393 } 394 395 func TestAPI_LockMonitorRetry(t *testing.T) { 396 t.Parallel() 397 raw, s := makeClientWithoutConnect(t) 398 defer s.Stop() 399 400 s.WaitForSerfCheck(t) 401 402 // Set up a server that always responds with 500 errors. 403 failer := func(w http.ResponseWriter, req *http.Request) { 404 w.WriteHeader(500) 405 } 406 outage := httptest.NewServer(http.HandlerFunc(failer)) 407 defer outage.Close() 408 409 // Set up a reverse proxy that will send some requests to the 410 // 500 server and pass everything else through to the real Consul 411 // server. 412 var mutex sync.Mutex 413 errors := 0 414 director := func(req *http.Request) { 415 mutex.Lock() 416 defer mutex.Unlock() 417 418 req.URL.Scheme = "http" 419 if errors > 0 && req.Method == "GET" && strings.Contains(req.URL.Path, "/v1/kv/test/lock") { 420 req.URL.Host = outage.URL[7:] // Strip off "http://". 421 errors-- 422 } else { 423 req.URL.Host = raw.config.Address 424 } 425 } 426 proxy := httptest.NewServer(&httputil.ReverseProxy{Director: director}) 427 defer proxy.Close() 428 429 // Make another client that points at the proxy instead of the real 430 // Consul server. 431 config := raw.config 432 config.Address = proxy.URL[7:] // Strip off "http://". 433 c, err := NewClient(&config) 434 if err != nil { 435 t.Fatalf("err: %v", err) 436 } 437 438 // Set up a lock with retries enabled. 439 opts := &LockOptions{ 440 Key: "test/lock", 441 SessionTTL: "60s", 442 MonitorRetries: 3, 443 } 444 lock, err := c.LockOpts(opts) 445 if err != nil { 446 t.Fatalf("err: %v", err) 447 } 448 449 // Make sure the default got set. 450 if lock.opts.MonitorRetryTime != DefaultMonitorRetryTime { 451 t.Fatalf("bad: %d", lock.opts.MonitorRetryTime) 452 } 453 454 // Now set a custom time for the test. 455 opts.MonitorRetryTime = 250 * time.Millisecond 456 lock, err = c.LockOpts(opts) 457 if err != nil { 458 t.Fatalf("err: %v", err) 459 } 460 if lock.opts.MonitorRetryTime != 250*time.Millisecond { 461 t.Fatalf("bad: %d", lock.opts.MonitorRetryTime) 462 } 463 464 // Should get the lock. 465 leaderCh, err := lock.Lock(nil) 466 if err != nil { 467 t.Fatalf("err: %v", err) 468 } 469 if leaderCh == nil { 470 t.Fatalf("not leader") 471 } 472 473 // Poke the key using the raw client to force the monitor to wake up 474 // and check the lock again. This time we will return errors for some 475 // of the responses. 476 mutex.Lock() 477 errors = 2 478 mutex.Unlock() 479 pair, _, err := raw.KV().Get("test/lock", &QueryOptions{}) 480 if err != nil { 481 t.Fatalf("err: %v", err) 482 } 483 if _, err := raw.KV().Put(pair, &WriteOptions{}); err != nil { 484 t.Fatalf("err: %v", err) 485 } 486 time.Sleep(5 * opts.MonitorRetryTime) 487 488 // Should still be the leader. 489 select { 490 case <-leaderCh: 491 t.Fatalf("should be leader") 492 default: 493 } 494 495 // Now return an overwhelming number of errors. 496 mutex.Lock() 497 errors = 10 498 mutex.Unlock() 499 if _, err := raw.KV().Put(pair, &WriteOptions{}); err != nil { 500 t.Fatalf("err: %v", err) 501 } 502 time.Sleep(5 * opts.MonitorRetryTime) 503 504 // Should lose leadership. 505 select { 506 case <-leaderCh: 507 case <-time.After(time.Second): 508 t.Fatalf("should not be leader") 509 } 510 } 511 512 func TestAPI_LockOneShot(t *testing.T) { 513 t.Parallel() 514 c, s := makeClientWithoutConnect(t) 515 defer s.Stop() 516 517 s.WaitForSerfCheck(t) 518 519 // Set up a lock as a one-shot. 520 opts := &LockOptions{ 521 Key: "test/lock", 522 LockTryOnce: true, 523 } 524 lock, err := c.LockOpts(opts) 525 if err != nil { 526 t.Fatalf("err: %v", err) 527 } 528 529 // Make sure the default got set. 530 if lock.opts.LockWaitTime != DefaultLockWaitTime { 531 t.Fatalf("bad: %d", lock.opts.LockWaitTime) 532 } 533 534 // Now set a custom time for the test. 535 opts.LockWaitTime = 250 * time.Millisecond 536 lock, err = c.LockOpts(opts) 537 if err != nil { 538 t.Fatalf("err: %v", err) 539 } 540 if lock.opts.LockWaitTime != 250*time.Millisecond { 541 t.Fatalf("bad: %d", lock.opts.LockWaitTime) 542 } 543 544 // Should get the lock. 545 ch, err := lock.Lock(nil) 546 if err != nil { 547 t.Fatalf("err: %v", err) 548 } 549 if ch == nil { 550 t.Fatalf("not leader") 551 } 552 553 // Now try with another session. 554 contender, err := c.LockOpts(opts) 555 if err != nil { 556 t.Fatalf("err: %v", err) 557 } 558 start := time.Now() 559 ch, err = contender.Lock(nil) 560 if err != nil { 561 t.Fatalf("err: %v", err) 562 } 563 if ch != nil { 564 t.Fatalf("should not be leader") 565 } 566 diff := time.Since(start) 567 if diff < contender.opts.LockWaitTime || diff > 2*contender.opts.LockWaitTime { 568 t.Fatalf("time out of bounds: %9.6f", diff.Seconds()) 569 } 570 571 // Unlock and then make sure the contender can get it. 572 if err := lock.Unlock(); err != nil { 573 t.Fatalf("err: %v", err) 574 } 575 ch, err = contender.Lock(nil) 576 if err != nil { 577 t.Fatalf("err: %v", err) 578 } 579 if ch == nil { 580 t.Fatalf("should be leader") 581 } 582 }