github.com/outbrain/consul@v1.4.5/api/semaphore_test.go (about) 1 package api 2 3 import ( 4 "log" 5 "net/http" 6 "net/http/httptest" 7 "net/http/httputil" 8 "strings" 9 "sync" 10 "testing" 11 "time" 12 ) 13 14 func createTestSemaphore(t *testing.T, c *Client, prefix string, limit int) (*Semaphore, *Session) { 15 t.Helper() 16 session := c.Session() 17 18 se := &SessionEntry{ 19 Name: DefaultSemaphoreSessionName, 20 TTL: DefaultSemaphoreSessionTTL, 21 Behavior: SessionBehaviorDelete, 22 } 23 id, _, err := session.CreateNoChecks(se, nil) 24 if err != nil { 25 t.Fatalf("err: %v", err) 26 } 27 28 opts := &SemaphoreOptions{ 29 Prefix: prefix, 30 Limit: limit, 31 Session: id, 32 SessionName: se.Name, 33 SessionTTL: se.TTL, 34 } 35 sema, err := c.SemaphoreOpts(opts) 36 if err != nil { 37 t.Fatalf("err: %v", err) 38 } 39 40 return sema, session 41 } 42 43 func TestAPI_SemaphoreAcquireRelease(t *testing.T) { 44 t.Parallel() 45 c, s := makeClient(t) 46 defer s.Stop() 47 48 sema, session := createTestSemaphore(t, c, "test/semaphore", 2) 49 defer session.Destroy(sema.opts.Session, nil) 50 51 // Initial release should fail 52 err := sema.Release() 53 if err != ErrSemaphoreNotHeld { 54 t.Fatalf("err: %v", err) 55 } 56 57 // Should work 58 lockCh, err := sema.Acquire(nil) 59 if err != nil { 60 t.Fatalf("err: %v", err) 61 } 62 if lockCh == nil { 63 t.Fatalf("not hold") 64 } 65 66 // Double lock should fail 67 _, err = sema.Acquire(nil) 68 if err != ErrSemaphoreHeld { 69 t.Fatalf("err: %v", err) 70 } 71 72 // Should be held 73 select { 74 case <-lockCh: 75 t.Fatalf("should be held") 76 default: 77 } 78 79 // Initial release should work 80 err = sema.Release() 81 if err != nil { 82 t.Fatalf("err: %v", err) 83 } 84 85 // Double unlock should fail 86 err = sema.Release() 87 if err != ErrSemaphoreNotHeld { 88 t.Fatalf("err: %v", err) 89 } 90 91 // Should lose resource 92 select { 93 case <-lockCh: 94 case <-time.After(time.Second): 95 t.Fatalf("should not be held") 96 } 97 } 98 99 func TestAPI_SemaphoreForceInvalidate(t *testing.T) { 100 t.Parallel() 101 c, s := makeClient(t) 102 defer s.Stop() 103 104 sema, session := createTestSemaphore(t, c, "test/semaphore", 2) 105 defer session.Destroy(sema.opts.Session, nil) 106 107 // Should work 108 lockCh, err := sema.Acquire(nil) 109 if err != nil { 110 t.Fatalf("err: %v", err) 111 } 112 if lockCh == nil { 113 t.Fatalf("not acquired") 114 } 115 defer sema.Release() 116 117 go func() { 118 // Nuke the session, simulator an operator invalidation 119 // or a health check failure 120 session := c.Session() 121 session.Destroy(sema.lockSession, nil) 122 }() 123 124 // Should loose slot 125 select { 126 case <-lockCh: 127 case <-time.After(time.Second): 128 t.Fatalf("should not be locked") 129 } 130 } 131 132 func TestAPI_SemaphoreDeleteKey(t *testing.T) { 133 t.Parallel() 134 c, s := makeClient(t) 135 defer s.Stop() 136 137 sema, session := createTestSemaphore(t, c, "test/semaphore", 2) 138 defer session.Destroy(sema.opts.Session, nil) 139 140 // Should work 141 lockCh, err := sema.Acquire(nil) 142 if err != nil { 143 t.Fatalf("err: %v", err) 144 } 145 if lockCh == nil { 146 t.Fatalf("not locked") 147 } 148 defer sema.Release() 149 150 go func() { 151 // Nuke the key, simulate an operator intervention 152 kv := c.KV() 153 kv.DeleteTree("test/semaphore", nil) 154 }() 155 156 // Should loose leadership 157 select { 158 case <-lockCh: 159 case <-time.After(time.Second): 160 t.Fatalf("should not be locked") 161 } 162 } 163 164 func TestAPI_SemaphoreContend(t *testing.T) { 165 t.Parallel() 166 c, s := makeClient(t) 167 defer s.Stop() 168 169 wg := &sync.WaitGroup{} 170 acquired := make([]bool, 4) 171 for idx := range acquired { 172 wg.Add(1) 173 go func(idx int) { 174 defer wg.Done() 175 sema, session := createTestSemaphore(t, c, "test/semaphore", 2) 176 defer session.Destroy(sema.opts.Session, nil) 177 178 // Should work eventually, will contend 179 lockCh, err := sema.Acquire(nil) 180 if err != nil { 181 t.Fatalf("err: %v", err) 182 } 183 if lockCh == nil { 184 t.Fatalf("not locked") 185 } 186 defer sema.Release() 187 log.Printf("Contender %d acquired", idx) 188 189 // Set acquired and then leave 190 acquired[idx] = true 191 }(idx) 192 } 193 194 // Wait for termination 195 doneCh := make(chan struct{}) 196 go func() { 197 wg.Wait() 198 close(doneCh) 199 }() 200 201 // Wait for everybody to get a turn 202 select { 203 case <-doneCh: 204 case <-time.After(3 * DefaultLockRetryTime): 205 t.Fatalf("timeout") 206 } 207 208 for idx, did := range acquired { 209 if !did { 210 t.Fatalf("contender %d never acquired", idx) 211 } 212 } 213 } 214 215 func TestAPI_SemaphoreBadLimit(t *testing.T) { 216 t.Parallel() 217 c, s := makeClient(t) 218 defer s.Stop() 219 220 sema, err := c.SemaphorePrefix("test/semaphore", 0) 221 if err == nil { 222 t.Fatalf("should error, limit must be positive") 223 } 224 225 sema, session := createTestSemaphore(t, c, "test/semaphore", 1) 226 defer session.Destroy(sema.opts.Session, nil) 227 228 _, err = sema.Acquire(nil) 229 if err != nil { 230 t.Fatalf("err: %v", err) 231 } 232 233 sema2, session := createTestSemaphore(t, c, "test/semaphore", 2) 234 defer session.Destroy(sema.opts.Session, nil) 235 236 _, err = sema2.Acquire(nil) 237 if err.Error() != "semaphore limit conflict (lock: 1, local: 2)" { 238 t.Fatalf("err: %v", err) 239 } 240 } 241 242 func TestAPI_SemaphoreDestroy(t *testing.T) { 243 t.Parallel() 244 c, s := makeClient(t) 245 defer s.Stop() 246 247 sema, session := createTestSemaphore(t, c, "test/semaphore", 2) 248 defer session.Destroy(sema.opts.Session, nil) 249 250 sema2, session := createTestSemaphore(t, c, "test/semaphore", 2) 251 defer session.Destroy(sema.opts.Session, nil) 252 253 _, err := sema.Acquire(nil) 254 if err != nil { 255 t.Fatalf("err: %v", err) 256 } 257 258 _, err = sema2.Acquire(nil) 259 if err != nil { 260 t.Fatalf("err: %v", err) 261 } 262 263 // Destroy should fail, still held 264 if err := sema.Destroy(); err != ErrSemaphoreHeld { 265 t.Fatalf("err: %v", err) 266 } 267 268 err = sema.Release() 269 if err != nil { 270 t.Fatalf("err: %v", err) 271 } 272 273 // Destroy should fail, still in use 274 if err := sema.Destroy(); err != ErrSemaphoreInUse { 275 t.Fatalf("err: %v", err) 276 } 277 278 err = sema2.Release() 279 if err != nil { 280 t.Fatalf("err: %v", err) 281 } 282 283 // Destroy should work 284 if err := sema.Destroy(); err != nil { 285 t.Fatalf("err: %v", err) 286 } 287 288 // Destroy should work 289 if err := sema2.Destroy(); err != nil { 290 t.Fatalf("err: %v", err) 291 } 292 } 293 294 func TestAPI_SemaphoreConflict(t *testing.T) { 295 t.Parallel() 296 c, s := makeClient(t) 297 defer s.Stop() 298 299 lock, session := createTestLock(t, c, "test/sema/.lock") 300 defer session.Destroy(lock.opts.Session, nil) 301 302 // Should work 303 leaderCh, err := lock.Lock(nil) 304 if err != nil { 305 t.Fatalf("err: %v", err) 306 } 307 if leaderCh == nil { 308 t.Fatalf("not leader") 309 } 310 defer lock.Unlock() 311 312 sema, session := createTestSemaphore(t, c, "test/sema/", 2) 313 defer session.Destroy(sema.opts.Session, nil) 314 315 // Should conflict with lock 316 _, err = sema.Acquire(nil) 317 if err != ErrSemaphoreConflict { 318 t.Fatalf("err: %v", err) 319 } 320 321 // Should conflict with lock 322 err = sema.Destroy() 323 if err != ErrSemaphoreConflict { 324 t.Fatalf("err: %v", err) 325 } 326 } 327 328 func TestAPI_SemaphoreMonitorRetry(t *testing.T) { 329 t.Parallel() 330 raw, s := makeClient(t) 331 defer s.Stop() 332 333 s.WaitForSerfCheck(t) 334 335 // Set up a server that always responds with 500 errors. 336 failer := func(w http.ResponseWriter, req *http.Request) { 337 w.WriteHeader(500) 338 } 339 outage := httptest.NewServer(http.HandlerFunc(failer)) 340 defer outage.Close() 341 342 // Set up a reverse proxy that will send some requests to the 343 // 500 server and pass everything else through to the real Consul 344 // server. 345 var mutex sync.Mutex 346 errors := 0 347 director := func(req *http.Request) { 348 mutex.Lock() 349 defer mutex.Unlock() 350 351 req.URL.Scheme = "http" 352 if errors > 0 && req.Method == "GET" && strings.Contains(req.URL.Path, "/v1/kv/test/sema/.lock") { 353 req.URL.Host = outage.URL[7:] // Strip off "http://". 354 errors-- 355 } else { 356 req.URL.Host = raw.config.Address 357 } 358 } 359 proxy := httptest.NewServer(&httputil.ReverseProxy{Director: director}) 360 defer proxy.Close() 361 362 // Make another client that points at the proxy instead of the real 363 // Consul server. 364 config := raw.config 365 config.Address = proxy.URL[7:] // Strip off "http://". 366 c, err := NewClient(&config) 367 if err != nil { 368 t.Fatalf("err: %v", err) 369 } 370 371 // Set up a lock with retries enabled. 372 opts := &SemaphoreOptions{ 373 Prefix: "test/sema/.lock", 374 Limit: 2, 375 SessionTTL: "60s", 376 MonitorRetries: 3, 377 } 378 sema, err := c.SemaphoreOpts(opts) 379 if err != nil { 380 t.Fatalf("err: %v", err) 381 } 382 383 // Make sure the default got set. 384 if sema.opts.MonitorRetryTime != DefaultMonitorRetryTime { 385 t.Fatalf("bad: %d", sema.opts.MonitorRetryTime) 386 } 387 388 // Now set a custom time for the test. 389 opts.MonitorRetryTime = 250 * time.Millisecond 390 sema, err = c.SemaphoreOpts(opts) 391 if err != nil { 392 t.Fatalf("err: %v", err) 393 } 394 if sema.opts.MonitorRetryTime != 250*time.Millisecond { 395 t.Fatalf("bad: %d", sema.opts.MonitorRetryTime) 396 } 397 398 // Should get the lock. 399 ch, err := sema.Acquire(nil) 400 if err != nil { 401 t.Fatalf("err: %v", err) 402 } 403 if ch == nil { 404 t.Fatalf("didn't acquire") 405 } 406 407 // Take the semaphore using the raw client to force the monitor to wake 408 // up and check the lock again. This time we will return errors for some 409 // of the responses. 410 mutex.Lock() 411 errors = 2 412 mutex.Unlock() 413 another, err := raw.SemaphoreOpts(opts) 414 if err != nil { 415 t.Fatalf("err: %v", err) 416 } 417 if _, err := another.Acquire(nil); err != nil { 418 t.Fatalf("err: %v", err) 419 } 420 time.Sleep(5 * opts.MonitorRetryTime) 421 422 // Should still have the semaphore. 423 select { 424 case <-ch: 425 t.Fatalf("lost the semaphore") 426 default: 427 } 428 429 // Now return an overwhelming number of errors, using the raw client to 430 // poke the key and get the monitor to run again. 431 mutex.Lock() 432 errors = 10 433 mutex.Unlock() 434 if err := another.Release(); err != nil { 435 t.Fatalf("err: %v", err) 436 } 437 time.Sleep(5 * opts.MonitorRetryTime) 438 439 // Should lose the semaphore. 440 select { 441 case <-ch: 442 case <-time.After(time.Second): 443 t.Fatalf("should not have the semaphore") 444 } 445 } 446 447 func TestAPI_SemaphoreOneShot(t *testing.T) { 448 t.Parallel() 449 c, s := makeClient(t) 450 defer s.Stop() 451 452 s.WaitForSerfCheck(t) 453 454 // Set up a semaphore as a one-shot. 455 opts := &SemaphoreOptions{ 456 Prefix: "test/sema/.lock", 457 Limit: 2, 458 SemaphoreTryOnce: true, 459 } 460 sema, err := c.SemaphoreOpts(opts) 461 if err != nil { 462 t.Fatalf("err: %v", err) 463 } 464 465 // Make sure the default got set. 466 if sema.opts.SemaphoreWaitTime != DefaultSemaphoreWaitTime { 467 t.Fatalf("bad: %d", sema.opts.SemaphoreWaitTime) 468 } 469 470 // Now set a custom time for the test. 471 opts.SemaphoreWaitTime = 250 * time.Millisecond 472 sema, err = c.SemaphoreOpts(opts) 473 if err != nil { 474 t.Fatalf("err: %v", err) 475 } 476 if sema.opts.SemaphoreWaitTime != 250*time.Millisecond { 477 t.Fatalf("bad: %d", sema.opts.SemaphoreWaitTime) 478 } 479 480 // Should acquire the semaphore. 481 ch, err := sema.Acquire(nil) 482 if err != nil { 483 t.Fatalf("err: %v", err) 484 } 485 if ch == nil { 486 t.Fatalf("should have acquired the semaphore") 487 } 488 489 // Try with another session. 490 another, err := c.SemaphoreOpts(opts) 491 if err != nil { 492 t.Fatalf("err: %v", err) 493 } 494 ch, err = another.Acquire(nil) 495 if err != nil { 496 t.Fatalf("err: %v", err) 497 } 498 if ch == nil { 499 t.Fatalf("should have acquired the semaphore") 500 } 501 502 // Try with a third one that shouldn't get it. 503 contender, err := c.SemaphoreOpts(opts) 504 if err != nil { 505 t.Fatalf("err: %v", err) 506 } 507 start := time.Now() 508 ch, err = contender.Acquire(nil) 509 if err != nil { 510 t.Fatalf("err: %v", err) 511 } 512 if ch != nil { 513 t.Fatalf("should not have acquired the semaphore") 514 } 515 diff := time.Since(start) 516 if diff < contender.opts.SemaphoreWaitTime { 517 t.Fatalf("time out of bounds: %9.6f", diff.Seconds()) 518 } 519 520 // Give up a slot and make sure the third one can get it. 521 if err := another.Release(); err != nil { 522 t.Fatalf("err: %v", err) 523 } 524 ch, err = contender.Acquire(nil) 525 if err != nil { 526 t.Fatalf("err: %v", err) 527 } 528 if ch == nil { 529 t.Fatalf("should have acquired the semaphore") 530 } 531 }