github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/container_repository_test.go (about) 1 package db_test 2 3 import ( 4 "time" 5 6 sq "github.com/Masterminds/squirrel" 7 "github.com/pf-qiu/concourse/v6/atc" 8 "github.com/pf-qiu/concourse/v6/atc/db" 9 "github.com/lib/pq" 10 11 . "github.com/onsi/ginkgo" 12 . "github.com/onsi/gomega" 13 ) 14 15 var _ = Describe("ContainerRepository", func() { 16 Describe("FindOrphanedContainers", func() { 17 Describe("check containers", func() { 18 var ( 19 creatingContainer db.CreatingContainer 20 resourceConfig db.ResourceConfig 21 ) 22 23 expiries := db.ContainerOwnerExpiries{ 24 Min: 5 * time.Minute, 25 Max: 1 * time.Hour, 26 } 27 28 BeforeEach(func() { 29 var err error 30 resourceConfig, err = resourceConfigFactory.FindOrCreateResourceConfig( 31 "some-base-resource-type", 32 atc.Source{"some": "source"}, 33 atc.VersionedResourceTypes{}, 34 ) 35 Expect(err).NotTo(HaveOccurred()) 36 37 creatingContainer, err = defaultWorker.CreateContainer( 38 db.NewResourceConfigCheckSessionContainerOwner( 39 resourceConfig.ID(), 40 resourceConfig.OriginBaseResourceType().ID, 41 expiries, 42 ), 43 fullMetadata, 44 ) 45 Expect(err).NotTo(HaveOccurred()) 46 }) 47 48 JustBeforeEach(func() { 49 err := resourceConfigCheckSessionLifecycle.CleanExpiredResourceConfigCheckSessions() 50 Expect(err).NotTo(HaveOccurred()) 51 }) 52 53 Context("when check container best if use by date is expired", func() { 54 BeforeEach(func() { 55 var rccsID int 56 err := psql.Select("id").From("resource_config_check_sessions"). 57 Where(sq.Eq{"resource_config_id": resourceConfig.ID()}).RunWith(dbConn).QueryRow().Scan(&rccsID) 58 59 _, err = psql.Update("resource_config_check_sessions"). 60 Set("expires_at", sq.Expr("NOW() - '1 second'::INTERVAL")). 61 Where(sq.Eq{"id": rccsID}). 62 RunWith(dbConn).Exec() 63 Expect(err).NotTo(HaveOccurred()) 64 }) 65 66 Context("when container is creating", func() { 67 It("finds the container for deletion", func() { 68 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 69 Expect(err).NotTo(HaveOccurred()) 70 71 Expect(creatingContainers).To(HaveLen(1)) 72 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 73 Expect(createdContainers).To(BeEmpty()) 74 Expect(destroyingContainers).To(BeEmpty()) 75 }) 76 }) 77 78 Context("when container is created", func() { 79 BeforeEach(func() { 80 _, err := creatingContainer.Created() 81 Expect(err).NotTo(HaveOccurred()) 82 }) 83 84 It("finds the container for deletion", func() { 85 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 86 Expect(err).NotTo(HaveOccurred()) 87 88 Expect(creatingContainers).To(BeEmpty()) 89 Expect(createdContainers).To(HaveLen(1)) 90 Expect(createdContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 91 Expect(destroyingContainers).To(BeEmpty()) 92 }) 93 }) 94 95 Context("when container is destroying", func() { 96 BeforeEach(func() { 97 createdContainer, err := creatingContainer.Created() 98 Expect(err).NotTo(HaveOccurred()) 99 _, err = createdContainer.Destroying() 100 Expect(err).NotTo(HaveOccurred()) 101 }) 102 103 It("finds the container for deletion", func() { 104 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 105 Expect(err).NotTo(HaveOccurred()) 106 107 Expect(creatingContainers).To(BeEmpty()) 108 Expect(createdContainers).To(BeEmpty()) 109 Expect(destroyingContainers).To(HaveLen(1)) 110 Expect(destroyingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 111 }) 112 }) 113 }) 114 115 Context("when check container best if use by date did not expire", func() { 116 BeforeEach(func() { 117 var rccsID int 118 err := psql.Select("id").From("resource_config_check_sessions"). 119 Where(sq.Eq{"resource_config_id": resourceConfig.ID()}).RunWith(dbConn).QueryRow().Scan(&rccsID) 120 121 _, err = psql.Update("resource_config_check_sessions"). 122 Set("expires_at", sq.Expr("NOW() + '1 hour'::INTERVAL")). 123 Where(sq.Eq{"id": rccsID}). 124 RunWith(dbConn).Exec() 125 Expect(err).NotTo(HaveOccurred()) 126 }) 127 128 It("does not find the container for deletion", func() { 129 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 130 Expect(err).NotTo(HaveOccurred()) 131 132 Expect(creatingContainers).To(BeEmpty()) 133 Expect(createdContainers).To(BeEmpty()) 134 Expect(destroyingContainers).To(BeEmpty()) 135 }) 136 }) 137 138 Context("when resource configs are cleaned up", func() { 139 BeforeEach(func() { 140 _, err := psql.Delete("resource_config_check_sessions"). 141 RunWith(dbConn).Exec() 142 Expect(err).NotTo(HaveOccurred()) 143 144 err = resourceConfigFactory.CleanUnreferencedConfigs(0) 145 Expect(err).NotTo(HaveOccurred()) 146 }) 147 148 It("finds the container for deletion", func() { 149 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 150 Expect(err).NotTo(HaveOccurred()) 151 152 Expect(creatingContainers).To(HaveLen(1)) 153 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 154 Expect(createdContainers).To(BeEmpty()) 155 Expect(destroyingContainers).To(BeEmpty()) 156 }) 157 }) 158 159 Context("when the worker base resource type has a new version", func() { 160 BeforeEach(func() { 161 var err error 162 newlyUpdatedWorker := defaultWorkerPayload 163 newlyUpdatedResource := defaultWorkerPayload.ResourceTypes[0] 164 newlyUpdatedResource.Version = newlyUpdatedResource.Version + "-new" 165 newlyUpdatedWorker.ResourceTypes = []atc.WorkerResourceType{newlyUpdatedResource} 166 167 defaultWorker, err = workerFactory.SaveWorker(newlyUpdatedWorker, 0) 168 Expect(err).NotTo(HaveOccurred()) 169 }) 170 171 It("finds the container for deletion", func() { 172 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 173 Expect(err).NotTo(HaveOccurred()) 174 Expect(creatingContainers).To(HaveLen(1)) 175 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 176 Expect(createdContainers).To(HaveLen(0)) 177 Expect(destroyingContainers).To(BeEmpty()) 178 }) 179 }) 180 181 Context("when the same worker base resource type is saved", func() { 182 BeforeEach(func() { 183 var err error 184 sameWorker := defaultWorkerPayload 185 186 defaultWorker, err = workerFactory.SaveWorker(sameWorker, 0) 187 Expect(err).NotTo(HaveOccurred()) 188 }) 189 190 It("does not find the container for deletion", func() { 191 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 192 Expect(err).NotTo(HaveOccurred()) 193 194 Expect(creatingContainers).To(BeEmpty()) 195 Expect(createdContainers).To(BeEmpty()) 196 Expect(destroyingContainers).To(BeEmpty()) 197 }) 198 }) 199 }) 200 201 Describe("containers owned by a build", func() { 202 var ( 203 creatingContainer db.CreatingContainer 204 build db.Build 205 ) 206 207 BeforeEach(func() { 208 var err error 209 build, err = defaultJob.CreateBuild() 210 Expect(err).NotTo(HaveOccurred()) 211 212 creatingContainer, err = defaultWorker.CreateContainer( 213 db.NewBuildStepContainerOwner(build.ID(), "simple-plan", defaultTeam.ID()), 214 fullMetadata, 215 ) 216 Expect(err).NotTo(HaveOccurred()) 217 }) 218 219 Context("when the build is interceptible", func() { 220 BeforeEach(func() { 221 err := build.SetInterceptible(true) 222 Expect(err).NotTo(HaveOccurred()) 223 }) 224 225 It("does not find container for deletion", func() { 226 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 227 Expect(err).NotTo(HaveOccurred()) 228 229 Expect(creatingContainers).To(BeEmpty()) 230 Expect(createdContainers).To(BeEmpty()) 231 Expect(destroyingContainers).To(BeEmpty()) 232 }) 233 }) 234 235 Context("when the build is marked as non-interceptible", func() { 236 BeforeEach(func() { 237 err := build.SetInterceptible(false) 238 Expect(err).NotTo(HaveOccurred()) 239 }) 240 241 Context("when the container is creating", func() { 242 It("finds container for deletion", func() { 243 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 244 Expect(err).NotTo(HaveOccurred()) 245 246 Expect(creatingContainers).To(HaveLen(1)) 247 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 248 Expect(createdContainers).To(BeEmpty()) 249 Expect(destroyingContainers).To(BeEmpty()) 250 }) 251 }) 252 253 Context("when the container is created", func() { 254 BeforeEach(func() { 255 _, err := creatingContainer.Created() 256 Expect(err).NotTo(HaveOccurred()) 257 }) 258 259 It("finds container for deletion", func() { 260 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 261 Expect(err).NotTo(HaveOccurred()) 262 263 Expect(creatingContainers).To(BeEmpty()) 264 Expect(createdContainers).To(HaveLen(1)) 265 Expect(createdContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 266 Expect(destroyingContainers).To(BeEmpty()) 267 }) 268 }) 269 270 Context("when the container is destroying", func() { 271 BeforeEach(func() { 272 createdContainer, err := creatingContainer.Created() 273 Expect(err).NotTo(HaveOccurred()) 274 _, err = createdContainer.Destroying() 275 Expect(err).NotTo(HaveOccurred()) 276 }) 277 278 It("finds container for deletion", func() { 279 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 280 Expect(err).NotTo(HaveOccurred()) 281 282 Expect(creatingContainers).To(BeEmpty()) 283 Expect(createdContainers).To(BeEmpty()) 284 Expect(destroyingContainers).To(HaveLen(1)) 285 Expect(destroyingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 286 }) 287 }) 288 }) 289 290 Context("when build is deleted", func() { 291 BeforeEach(func() { 292 err := defaultPipeline.Destroy() 293 Expect(err).NotTo(HaveOccurred()) 294 }) 295 296 It("finds container for deletion", func() { 297 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 298 Expect(err).NotTo(HaveOccurred()) 299 300 Expect(creatingContainers).To(HaveLen(1)) 301 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 302 Expect(createdContainers).To(BeEmpty()) 303 Expect(destroyingContainers).To(BeEmpty()) 304 }) 305 }) 306 }) 307 308 Describe("containers for checking images for creating containers", func() { 309 var ( 310 creatingTaskContainer db.CreatingContainer 311 creatingContainer db.CreatingContainer 312 build db.Build 313 ) 314 315 BeforeEach(func() { 316 var err error 317 build, err = defaultJob.CreateBuild() 318 Expect(err).NotTo(HaveOccurred()) 319 320 creatingTaskContainer, err = defaultWorker.CreateContainer( 321 db.NewBuildStepContainerOwner(build.ID(), "simple-plan", defaultTeam.ID()), 322 fullMetadata, 323 ) 324 Expect(err).NotTo(HaveOccurred()) 325 326 creatingContainer, err = defaultWorker.CreateContainer(db.NewImageCheckContainerOwner(creatingTaskContainer, defaultTeam.ID()), fullMetadata) 327 Expect(err).NotTo(HaveOccurred()) 328 }) 329 330 Context("when the container they're for is still creating", func() { 331 It("does not find the container for deletion", func() { 332 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 333 Expect(err).NotTo(HaveOccurred()) 334 335 Expect(creatingContainers).To(BeEmpty()) 336 Expect(createdContainers).To(BeEmpty()) 337 Expect(destroyingContainers).To(BeEmpty()) 338 }) 339 }) 340 341 Context("when the container they're for is created", func() { 342 BeforeEach(func() { 343 _, err := creatingTaskContainer.Created() 344 Expect(err).ToNot(HaveOccurred()) 345 }) 346 347 It("does not find the container for deletion", func() { 348 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 349 Expect(err).NotTo(HaveOccurred()) 350 351 Expect(creatingContainers).To(HaveLen(1)) 352 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 353 Expect(createdContainers).To(BeEmpty()) 354 Expect(destroyingContainers).To(BeEmpty()) 355 }) 356 }) 357 358 Context("when the container they're for is gone", func() { 359 BeforeEach(func() { 360 _, err := creatingTaskContainer.Created() 361 Expect(err).ToNot(HaveOccurred()) 362 }) 363 364 It("finds the container for deletion", func() { 365 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 366 Expect(err).NotTo(HaveOccurred()) 367 368 Expect(creatingContainers).To(HaveLen(1)) 369 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 370 Expect(createdContainers).To(BeEmpty()) 371 Expect(destroyingContainers).To(BeEmpty()) 372 }) 373 }) 374 }) 375 376 Describe("containers for fetching images for creating containers", func() { 377 var ( 378 creatingTaskContainer db.CreatingContainer 379 creatingContainer db.CreatingContainer 380 build db.Build 381 ) 382 383 BeforeEach(func() { 384 var err error 385 build, err = defaultJob.CreateBuild() 386 Expect(err).NotTo(HaveOccurred()) 387 388 creatingTaskContainer, err = defaultWorker.CreateContainer( 389 db.NewBuildStepContainerOwner(build.ID(), "simple-plan", defaultTeam.ID()), 390 fullMetadata, 391 ) 392 Expect(err).NotTo(HaveOccurred()) 393 394 creatingContainer, err = defaultWorker.CreateContainer(db.NewImageGetContainerOwner(creatingTaskContainer, defaultTeam.ID()), fullMetadata) 395 Expect(err).NotTo(HaveOccurred()) 396 }) 397 398 Context("when the container they're for is still creating", func() { 399 It("does not find the container for deletion", func() { 400 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 401 Expect(err).NotTo(HaveOccurred()) 402 403 Expect(creatingContainers).To(BeEmpty()) 404 Expect(createdContainers).To(BeEmpty()) 405 Expect(destroyingContainers).To(BeEmpty()) 406 }) 407 }) 408 409 Context("when the container they're for is created", func() { 410 BeforeEach(func() { 411 _, err := creatingTaskContainer.Created() 412 Expect(err).ToNot(HaveOccurred()) 413 }) 414 415 It("does not find the container for deletion", func() { 416 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 417 Expect(err).NotTo(HaveOccurred()) 418 419 Expect(creatingContainers).To(HaveLen(1)) 420 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 421 Expect(createdContainers).To(BeEmpty()) 422 Expect(destroyingContainers).To(BeEmpty()) 423 }) 424 }) 425 426 Context("when the container they're for is gone", func() { 427 BeforeEach(func() { 428 _, err := creatingTaskContainer.Created() 429 Expect(err).ToNot(HaveOccurred()) 430 }) 431 432 It("finds the container for deletion", func() { 433 creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers() 434 Expect(err).NotTo(HaveOccurred()) 435 436 Expect(creatingContainers).To(HaveLen(1)) 437 Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle())) 438 Expect(createdContainers).To(BeEmpty()) 439 Expect(destroyingContainers).To(BeEmpty()) 440 }) 441 }) 442 }) 443 }) 444 445 Describe("DestroyFailedContainers", func() { 446 var failedErr error 447 var failedContainersLen int 448 449 JustBeforeEach(func() { 450 failedContainersLen, failedErr = containerRepository.DestroyFailedContainers() 451 }) 452 453 Context("when there are failed containers", func() { 454 BeforeEach(func() { 455 result, err := psql.Insert("containers"). 456 SetMap(map[string]interface{}{ 457 "state": atc.ContainerStateFailed, 458 "handle": "123-456-abc-def", 459 "worker_name": defaultWorker.Name(), 460 }).RunWith(dbConn).Exec() 461 Expect(err).ToNot(HaveOccurred()) 462 Expect(result.RowsAffected()).To(Equal(int64(1))) 463 }) 464 465 It("returns all failed containers", func() { 466 Expect(failedContainersLen).To(Equal(1)) 467 }) 468 469 It("does not return an error", func() { 470 Expect(failedErr).ToNot(HaveOccurred()) 471 }) 472 }) 473 474 Context("when there are no failed containers", func() { 475 It("returns an empty array", func() { 476 Expect(failedContainersLen).To(Equal(0)) 477 }) 478 It("does not return an error", func() { 479 Expect(failedErr).ToNot(HaveOccurred()) 480 }) 481 }) 482 483 Describe("errors", func() { 484 Context("when the query cannot be executed", func() { 485 BeforeEach(func() { 486 err := dbConn.Close() 487 Expect(err).ToNot(HaveOccurred()) 488 }) 489 AfterEach(func() { 490 dbConn = postgresRunner.OpenConn() 491 }) 492 It("returns an error", func() { 493 Expect(failedErr).To(HaveOccurred()) 494 }) 495 }) 496 }) 497 }) 498 499 Describe("FindDestroyingContainers", func() { 500 var failedErr error 501 var destroyingContainers []string 502 503 JustBeforeEach(func() { 504 destroyingContainers, failedErr = containerRepository.FindDestroyingContainers(defaultWorker.Name()) 505 }) 506 ItClosesConnection := func() { 507 It("closes the connection", func() { 508 closed := make(chan bool) 509 510 go func() { 511 _, _ = containerRepository.FindDestroyingContainers(defaultWorker.Name()) 512 closed <- true 513 }() 514 515 Eventually(closed).Should(Receive()) 516 }) 517 } 518 519 Context("when there are destroying containers", func() { 520 BeforeEach(func() { 521 result, err := psql.Insert("containers").SetMap(map[string]interface{}{ 522 "state": "destroying", 523 "handle": "123-456-abc-def", 524 "worker_name": defaultWorker.Name(), 525 }).RunWith(dbConn).Exec() 526 527 Expect(err).ToNot(HaveOccurred()) 528 Expect(result.RowsAffected()).To(Equal(int64(1))) 529 }) 530 531 It("returns all destroying containers", func() { 532 Expect(destroyingContainers).To(HaveLen(1)) 533 Expect(destroyingContainers[0]).To(Equal("123-456-abc-def")) 534 }) 535 536 It("does not return an error", func() { 537 Expect(failedErr).ToNot(HaveOccurred()) 538 }) 539 540 ItClosesConnection() 541 }) 542 543 Describe("errors", func() { 544 Context("when the query cannot be executed", func() { 545 BeforeEach(func() { 546 err := dbConn.Close() 547 Expect(err).ToNot(HaveOccurred()) 548 }) 549 550 AfterEach(func() { 551 dbConn = postgresRunner.OpenConn() 552 }) 553 554 It("returns an error", func() { 555 Expect(failedErr).To(HaveOccurred()) 556 }) 557 558 ItClosesConnection() 559 }) 560 561 Context("when there is an error iterating through the rows", func() { 562 BeforeEach(func() { 563 By("adding a row without expected values") 564 result, err := psql.Insert("containers").SetMap(map[string]interface{}{ 565 "state": "destroying", 566 "handle": "123-456-abc-def", 567 }).RunWith(dbConn).Exec() 568 569 Expect(err).ToNot(HaveOccurred()) 570 Expect(result.RowsAffected()).To(Equal(int64(1))) 571 572 }) 573 574 It("returns empty list", func() { 575 Expect(destroyingContainers).To(HaveLen(0)) 576 }) 577 578 ItClosesConnection() 579 }) 580 }) 581 }) 582 583 Describe("RemoveMissingContainers", func() { 584 var ( 585 today time.Time 586 gracePeriod time.Duration 587 rowsAffected int 588 err error 589 ) 590 591 BeforeEach(func() { 592 today = time.Now() 593 594 _, err = psql.Insert("workers").SetMap(map[string]interface{}{ 595 "name": "running-worker", 596 "state": "running", 597 }).RunWith(dbConn).Exec() 598 Expect(err).NotTo(HaveOccurred()) 599 600 _, err = psql.Insert("containers").SetMap(map[string]interface{}{ 601 "handle": "created-handle-1", 602 "state": atc.ContainerStateCreated, 603 "worker_name": "running-worker", 604 }).RunWith(dbConn).Exec() 605 Expect(err).NotTo(HaveOccurred()) 606 607 _, err = psql.Insert("containers").SetMap(map[string]interface{}{ 608 "handle": "created-handle-2", 609 "state": atc.ContainerStateCreated, 610 "worker_name": "running-worker", 611 "missing_since": today.Add(-5 * time.Minute), 612 }).RunWith(dbConn).Exec() 613 Expect(err).NotTo(HaveOccurred()) 614 615 _, err = psql.Insert("containers").SetMap(map[string]interface{}{ 616 "handle": "failed-handle-3", 617 "state": atc.ContainerStateFailed, 618 "worker_name": "running-worker", 619 "missing_since": today.Add(-5 * time.Minute), 620 }).RunWith(dbConn).Exec() 621 Expect(err).NotTo(HaveOccurred()) 622 623 _, err = psql.Insert("containers").SetMap(map[string]interface{}{ 624 "handle": "destroying-handle-4", 625 "state": atc.ContainerStateDestroying, 626 "worker_name": "running-worker", 627 "missing_since": today.Add(-10 * time.Minute), 628 }).RunWith(dbConn).Exec() 629 Expect(err).NotTo(HaveOccurred()) 630 }) 631 632 JustBeforeEach(func() { 633 rowsAffected, err = containerRepository.RemoveMissingContainers(gracePeriod) 634 }) 635 636 Context("when no created/failed containers have expired", func() { 637 BeforeEach(func() { 638 gracePeriod = 7 * time.Minute 639 }) 640 641 It("affects no containers", func() { 642 Expect(err).ToNot(HaveOccurred()) 643 Expect(rowsAffected).To(Equal(0)) 644 }) 645 }) 646 647 Context("when some created containers have expired", func() { 648 BeforeEach(func() { 649 gracePeriod = 3 * time.Minute 650 }) 651 652 It("affects the right containers and deletes created-handle-2", func() { 653 result, err := psql.Select("*").From("containers"). 654 RunWith(dbConn).Exec() 655 Expect(err).ToNot(HaveOccurred()) 656 Expect(result.RowsAffected()).To(Equal(int64(3))) 657 658 result, err = psql.Select("*").From("containers"). 659 Where(sq.Eq{"handle": "created-handle-1"}).RunWith(dbConn).Exec() 660 Expect(err).ToNot(HaveOccurred()) 661 Expect(result.RowsAffected()).To(Equal(int64(1))) 662 663 result, err = psql.Select("*").From("containers"). 664 Where(sq.Eq{"handle": "created-handle-2"}).RunWith(dbConn).Exec() 665 Expect(err).ToNot(HaveOccurred()) 666 Expect(result.RowsAffected()).To(Equal(int64(0))) 667 668 result, err = psql.Select("*").From("containers"). 669 Where(sq.Eq{"handle": "failed-handle-3"}).RunWith(dbConn).Exec() 670 Expect(err).ToNot(HaveOccurred()) 671 Expect(result.RowsAffected()).To(Equal(int64(1))) 672 673 result, err = psql.Select("*").From("containers"). 674 Where(sq.Eq{"handle": "destroying-handle-4"}).RunWith(dbConn).Exec() 675 Expect(err).ToNot(HaveOccurred()) 676 Expect(result.RowsAffected()).To(Equal(int64(1))) 677 }) 678 }) 679 680 Context("when worker is in stalled state", func() { 681 BeforeEach(func() { 682 gracePeriod = 3 * time.Minute 683 684 _, err = psql.Insert("workers").SetMap(map[string]interface{}{ 685 "name": "stalled-worker", 686 "state": "stalled", 687 }).RunWith(dbConn).Exec() 688 Expect(err).NotTo(HaveOccurred()) 689 690 _, err = psql.Insert("containers").SetMap(map[string]interface{}{ 691 "handle": "stalled-handle-5", 692 "state": atc.ContainerStateCreated, 693 "worker_name": "stalled-worker", 694 "missing_since": today.Add(-10 * time.Minute), 695 }).RunWith(dbConn).Exec() 696 Expect(err).NotTo(HaveOccurred()) 697 698 _, err = psql.Update("containers"). 699 Set("worker_name", "stalled-worker"). 700 Where(sq.Eq{"handle": "failed-handle-3"}). 701 RunWith(dbConn).Exec() 702 Expect(err).NotTo(HaveOccurred()) 703 704 _, err = psql.Update("containers"). 705 Set("missing_since", today.Add(-5*time.Minute)). 706 Where(sq.Eq{"handle": "destroying-handle-4"}). 707 RunWith(dbConn).Exec() 708 Expect(err).NotTo(HaveOccurred()) 709 }) 710 711 It("deletes containers missing for more than grace period, on running (unstalled) workers", func() { 712 Expect(err).ToNot(HaveOccurred()) 713 Expect(rowsAffected).To(Equal(1)) 714 }) 715 716 It("does not delete containers on stalled workers", func() { 717 result, err := psql.Select("*").From("containers"). 718 RunWith(dbConn).Exec() 719 Expect(err).ToNot(HaveOccurred()) 720 Expect(result.RowsAffected()).To(Equal(int64(4))) 721 722 result, err = psql.Select("*").From("containers"). 723 Where(sq.Eq{"handle": "created-handle-1"}).RunWith(dbConn).Exec() 724 Expect(err).ToNot(HaveOccurred()) 725 Expect(result.RowsAffected()).To(Equal(int64(1))) 726 727 result, err = psql.Select("*").From("containers"). 728 Where(sq.Eq{"handle": "created-handle-2"}).RunWith(dbConn).Exec() 729 Expect(err).ToNot(HaveOccurred()) 730 Expect(result.RowsAffected()).To(Equal(int64(0))) 731 732 result, err = psql.Select("*").From("containers"). 733 Where(sq.Eq{"handle": "failed-handle-3"}).RunWith(dbConn).Exec() 734 Expect(err).ToNot(HaveOccurred()) 735 Expect(result.RowsAffected()).To(Equal(int64(1))) 736 737 result, err = psql.Select("*").From("containers"). 738 Where(sq.Eq{"handle": "destroying-handle-4"}).RunWith(dbConn).Exec() 739 Expect(err).ToNot(HaveOccurred()) 740 Expect(result.RowsAffected()).To(Equal(int64(1))) 741 742 result, err = psql.Select("*").From("containers"). 743 Where(sq.Eq{"handle": "stalled-handle-5"}).RunWith(dbConn).Exec() 744 Expect(err).ToNot(HaveOccurred()) 745 Expect(result.RowsAffected()).To(Equal(int64(1))) 746 }) 747 748 }) 749 }) 750 751 Describe("RemoveDestroyingContainers", func() { 752 var failedErr error 753 var numDeleted int 754 var handles []string 755 756 JustBeforeEach(func() { 757 numDeleted, failedErr = containerRepository.RemoveDestroyingContainers(defaultWorker.Name(), handles) 758 }) 759 760 Context("when there are containers to destroy", func() { 761 762 Context("when container is in destroying state", func() { 763 BeforeEach(func() { 764 handles = []string{"some-handle1", "some-handle2"} 765 result, err := psql.Insert("containers").SetMap(map[string]interface{}{ 766 "state": atc.ContainerStateDestroying, 767 "handle": "123-456-abc-def", 768 "worker_name": defaultWorker.Name(), 769 }).RunWith(dbConn).Exec() 770 771 Expect(err).ToNot(HaveOccurred()) 772 Expect(result.RowsAffected()).To(Equal(int64(1))) 773 }) 774 It("should destroy", func() { 775 result, err := psql.Select("*").From("containers"). 776 Where(sq.Eq{"handle": "123-456-abc-def"}).RunWith(dbConn).Exec() 777 778 Expect(err).ToNot(HaveOccurred()) 779 Expect(result.RowsAffected()).To(Equal(int64(0))) 780 }) 781 It("returns the correct number of rows removed", func() { 782 Expect(numDeleted).To(Equal(1)) 783 }) 784 It("does not return an error", func() { 785 Expect(failedErr).ToNot(HaveOccurred()) 786 }) 787 }) 788 789 Context("when handles are empty list", func() { 790 BeforeEach(func() { 791 handles = []string{} 792 result, err := psql.Insert("containers").SetMap(map[string]interface{}{ 793 "state": atc.ContainerStateDestroying, 794 "handle": "123-456-abc-def", 795 "worker_name": defaultWorker.Name(), 796 }).RunWith(dbConn).Exec() 797 798 Expect(err).ToNot(HaveOccurred()) 799 Expect(result.RowsAffected()).To(Equal(int64(1))) 800 }) 801 802 It("should destroy", func() { 803 result, err := psql.Select("*").From("containers"). 804 Where(sq.Eq{"handle": "123-456-abc-def"}).RunWith(dbConn).Exec() 805 806 Expect(err).ToNot(HaveOccurred()) 807 Expect(result.RowsAffected()).To(Equal(int64(0))) 808 }) 809 810 It("returns the correct number of rows removed", func() { 811 Expect(numDeleted).To(Equal(1)) 812 }) 813 814 It("does not return an error", func() { 815 Expect(failedErr).ToNot(HaveOccurred()) 816 }) 817 }) 818 819 Context("when container is in create/creating state", func() { 820 BeforeEach(func() { 821 handles = []string{"some-handle1", "some-handle2"} 822 result, err := psql.Insert("containers").SetMap(map[string]interface{}{ 823 "state": "creating", 824 "handle": "123-456-abc-def", 825 "worker_name": defaultWorker.Name(), 826 }).RunWith(dbConn).Exec() 827 828 Expect(err).ToNot(HaveOccurred()) 829 Expect(result.RowsAffected()).To(Equal(int64(1))) 830 }) 831 It("should not destroy", func() { 832 result, err := psql.Select("*").From("containers"). 833 Where(sq.Eq{"handle": "123-456-abc-def"}).RunWith(dbConn).Exec() 834 835 Expect(err).ToNot(HaveOccurred()) 836 Expect(result.RowsAffected()).To(Equal(int64(1))) 837 }) 838 It("returns the correct number of rows removed", func() { 839 Expect(numDeleted).To(Equal(0)) 840 }) 841 It("does not return an error", func() { 842 Expect(failedErr).ToNot(HaveOccurred()) 843 }) 844 }) 845 }) 846 847 Context("when there are no containers to destroy", func() { 848 BeforeEach(func() { 849 handles = []string{"some-handle1", "some-handle2"} 850 851 result, err := psql.Insert("containers").SetMap( 852 map[string]interface{}{ 853 "state": "destroying", 854 "handle": "some-handle1", 855 "worker_name": defaultWorker.Name(), 856 }, 857 ).RunWith(dbConn).Exec() 858 Expect(err).ToNot(HaveOccurred()) 859 Expect(result.RowsAffected()).To(Equal(int64(1))) 860 861 result, err = psql.Insert("containers").SetMap( 862 map[string]interface{}{ 863 "state": "destroying", 864 "handle": "some-handle2", 865 "worker_name": defaultWorker.Name(), 866 }, 867 ).RunWith(dbConn).Exec() 868 Expect(err).ToNot(HaveOccurred()) 869 Expect(result.RowsAffected()).To(Equal(int64(1))) 870 }) 871 872 It("doesn't destroy containers that are in handles", func() { 873 result, err := psql.Select("*").From("containers"). 874 Where(sq.Eq{"handle": handles}).RunWith(dbConn).Exec() 875 876 Expect(err).ToNot(HaveOccurred()) 877 Expect(result.RowsAffected()).To(Equal(int64(2))) 878 }) 879 880 It("does not return an error", func() { 881 Expect(failedErr).ToNot(HaveOccurred()) 882 }) 883 It("returns the correct number of rows removed", func() { 884 Expect(numDeleted).To(Equal(0)) 885 }) 886 }) 887 888 Describe("errors", func() { 889 Context("when the query cannot be executed", func() { 890 BeforeEach(func() { 891 err := dbConn.Close() 892 Expect(err).ToNot(HaveOccurred()) 893 }) 894 895 AfterEach(func() { 896 dbConn = postgresRunner.OpenConn() 897 }) 898 899 It("returns an error", func() { 900 Expect(failedErr).To(HaveOccurred()) 901 }) 902 }) 903 }) 904 }) 905 906 Describe("UpdateContainersMissingSince", func() { 907 var ( 908 today time.Time 909 err error 910 handles []string 911 missingSince pq.NullTime 912 ) 913 914 BeforeEach(func() { 915 result, err := psql.Insert("containers").SetMap(map[string]interface{}{ 916 "state": atc.ContainerStateDestroying, 917 "handle": "some-handle1", 918 "worker_name": defaultWorker.Name(), 919 }).RunWith(dbConn).Exec() 920 921 Expect(err).ToNot(HaveOccurred()) 922 Expect(result.RowsAffected()).To(Equal(int64(1))) 923 924 result, err = psql.Insert("containers").SetMap(map[string]interface{}{ 925 "state": atc.ContainerStateDestroying, 926 "handle": "some-handle2", 927 "worker_name": defaultWorker.Name(), 928 }).RunWith(dbConn).Exec() 929 930 Expect(err).ToNot(HaveOccurred()) 931 Expect(result.RowsAffected()).To(Equal(int64(1))) 932 933 today = time.Date(2018, 9, 24, 0, 0, 0, 0, time.UTC) 934 935 result, err = psql.Insert("containers").SetMap(map[string]interface{}{ 936 "state": atc.ContainerStateCreated, 937 "handle": "some-handle3", 938 "worker_name": defaultWorker.Name(), 939 "missing_since": today, 940 }).RunWith(dbConn).Exec() 941 942 Expect(err).ToNot(HaveOccurred()) 943 Expect(result.RowsAffected()).To(Equal(int64(1))) 944 }) 945 946 JustBeforeEach(func() { 947 err = containerRepository.UpdateContainersMissingSince(defaultWorker.Name(), handles) 948 }) 949 950 Context("when the reported handles is a subset", func() { 951 BeforeEach(func() { 952 handles = []string{"some-handle1"} 953 }) 954 955 Context("having the containers in the creating state in the db", func() { 956 BeforeEach(func() { 957 result, err := psql.Update("containers"). 958 Where(sq.Eq{"handle": "some-handle3"}). 959 SetMap(map[string]interface{}{ 960 "state": atc.ContainerStateCreating, 961 "missing_since": nil, 962 }).RunWith(dbConn).Exec() 963 Expect(err).NotTo(HaveOccurred()) 964 Expect(result.RowsAffected()).To(Equal(int64(1))) 965 }) 966 967 It("does not mark as missing", func() { 968 err = psql.Select("missing_since").From("containers"). 969 Where(sq.Eq{"handle": "some-handle3"}).RunWith(dbConn).QueryRow(). 970 Scan(&missingSince) 971 Expect(err).ToNot(HaveOccurred()) 972 Expect(missingSince.Valid).To(BeFalse()) 973 }) 974 }) 975 976 It("should mark containers not in the subset and not already marked as missing", func() { 977 err = psql.Select("missing_since").From("containers"). 978 Where(sq.Eq{"handle": "some-handle1"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 979 Expect(err).ToNot(HaveOccurred()) 980 Expect(missingSince.Valid).To(BeFalse()) 981 982 err = psql.Select("missing_since").From("containers"). 983 Where(sq.Eq{"handle": "some-handle2"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 984 Expect(err).ToNot(HaveOccurred()) 985 Expect(missingSince.Valid).To(BeTrue()) 986 987 err = psql.Select("missing_since").From("containers"). 988 Where(sq.Eq{"handle": "some-handle3"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 989 Expect(err).ToNot(HaveOccurred()) 990 Expect(missingSince.Valid).To(BeTrue()) 991 Expect(missingSince.Time.Unix()).To(Equal(today.Unix())) 992 }) 993 994 It("does not return an error", func() { 995 Expect(err).ToNot(HaveOccurred()) 996 }) 997 }) 998 999 Context("when the reported handles is the full set", func() { 1000 BeforeEach(func() { 1001 handles = []string{"some-handle1", "some-handle2"} 1002 }) 1003 1004 It("should not update", func() { 1005 err = psql.Select("missing_since").From("containers"). 1006 Where(sq.Eq{"handle": "some-handle1"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 1007 Expect(err).ToNot(HaveOccurred()) 1008 Expect(missingSince.Valid).To(BeFalse()) 1009 1010 err = psql.Select("missing_since").From("containers"). 1011 Where(sq.Eq{"handle": "some-handle2"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 1012 Expect(err).ToNot(HaveOccurred()) 1013 Expect(missingSince.Valid).To(BeFalse()) 1014 }) 1015 1016 It("does not return an error", func() { 1017 Expect(err).ToNot(HaveOccurred()) 1018 }) 1019 }) 1020 1021 Context("when the reported handles includes a container marked as missing", func() { 1022 BeforeEach(func() { 1023 handles = []string{"some-handle1", "some-handle2", "some-handle3"} 1024 }) 1025 1026 It("should mark the previously missing container as not missing", func() { 1027 err = psql.Select("missing_since").From("containers"). 1028 Where(sq.Eq{"handle": "some-handle1"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 1029 Expect(err).ToNot(HaveOccurred()) 1030 Expect(missingSince.Valid).To(BeFalse()) 1031 1032 err = psql.Select("missing_since").From("containers"). 1033 Where(sq.Eq{"handle": "some-handle2"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 1034 Expect(err).ToNot(HaveOccurred()) 1035 Expect(missingSince.Valid).To(BeFalse()) 1036 1037 err = psql.Select("missing_since").From("containers"). 1038 Where(sq.Eq{"handle": "some-handle3"}).RunWith(dbConn).QueryRow().Scan(&missingSince) 1039 Expect(err).ToNot(HaveOccurred()) 1040 Expect(missingSince.Valid).To(BeFalse()) 1041 }) 1042 1043 It("does not return an error", func() { 1044 Expect(err).ToNot(HaveOccurred()) 1045 }) 1046 }) 1047 }) 1048 1049 Describe("DestroyUnknownContainers", func() { 1050 var ( 1051 err error 1052 workerReportedHandles []string 1053 numberUnknownContainers int 1054 ) 1055 1056 BeforeEach(func() { 1057 result, err := psql.Insert("containers").SetMap(map[string]interface{}{ 1058 "state": atc.ContainerStateDestroying, 1059 "handle": "some-handle1", 1060 "worker_name": defaultWorker.Name(), 1061 }).RunWith(dbConn).Exec() 1062 1063 Expect(err).ToNot(HaveOccurred()) 1064 Expect(result.RowsAffected()).To(Equal(int64(1))) 1065 1066 result, err = psql.Insert("containers").SetMap(map[string]interface{}{ 1067 "state": atc.ContainerStateCreated, 1068 "handle": "some-handle2", 1069 "worker_name": defaultWorker.Name(), 1070 }).RunWith(dbConn).Exec() 1071 1072 Expect(err).ToNot(HaveOccurred()) 1073 Expect(result.RowsAffected()).To(Equal(int64(1))) 1074 }) 1075 1076 JustBeforeEach(func() { 1077 numberUnknownContainers, err = containerRepository.DestroyUnknownContainers(defaultWorker.Name(), workerReportedHandles) 1078 Expect(err).ToNot(HaveOccurred()) 1079 }) 1080 1081 Context("when there are containers on the worker that are not in the db", func() { 1082 var destroyingContainerHandles []string 1083 BeforeEach(func() { 1084 workerReportedHandles = []string{"some-handle3", "some-handle4"} 1085 destroyingContainerHandles = append(workerReportedHandles, "some-handle1") 1086 }) 1087 1088 It("adds new destroying containers to the database", func() { 1089 result, err := psql.Select("handle"). 1090 From("containers"). 1091 Where(sq.Eq{"state": atc.ContainerStateDestroying}). 1092 RunWith(dbConn).Query() 1093 1094 Expect(err).ToNot(HaveOccurred()) 1095 1096 var handle string 1097 for result.Next() { 1098 err = result.Scan(&handle) 1099 Expect(err).ToNot(HaveOccurred()) 1100 Expect(handle).Should(BeElementOf(destroyingContainerHandles)) 1101 } 1102 Expect(numberUnknownContainers).To(Equal(2)) 1103 }) 1104 1105 It("does not affect containers in any other state", func() { 1106 rows, err := psql.Select("handle"). 1107 From("containers"). 1108 Where(sq.Eq{"state": atc.ContainerStateCreated}). 1109 RunWith(dbConn).Query() 1110 1111 Expect(err).ToNot(HaveOccurred()) 1112 1113 var handle string 1114 var rowsAffected int 1115 for rows.Next() { 1116 err = rows.Scan(&handle) 1117 Expect(err).ToNot(HaveOccurred()) 1118 Expect(handle).To(Equal("some-handle2")) 1119 rowsAffected++ 1120 } 1121 1122 Expect(rowsAffected).To(Equal(1)) 1123 }) 1124 }) 1125 1126 Context("when there are no unknown containers on the worker", func() { 1127 BeforeEach(func() { 1128 workerReportedHandles = []string{"some-handle1", "some-handle2"} 1129 }) 1130 1131 It("should not try to destroy anything", func() { 1132 Expect(numberUnknownContainers).To(Equal(0)) 1133 1134 rows, err := psql.Select("handle"). 1135 From("containers"). 1136 Where(sq.Eq{"state": atc.ContainerStateDestroying}). 1137 RunWith(dbConn).Query() 1138 1139 Expect(err).ToNot(HaveOccurred()) 1140 1141 var handle string 1142 var rowsAffected int 1143 for rows.Next() { 1144 err = rows.Scan(&handle) 1145 Expect(err).ToNot(HaveOccurred()) 1146 Expect(handle).To(Equal("some-handle1")) 1147 rowsAffected++ 1148 } 1149 1150 Expect(rowsAffected).To(Equal(1)) 1151 }) 1152 }) 1153 }) 1154 })