github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/container_repository_test.go (about)

     1  package db_test
     2  
     3  import (
     4  	"time"
     5  
     6  	sq "github.com/Masterminds/squirrel"
     7  	"github.com/pf-qiu/concourse/v6/atc"
     8  	"github.com/pf-qiu/concourse/v6/atc/db"
     9  	"github.com/lib/pq"
    10  
    11  	. "github.com/onsi/ginkgo"
    12  	. "github.com/onsi/gomega"
    13  )
    14  
    15  var _ = Describe("ContainerRepository", func() {
    16  	Describe("FindOrphanedContainers", func() {
    17  		Describe("check containers", func() {
    18  			var (
    19  				creatingContainer db.CreatingContainer
    20  				resourceConfig    db.ResourceConfig
    21  			)
    22  
    23  			expiries := db.ContainerOwnerExpiries{
    24  				Min: 5 * time.Minute,
    25  				Max: 1 * time.Hour,
    26  			}
    27  
    28  			BeforeEach(func() {
    29  				var err error
    30  				resourceConfig, err = resourceConfigFactory.FindOrCreateResourceConfig(
    31  					"some-base-resource-type",
    32  					atc.Source{"some": "source"},
    33  					atc.VersionedResourceTypes{},
    34  				)
    35  				Expect(err).NotTo(HaveOccurred())
    36  
    37  				creatingContainer, err = defaultWorker.CreateContainer(
    38  					db.NewResourceConfigCheckSessionContainerOwner(
    39  						resourceConfig.ID(),
    40  						resourceConfig.OriginBaseResourceType().ID,
    41  						expiries,
    42  					),
    43  					fullMetadata,
    44  				)
    45  				Expect(err).NotTo(HaveOccurred())
    46  			})
    47  
    48  			JustBeforeEach(func() {
    49  				err := resourceConfigCheckSessionLifecycle.CleanExpiredResourceConfigCheckSessions()
    50  				Expect(err).NotTo(HaveOccurred())
    51  			})
    52  
    53  			Context("when check container best if use by date is expired", func() {
    54  				BeforeEach(func() {
    55  					var rccsID int
    56  					err := psql.Select("id").From("resource_config_check_sessions").
    57  						Where(sq.Eq{"resource_config_id": resourceConfig.ID()}).RunWith(dbConn).QueryRow().Scan(&rccsID)
    58  
    59  					_, err = psql.Update("resource_config_check_sessions").
    60  						Set("expires_at", sq.Expr("NOW() - '1 second'::INTERVAL")).
    61  						Where(sq.Eq{"id": rccsID}).
    62  						RunWith(dbConn).Exec()
    63  					Expect(err).NotTo(HaveOccurred())
    64  				})
    65  
    66  				Context("when container is creating", func() {
    67  					It("finds the container for deletion", func() {
    68  						creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
    69  						Expect(err).NotTo(HaveOccurred())
    70  
    71  						Expect(creatingContainers).To(HaveLen(1))
    72  						Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
    73  						Expect(createdContainers).To(BeEmpty())
    74  						Expect(destroyingContainers).To(BeEmpty())
    75  					})
    76  				})
    77  
    78  				Context("when container is created", func() {
    79  					BeforeEach(func() {
    80  						_, err := creatingContainer.Created()
    81  						Expect(err).NotTo(HaveOccurred())
    82  					})
    83  
    84  					It("finds the container for deletion", func() {
    85  						creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
    86  						Expect(err).NotTo(HaveOccurred())
    87  
    88  						Expect(creatingContainers).To(BeEmpty())
    89  						Expect(createdContainers).To(HaveLen(1))
    90  						Expect(createdContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
    91  						Expect(destroyingContainers).To(BeEmpty())
    92  					})
    93  				})
    94  
    95  				Context("when container is destroying", func() {
    96  					BeforeEach(func() {
    97  						createdContainer, err := creatingContainer.Created()
    98  						Expect(err).NotTo(HaveOccurred())
    99  						_, err = createdContainer.Destroying()
   100  						Expect(err).NotTo(HaveOccurred())
   101  					})
   102  
   103  					It("finds the container for deletion", func() {
   104  						creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   105  						Expect(err).NotTo(HaveOccurred())
   106  
   107  						Expect(creatingContainers).To(BeEmpty())
   108  						Expect(createdContainers).To(BeEmpty())
   109  						Expect(destroyingContainers).To(HaveLen(1))
   110  						Expect(destroyingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   111  					})
   112  				})
   113  			})
   114  
   115  			Context("when check container best if use by date did not expire", func() {
   116  				BeforeEach(func() {
   117  					var rccsID int
   118  					err := psql.Select("id").From("resource_config_check_sessions").
   119  						Where(sq.Eq{"resource_config_id": resourceConfig.ID()}).RunWith(dbConn).QueryRow().Scan(&rccsID)
   120  
   121  					_, err = psql.Update("resource_config_check_sessions").
   122  						Set("expires_at", sq.Expr("NOW() + '1 hour'::INTERVAL")).
   123  						Where(sq.Eq{"id": rccsID}).
   124  						RunWith(dbConn).Exec()
   125  					Expect(err).NotTo(HaveOccurred())
   126  				})
   127  
   128  				It("does not find the container for deletion", func() {
   129  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   130  					Expect(err).NotTo(HaveOccurred())
   131  
   132  					Expect(creatingContainers).To(BeEmpty())
   133  					Expect(createdContainers).To(BeEmpty())
   134  					Expect(destroyingContainers).To(BeEmpty())
   135  				})
   136  			})
   137  
   138  			Context("when resource configs are cleaned up", func() {
   139  				BeforeEach(func() {
   140  					_, err := psql.Delete("resource_config_check_sessions").
   141  						RunWith(dbConn).Exec()
   142  					Expect(err).NotTo(HaveOccurred())
   143  
   144  					err = resourceConfigFactory.CleanUnreferencedConfigs(0)
   145  					Expect(err).NotTo(HaveOccurred())
   146  				})
   147  
   148  				It("finds the container for deletion", func() {
   149  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   150  					Expect(err).NotTo(HaveOccurred())
   151  
   152  					Expect(creatingContainers).To(HaveLen(1))
   153  					Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   154  					Expect(createdContainers).To(BeEmpty())
   155  					Expect(destroyingContainers).To(BeEmpty())
   156  				})
   157  			})
   158  
   159  			Context("when the worker base resource type has a new version", func() {
   160  				BeforeEach(func() {
   161  					var err error
   162  					newlyUpdatedWorker := defaultWorkerPayload
   163  					newlyUpdatedResource := defaultWorkerPayload.ResourceTypes[0]
   164  					newlyUpdatedResource.Version = newlyUpdatedResource.Version + "-new"
   165  					newlyUpdatedWorker.ResourceTypes = []atc.WorkerResourceType{newlyUpdatedResource}
   166  
   167  					defaultWorker, err = workerFactory.SaveWorker(newlyUpdatedWorker, 0)
   168  					Expect(err).NotTo(HaveOccurred())
   169  				})
   170  
   171  				It("finds the container for deletion", func() {
   172  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   173  					Expect(err).NotTo(HaveOccurred())
   174  					Expect(creatingContainers).To(HaveLen(1))
   175  					Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   176  					Expect(createdContainers).To(HaveLen(0))
   177  					Expect(destroyingContainers).To(BeEmpty())
   178  				})
   179  			})
   180  
   181  			Context("when the same worker base resource type is saved", func() {
   182  				BeforeEach(func() {
   183  					var err error
   184  					sameWorker := defaultWorkerPayload
   185  
   186  					defaultWorker, err = workerFactory.SaveWorker(sameWorker, 0)
   187  					Expect(err).NotTo(HaveOccurred())
   188  				})
   189  
   190  				It("does not find the container for deletion", func() {
   191  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   192  					Expect(err).NotTo(HaveOccurred())
   193  
   194  					Expect(creatingContainers).To(BeEmpty())
   195  					Expect(createdContainers).To(BeEmpty())
   196  					Expect(destroyingContainers).To(BeEmpty())
   197  				})
   198  			})
   199  		})
   200  
   201  		Describe("containers owned by a build", func() {
   202  			var (
   203  				creatingContainer db.CreatingContainer
   204  				build             db.Build
   205  			)
   206  
   207  			BeforeEach(func() {
   208  				var err error
   209  				build, err = defaultJob.CreateBuild()
   210  				Expect(err).NotTo(HaveOccurred())
   211  
   212  				creatingContainer, err = defaultWorker.CreateContainer(
   213  					db.NewBuildStepContainerOwner(build.ID(), "simple-plan", defaultTeam.ID()),
   214  					fullMetadata,
   215  				)
   216  				Expect(err).NotTo(HaveOccurred())
   217  			})
   218  
   219  			Context("when the build is interceptible", func() {
   220  				BeforeEach(func() {
   221  					err := build.SetInterceptible(true)
   222  					Expect(err).NotTo(HaveOccurred())
   223  				})
   224  
   225  				It("does not find container for deletion", func() {
   226  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   227  					Expect(err).NotTo(HaveOccurred())
   228  
   229  					Expect(creatingContainers).To(BeEmpty())
   230  					Expect(createdContainers).To(BeEmpty())
   231  					Expect(destroyingContainers).To(BeEmpty())
   232  				})
   233  			})
   234  
   235  			Context("when the build is marked as non-interceptible", func() {
   236  				BeforeEach(func() {
   237  					err := build.SetInterceptible(false)
   238  					Expect(err).NotTo(HaveOccurred())
   239  				})
   240  
   241  				Context("when the container is creating", func() {
   242  					It("finds container for deletion", func() {
   243  						creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   244  						Expect(err).NotTo(HaveOccurred())
   245  
   246  						Expect(creatingContainers).To(HaveLen(1))
   247  						Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   248  						Expect(createdContainers).To(BeEmpty())
   249  						Expect(destroyingContainers).To(BeEmpty())
   250  					})
   251  				})
   252  
   253  				Context("when the container is created", func() {
   254  					BeforeEach(func() {
   255  						_, err := creatingContainer.Created()
   256  						Expect(err).NotTo(HaveOccurred())
   257  					})
   258  
   259  					It("finds container for deletion", func() {
   260  						creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   261  						Expect(err).NotTo(HaveOccurred())
   262  
   263  						Expect(creatingContainers).To(BeEmpty())
   264  						Expect(createdContainers).To(HaveLen(1))
   265  						Expect(createdContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   266  						Expect(destroyingContainers).To(BeEmpty())
   267  					})
   268  				})
   269  
   270  				Context("when the container is destroying", func() {
   271  					BeforeEach(func() {
   272  						createdContainer, err := creatingContainer.Created()
   273  						Expect(err).NotTo(HaveOccurred())
   274  						_, err = createdContainer.Destroying()
   275  						Expect(err).NotTo(HaveOccurred())
   276  					})
   277  
   278  					It("finds container for deletion", func() {
   279  						creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   280  						Expect(err).NotTo(HaveOccurred())
   281  
   282  						Expect(creatingContainers).To(BeEmpty())
   283  						Expect(createdContainers).To(BeEmpty())
   284  						Expect(destroyingContainers).To(HaveLen(1))
   285  						Expect(destroyingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   286  					})
   287  				})
   288  			})
   289  
   290  			Context("when build is deleted", func() {
   291  				BeforeEach(func() {
   292  					err := defaultPipeline.Destroy()
   293  					Expect(err).NotTo(HaveOccurred())
   294  				})
   295  
   296  				It("finds container for deletion", func() {
   297  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   298  					Expect(err).NotTo(HaveOccurred())
   299  
   300  					Expect(creatingContainers).To(HaveLen(1))
   301  					Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   302  					Expect(createdContainers).To(BeEmpty())
   303  					Expect(destroyingContainers).To(BeEmpty())
   304  				})
   305  			})
   306  		})
   307  
   308  		Describe("containers for checking images for creating containers", func() {
   309  			var (
   310  				creatingTaskContainer db.CreatingContainer
   311  				creatingContainer     db.CreatingContainer
   312  				build                 db.Build
   313  			)
   314  
   315  			BeforeEach(func() {
   316  				var err error
   317  				build, err = defaultJob.CreateBuild()
   318  				Expect(err).NotTo(HaveOccurred())
   319  
   320  				creatingTaskContainer, err = defaultWorker.CreateContainer(
   321  					db.NewBuildStepContainerOwner(build.ID(), "simple-plan", defaultTeam.ID()),
   322  					fullMetadata,
   323  				)
   324  				Expect(err).NotTo(HaveOccurred())
   325  
   326  				creatingContainer, err = defaultWorker.CreateContainer(db.NewImageCheckContainerOwner(creatingTaskContainer, defaultTeam.ID()), fullMetadata)
   327  				Expect(err).NotTo(HaveOccurred())
   328  			})
   329  
   330  			Context("when the container they're for is still creating", func() {
   331  				It("does not find the container for deletion", func() {
   332  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   333  					Expect(err).NotTo(HaveOccurred())
   334  
   335  					Expect(creatingContainers).To(BeEmpty())
   336  					Expect(createdContainers).To(BeEmpty())
   337  					Expect(destroyingContainers).To(BeEmpty())
   338  				})
   339  			})
   340  
   341  			Context("when the container they're for is created", func() {
   342  				BeforeEach(func() {
   343  					_, err := creatingTaskContainer.Created()
   344  					Expect(err).ToNot(HaveOccurred())
   345  				})
   346  
   347  				It("does not find the container for deletion", func() {
   348  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   349  					Expect(err).NotTo(HaveOccurred())
   350  
   351  					Expect(creatingContainers).To(HaveLen(1))
   352  					Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   353  					Expect(createdContainers).To(BeEmpty())
   354  					Expect(destroyingContainers).To(BeEmpty())
   355  				})
   356  			})
   357  
   358  			Context("when the container they're for is gone", func() {
   359  				BeforeEach(func() {
   360  					_, err := creatingTaskContainer.Created()
   361  					Expect(err).ToNot(HaveOccurred())
   362  				})
   363  
   364  				It("finds the container for deletion", func() {
   365  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   366  					Expect(err).NotTo(HaveOccurred())
   367  
   368  					Expect(creatingContainers).To(HaveLen(1))
   369  					Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   370  					Expect(createdContainers).To(BeEmpty())
   371  					Expect(destroyingContainers).To(BeEmpty())
   372  				})
   373  			})
   374  		})
   375  
   376  		Describe("containers for fetching images for creating containers", func() {
   377  			var (
   378  				creatingTaskContainer db.CreatingContainer
   379  				creatingContainer     db.CreatingContainer
   380  				build                 db.Build
   381  			)
   382  
   383  			BeforeEach(func() {
   384  				var err error
   385  				build, err = defaultJob.CreateBuild()
   386  				Expect(err).NotTo(HaveOccurred())
   387  
   388  				creatingTaskContainer, err = defaultWorker.CreateContainer(
   389  					db.NewBuildStepContainerOwner(build.ID(), "simple-plan", defaultTeam.ID()),
   390  					fullMetadata,
   391  				)
   392  				Expect(err).NotTo(HaveOccurred())
   393  
   394  				creatingContainer, err = defaultWorker.CreateContainer(db.NewImageGetContainerOwner(creatingTaskContainer, defaultTeam.ID()), fullMetadata)
   395  				Expect(err).NotTo(HaveOccurred())
   396  			})
   397  
   398  			Context("when the container they're for is still creating", func() {
   399  				It("does not find the container for deletion", func() {
   400  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   401  					Expect(err).NotTo(HaveOccurred())
   402  
   403  					Expect(creatingContainers).To(BeEmpty())
   404  					Expect(createdContainers).To(BeEmpty())
   405  					Expect(destroyingContainers).To(BeEmpty())
   406  				})
   407  			})
   408  
   409  			Context("when the container they're for is created", func() {
   410  				BeforeEach(func() {
   411  					_, err := creatingTaskContainer.Created()
   412  					Expect(err).ToNot(HaveOccurred())
   413  				})
   414  
   415  				It("does not find the container for deletion", func() {
   416  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   417  					Expect(err).NotTo(HaveOccurred())
   418  
   419  					Expect(creatingContainers).To(HaveLen(1))
   420  					Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   421  					Expect(createdContainers).To(BeEmpty())
   422  					Expect(destroyingContainers).To(BeEmpty())
   423  				})
   424  			})
   425  
   426  			Context("when the container they're for is gone", func() {
   427  				BeforeEach(func() {
   428  					_, err := creatingTaskContainer.Created()
   429  					Expect(err).ToNot(HaveOccurred())
   430  				})
   431  
   432  				It("finds the container for deletion", func() {
   433  					creatingContainers, createdContainers, destroyingContainers, err := containerRepository.FindOrphanedContainers()
   434  					Expect(err).NotTo(HaveOccurred())
   435  
   436  					Expect(creatingContainers).To(HaveLen(1))
   437  					Expect(creatingContainers[0].Handle()).To(Equal(creatingContainer.Handle()))
   438  					Expect(createdContainers).To(BeEmpty())
   439  					Expect(destroyingContainers).To(BeEmpty())
   440  				})
   441  			})
   442  		})
   443  	})
   444  
   445  	Describe("DestroyFailedContainers", func() {
   446  		var failedErr error
   447  		var failedContainersLen int
   448  
   449  		JustBeforeEach(func() {
   450  			failedContainersLen, failedErr = containerRepository.DestroyFailedContainers()
   451  		})
   452  
   453  		Context("when there are failed containers", func() {
   454  			BeforeEach(func() {
   455  				result, err := psql.Insert("containers").
   456  					SetMap(map[string]interface{}{
   457  						"state":       atc.ContainerStateFailed,
   458  						"handle":      "123-456-abc-def",
   459  						"worker_name": defaultWorker.Name(),
   460  					}).RunWith(dbConn).Exec()
   461  				Expect(err).ToNot(HaveOccurred())
   462  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   463  			})
   464  
   465  			It("returns all failed containers", func() {
   466  				Expect(failedContainersLen).To(Equal(1))
   467  			})
   468  
   469  			It("does not return an error", func() {
   470  				Expect(failedErr).ToNot(HaveOccurred())
   471  			})
   472  		})
   473  
   474  		Context("when there are no failed containers", func() {
   475  			It("returns an empty array", func() {
   476  				Expect(failedContainersLen).To(Equal(0))
   477  			})
   478  			It("does not return an error", func() {
   479  				Expect(failedErr).ToNot(HaveOccurred())
   480  			})
   481  		})
   482  
   483  		Describe("errors", func() {
   484  			Context("when the query cannot be executed", func() {
   485  				BeforeEach(func() {
   486  					err := dbConn.Close()
   487  					Expect(err).ToNot(HaveOccurred())
   488  				})
   489  				AfterEach(func() {
   490  					dbConn = postgresRunner.OpenConn()
   491  				})
   492  				It("returns an error", func() {
   493  					Expect(failedErr).To(HaveOccurred())
   494  				})
   495  			})
   496  		})
   497  	})
   498  
   499  	Describe("FindDestroyingContainers", func() {
   500  		var failedErr error
   501  		var destroyingContainers []string
   502  
   503  		JustBeforeEach(func() {
   504  			destroyingContainers, failedErr = containerRepository.FindDestroyingContainers(defaultWorker.Name())
   505  		})
   506  		ItClosesConnection := func() {
   507  			It("closes the connection", func() {
   508  				closed := make(chan bool)
   509  
   510  				go func() {
   511  					_, _ = containerRepository.FindDestroyingContainers(defaultWorker.Name())
   512  					closed <- true
   513  				}()
   514  
   515  				Eventually(closed).Should(Receive())
   516  			})
   517  		}
   518  
   519  		Context("when there are destroying containers", func() {
   520  			BeforeEach(func() {
   521  				result, err := psql.Insert("containers").SetMap(map[string]interface{}{
   522  					"state":       "destroying",
   523  					"handle":      "123-456-abc-def",
   524  					"worker_name": defaultWorker.Name(),
   525  				}).RunWith(dbConn).Exec()
   526  
   527  				Expect(err).ToNot(HaveOccurred())
   528  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   529  			})
   530  
   531  			It("returns all destroying containers", func() {
   532  				Expect(destroyingContainers).To(HaveLen(1))
   533  				Expect(destroyingContainers[0]).To(Equal("123-456-abc-def"))
   534  			})
   535  
   536  			It("does not return an error", func() {
   537  				Expect(failedErr).ToNot(HaveOccurred())
   538  			})
   539  
   540  			ItClosesConnection()
   541  		})
   542  
   543  		Describe("errors", func() {
   544  			Context("when the query cannot be executed", func() {
   545  				BeforeEach(func() {
   546  					err := dbConn.Close()
   547  					Expect(err).ToNot(HaveOccurred())
   548  				})
   549  
   550  				AfterEach(func() {
   551  					dbConn = postgresRunner.OpenConn()
   552  				})
   553  
   554  				It("returns an error", func() {
   555  					Expect(failedErr).To(HaveOccurred())
   556  				})
   557  
   558  				ItClosesConnection()
   559  			})
   560  
   561  			Context("when there is an error iterating through the rows", func() {
   562  				BeforeEach(func() {
   563  					By("adding a row without expected values")
   564  					result, err := psql.Insert("containers").SetMap(map[string]interface{}{
   565  						"state":  "destroying",
   566  						"handle": "123-456-abc-def",
   567  					}).RunWith(dbConn).Exec()
   568  
   569  					Expect(err).ToNot(HaveOccurred())
   570  					Expect(result.RowsAffected()).To(Equal(int64(1)))
   571  
   572  				})
   573  
   574  				It("returns empty list", func() {
   575  					Expect(destroyingContainers).To(HaveLen(0))
   576  				})
   577  
   578  				ItClosesConnection()
   579  			})
   580  		})
   581  	})
   582  
   583  	Describe("RemoveMissingContainers", func() {
   584  		var (
   585  			today        time.Time
   586  			gracePeriod  time.Duration
   587  			rowsAffected int
   588  			err          error
   589  		)
   590  
   591  		BeforeEach(func() {
   592  			today = time.Now()
   593  
   594  			_, err = psql.Insert("workers").SetMap(map[string]interface{}{
   595  				"name":  "running-worker",
   596  				"state": "running",
   597  			}).RunWith(dbConn).Exec()
   598  			Expect(err).NotTo(HaveOccurred())
   599  
   600  			_, err = psql.Insert("containers").SetMap(map[string]interface{}{
   601  				"handle":      "created-handle-1",
   602  				"state":       atc.ContainerStateCreated,
   603  				"worker_name": "running-worker",
   604  			}).RunWith(dbConn).Exec()
   605  			Expect(err).NotTo(HaveOccurred())
   606  
   607  			_, err = psql.Insert("containers").SetMap(map[string]interface{}{
   608  				"handle":        "created-handle-2",
   609  				"state":         atc.ContainerStateCreated,
   610  				"worker_name":   "running-worker",
   611  				"missing_since": today.Add(-5 * time.Minute),
   612  			}).RunWith(dbConn).Exec()
   613  			Expect(err).NotTo(HaveOccurred())
   614  
   615  			_, err = psql.Insert("containers").SetMap(map[string]interface{}{
   616  				"handle":        "failed-handle-3",
   617  				"state":         atc.ContainerStateFailed,
   618  				"worker_name":   "running-worker",
   619  				"missing_since": today.Add(-5 * time.Minute),
   620  			}).RunWith(dbConn).Exec()
   621  			Expect(err).NotTo(HaveOccurred())
   622  
   623  			_, err = psql.Insert("containers").SetMap(map[string]interface{}{
   624  				"handle":        "destroying-handle-4",
   625  				"state":         atc.ContainerStateDestroying,
   626  				"worker_name":   "running-worker",
   627  				"missing_since": today.Add(-10 * time.Minute),
   628  			}).RunWith(dbConn).Exec()
   629  			Expect(err).NotTo(HaveOccurred())
   630  		})
   631  
   632  		JustBeforeEach(func() {
   633  			rowsAffected, err = containerRepository.RemoveMissingContainers(gracePeriod)
   634  		})
   635  
   636  		Context("when no created/failed containers have expired", func() {
   637  			BeforeEach(func() {
   638  				gracePeriod = 7 * time.Minute
   639  			})
   640  
   641  			It("affects no containers", func() {
   642  				Expect(err).ToNot(HaveOccurred())
   643  				Expect(rowsAffected).To(Equal(0))
   644  			})
   645  		})
   646  
   647  		Context("when some created containers have expired", func() {
   648  			BeforeEach(func() {
   649  				gracePeriod = 3 * time.Minute
   650  			})
   651  
   652  			It("affects the right containers and deletes created-handle-2", func() {
   653  				result, err := psql.Select("*").From("containers").
   654  					RunWith(dbConn).Exec()
   655  				Expect(err).ToNot(HaveOccurred())
   656  				Expect(result.RowsAffected()).To(Equal(int64(3)))
   657  
   658  				result, err = psql.Select("*").From("containers").
   659  					Where(sq.Eq{"handle": "created-handle-1"}).RunWith(dbConn).Exec()
   660  				Expect(err).ToNot(HaveOccurred())
   661  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   662  
   663  				result, err = psql.Select("*").From("containers").
   664  					Where(sq.Eq{"handle": "created-handle-2"}).RunWith(dbConn).Exec()
   665  				Expect(err).ToNot(HaveOccurred())
   666  				Expect(result.RowsAffected()).To(Equal(int64(0)))
   667  
   668  				result, err = psql.Select("*").From("containers").
   669  					Where(sq.Eq{"handle": "failed-handle-3"}).RunWith(dbConn).Exec()
   670  				Expect(err).ToNot(HaveOccurred())
   671  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   672  
   673  				result, err = psql.Select("*").From("containers").
   674  					Where(sq.Eq{"handle": "destroying-handle-4"}).RunWith(dbConn).Exec()
   675  				Expect(err).ToNot(HaveOccurred())
   676  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   677  			})
   678  		})
   679  
   680  		Context("when worker is in stalled state", func() {
   681  			BeforeEach(func() {
   682  				gracePeriod = 3 * time.Minute
   683  
   684  				_, err = psql.Insert("workers").SetMap(map[string]interface{}{
   685  					"name":  "stalled-worker",
   686  					"state": "stalled",
   687  				}).RunWith(dbConn).Exec()
   688  				Expect(err).NotTo(HaveOccurred())
   689  
   690  				_, err = psql.Insert("containers").SetMap(map[string]interface{}{
   691  					"handle":        "stalled-handle-5",
   692  					"state":         atc.ContainerStateCreated,
   693  					"worker_name":   "stalled-worker",
   694  					"missing_since": today.Add(-10 * time.Minute),
   695  				}).RunWith(dbConn).Exec()
   696  				Expect(err).NotTo(HaveOccurred())
   697  
   698  				_, err = psql.Update("containers").
   699  					Set("worker_name", "stalled-worker").
   700  					Where(sq.Eq{"handle": "failed-handle-3"}).
   701  					RunWith(dbConn).Exec()
   702  				Expect(err).NotTo(HaveOccurred())
   703  
   704  				_, err = psql.Update("containers").
   705  					Set("missing_since", today.Add(-5*time.Minute)).
   706  					Where(sq.Eq{"handle": "destroying-handle-4"}).
   707  					RunWith(dbConn).Exec()
   708  				Expect(err).NotTo(HaveOccurred())
   709  			})
   710  
   711  			It("deletes containers missing for more than grace period, on running (unstalled) workers", func() {
   712  				Expect(err).ToNot(HaveOccurred())
   713  				Expect(rowsAffected).To(Equal(1))
   714  			})
   715  
   716  			It("does not delete containers on stalled workers", func() {
   717  				result, err := psql.Select("*").From("containers").
   718  					RunWith(dbConn).Exec()
   719  				Expect(err).ToNot(HaveOccurred())
   720  				Expect(result.RowsAffected()).To(Equal(int64(4)))
   721  
   722  				result, err = psql.Select("*").From("containers").
   723  					Where(sq.Eq{"handle": "created-handle-1"}).RunWith(dbConn).Exec()
   724  				Expect(err).ToNot(HaveOccurred())
   725  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   726  
   727  				result, err = psql.Select("*").From("containers").
   728  					Where(sq.Eq{"handle": "created-handle-2"}).RunWith(dbConn).Exec()
   729  				Expect(err).ToNot(HaveOccurred())
   730  				Expect(result.RowsAffected()).To(Equal(int64(0)))
   731  
   732  				result, err = psql.Select("*").From("containers").
   733  					Where(sq.Eq{"handle": "failed-handle-3"}).RunWith(dbConn).Exec()
   734  				Expect(err).ToNot(HaveOccurred())
   735  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   736  
   737  				result, err = psql.Select("*").From("containers").
   738  					Where(sq.Eq{"handle": "destroying-handle-4"}).RunWith(dbConn).Exec()
   739  				Expect(err).ToNot(HaveOccurred())
   740  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   741  
   742  				result, err = psql.Select("*").From("containers").
   743  					Where(sq.Eq{"handle": "stalled-handle-5"}).RunWith(dbConn).Exec()
   744  				Expect(err).ToNot(HaveOccurred())
   745  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   746  			})
   747  
   748  		})
   749  	})
   750  
   751  	Describe("RemoveDestroyingContainers", func() {
   752  		var failedErr error
   753  		var numDeleted int
   754  		var handles []string
   755  
   756  		JustBeforeEach(func() {
   757  			numDeleted, failedErr = containerRepository.RemoveDestroyingContainers(defaultWorker.Name(), handles)
   758  		})
   759  
   760  		Context("when there are containers to destroy", func() {
   761  
   762  			Context("when container is in destroying state", func() {
   763  				BeforeEach(func() {
   764  					handles = []string{"some-handle1", "some-handle2"}
   765  					result, err := psql.Insert("containers").SetMap(map[string]interface{}{
   766  						"state":       atc.ContainerStateDestroying,
   767  						"handle":      "123-456-abc-def",
   768  						"worker_name": defaultWorker.Name(),
   769  					}).RunWith(dbConn).Exec()
   770  
   771  					Expect(err).ToNot(HaveOccurred())
   772  					Expect(result.RowsAffected()).To(Equal(int64(1)))
   773  				})
   774  				It("should destroy", func() {
   775  					result, err := psql.Select("*").From("containers").
   776  						Where(sq.Eq{"handle": "123-456-abc-def"}).RunWith(dbConn).Exec()
   777  
   778  					Expect(err).ToNot(HaveOccurred())
   779  					Expect(result.RowsAffected()).To(Equal(int64(0)))
   780  				})
   781  				It("returns the correct number of rows removed", func() {
   782  					Expect(numDeleted).To(Equal(1))
   783  				})
   784  				It("does not return an error", func() {
   785  					Expect(failedErr).ToNot(HaveOccurred())
   786  				})
   787  			})
   788  
   789  			Context("when handles are empty list", func() {
   790  				BeforeEach(func() {
   791  					handles = []string{}
   792  					result, err := psql.Insert("containers").SetMap(map[string]interface{}{
   793  						"state":       atc.ContainerStateDestroying,
   794  						"handle":      "123-456-abc-def",
   795  						"worker_name": defaultWorker.Name(),
   796  					}).RunWith(dbConn).Exec()
   797  
   798  					Expect(err).ToNot(HaveOccurred())
   799  					Expect(result.RowsAffected()).To(Equal(int64(1)))
   800  				})
   801  
   802  				It("should destroy", func() {
   803  					result, err := psql.Select("*").From("containers").
   804  						Where(sq.Eq{"handle": "123-456-abc-def"}).RunWith(dbConn).Exec()
   805  
   806  					Expect(err).ToNot(HaveOccurred())
   807  					Expect(result.RowsAffected()).To(Equal(int64(0)))
   808  				})
   809  
   810  				It("returns the correct number of rows removed", func() {
   811  					Expect(numDeleted).To(Equal(1))
   812  				})
   813  
   814  				It("does not return an error", func() {
   815  					Expect(failedErr).ToNot(HaveOccurred())
   816  				})
   817  			})
   818  
   819  			Context("when container is in create/creating state", func() {
   820  				BeforeEach(func() {
   821  					handles = []string{"some-handle1", "some-handle2"}
   822  					result, err := psql.Insert("containers").SetMap(map[string]interface{}{
   823  						"state":       "creating",
   824  						"handle":      "123-456-abc-def",
   825  						"worker_name": defaultWorker.Name(),
   826  					}).RunWith(dbConn).Exec()
   827  
   828  					Expect(err).ToNot(HaveOccurred())
   829  					Expect(result.RowsAffected()).To(Equal(int64(1)))
   830  				})
   831  				It("should not destroy", func() {
   832  					result, err := psql.Select("*").From("containers").
   833  						Where(sq.Eq{"handle": "123-456-abc-def"}).RunWith(dbConn).Exec()
   834  
   835  					Expect(err).ToNot(HaveOccurred())
   836  					Expect(result.RowsAffected()).To(Equal(int64(1)))
   837  				})
   838  				It("returns the correct number of rows removed", func() {
   839  					Expect(numDeleted).To(Equal(0))
   840  				})
   841  				It("does not return an error", func() {
   842  					Expect(failedErr).ToNot(HaveOccurred())
   843  				})
   844  			})
   845  		})
   846  
   847  		Context("when there are no containers to destroy", func() {
   848  			BeforeEach(func() {
   849  				handles = []string{"some-handle1", "some-handle2"}
   850  
   851  				result, err := psql.Insert("containers").SetMap(
   852  					map[string]interface{}{
   853  						"state":       "destroying",
   854  						"handle":      "some-handle1",
   855  						"worker_name": defaultWorker.Name(),
   856  					},
   857  				).RunWith(dbConn).Exec()
   858  				Expect(err).ToNot(HaveOccurred())
   859  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   860  
   861  				result, err = psql.Insert("containers").SetMap(
   862  					map[string]interface{}{
   863  						"state":       "destroying",
   864  						"handle":      "some-handle2",
   865  						"worker_name": defaultWorker.Name(),
   866  					},
   867  				).RunWith(dbConn).Exec()
   868  				Expect(err).ToNot(HaveOccurred())
   869  				Expect(result.RowsAffected()).To(Equal(int64(1)))
   870  			})
   871  
   872  			It("doesn't destroy containers that are in handles", func() {
   873  				result, err := psql.Select("*").From("containers").
   874  					Where(sq.Eq{"handle": handles}).RunWith(dbConn).Exec()
   875  
   876  				Expect(err).ToNot(HaveOccurred())
   877  				Expect(result.RowsAffected()).To(Equal(int64(2)))
   878  			})
   879  
   880  			It("does not return an error", func() {
   881  				Expect(failedErr).ToNot(HaveOccurred())
   882  			})
   883  			It("returns the correct number of rows removed", func() {
   884  				Expect(numDeleted).To(Equal(0))
   885  			})
   886  		})
   887  
   888  		Describe("errors", func() {
   889  			Context("when the query cannot be executed", func() {
   890  				BeforeEach(func() {
   891  					err := dbConn.Close()
   892  					Expect(err).ToNot(HaveOccurred())
   893  				})
   894  
   895  				AfterEach(func() {
   896  					dbConn = postgresRunner.OpenConn()
   897  				})
   898  
   899  				It("returns an error", func() {
   900  					Expect(failedErr).To(HaveOccurred())
   901  				})
   902  			})
   903  		})
   904  	})
   905  
   906  	Describe("UpdateContainersMissingSince", func() {
   907  		var (
   908  			today        time.Time
   909  			err          error
   910  			handles      []string
   911  			missingSince pq.NullTime
   912  		)
   913  
   914  		BeforeEach(func() {
   915  			result, err := psql.Insert("containers").SetMap(map[string]interface{}{
   916  				"state":       atc.ContainerStateDestroying,
   917  				"handle":      "some-handle1",
   918  				"worker_name": defaultWorker.Name(),
   919  			}).RunWith(dbConn).Exec()
   920  
   921  			Expect(err).ToNot(HaveOccurred())
   922  			Expect(result.RowsAffected()).To(Equal(int64(1)))
   923  
   924  			result, err = psql.Insert("containers").SetMap(map[string]interface{}{
   925  				"state":       atc.ContainerStateDestroying,
   926  				"handle":      "some-handle2",
   927  				"worker_name": defaultWorker.Name(),
   928  			}).RunWith(dbConn).Exec()
   929  
   930  			Expect(err).ToNot(HaveOccurred())
   931  			Expect(result.RowsAffected()).To(Equal(int64(1)))
   932  
   933  			today = time.Date(2018, 9, 24, 0, 0, 0, 0, time.UTC)
   934  
   935  			result, err = psql.Insert("containers").SetMap(map[string]interface{}{
   936  				"state":         atc.ContainerStateCreated,
   937  				"handle":        "some-handle3",
   938  				"worker_name":   defaultWorker.Name(),
   939  				"missing_since": today,
   940  			}).RunWith(dbConn).Exec()
   941  
   942  			Expect(err).ToNot(HaveOccurred())
   943  			Expect(result.RowsAffected()).To(Equal(int64(1)))
   944  		})
   945  
   946  		JustBeforeEach(func() {
   947  			err = containerRepository.UpdateContainersMissingSince(defaultWorker.Name(), handles)
   948  		})
   949  
   950  		Context("when the reported handles is a subset", func() {
   951  			BeforeEach(func() {
   952  				handles = []string{"some-handle1"}
   953  			})
   954  
   955  			Context("having the containers in the creating state in the db", func() {
   956  				BeforeEach(func() {
   957  					result, err := psql.Update("containers").
   958  						Where(sq.Eq{"handle": "some-handle3"}).
   959  						SetMap(map[string]interface{}{
   960  							"state":         atc.ContainerStateCreating,
   961  							"missing_since": nil,
   962  						}).RunWith(dbConn).Exec()
   963  					Expect(err).NotTo(HaveOccurred())
   964  					Expect(result.RowsAffected()).To(Equal(int64(1)))
   965  				})
   966  
   967  				It("does not mark as missing", func() {
   968  					err = psql.Select("missing_since").From("containers").
   969  						Where(sq.Eq{"handle": "some-handle3"}).RunWith(dbConn).QueryRow().
   970  						Scan(&missingSince)
   971  					Expect(err).ToNot(HaveOccurred())
   972  					Expect(missingSince.Valid).To(BeFalse())
   973  				})
   974  			})
   975  
   976  			It("should mark containers not in the subset and not already marked as missing", func() {
   977  				err = psql.Select("missing_since").From("containers").
   978  					Where(sq.Eq{"handle": "some-handle1"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
   979  				Expect(err).ToNot(HaveOccurred())
   980  				Expect(missingSince.Valid).To(BeFalse())
   981  
   982  				err = psql.Select("missing_since").From("containers").
   983  					Where(sq.Eq{"handle": "some-handle2"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
   984  				Expect(err).ToNot(HaveOccurred())
   985  				Expect(missingSince.Valid).To(BeTrue())
   986  
   987  				err = psql.Select("missing_since").From("containers").
   988  					Where(sq.Eq{"handle": "some-handle3"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
   989  				Expect(err).ToNot(HaveOccurred())
   990  				Expect(missingSince.Valid).To(BeTrue())
   991  				Expect(missingSince.Time.Unix()).To(Equal(today.Unix()))
   992  			})
   993  
   994  			It("does not return an error", func() {
   995  				Expect(err).ToNot(HaveOccurred())
   996  			})
   997  		})
   998  
   999  		Context("when the reported handles is the full set", func() {
  1000  			BeforeEach(func() {
  1001  				handles = []string{"some-handle1", "some-handle2"}
  1002  			})
  1003  
  1004  			It("should not update", func() {
  1005  				err = psql.Select("missing_since").From("containers").
  1006  					Where(sq.Eq{"handle": "some-handle1"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
  1007  				Expect(err).ToNot(HaveOccurred())
  1008  				Expect(missingSince.Valid).To(BeFalse())
  1009  
  1010  				err = psql.Select("missing_since").From("containers").
  1011  					Where(sq.Eq{"handle": "some-handle2"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
  1012  				Expect(err).ToNot(HaveOccurred())
  1013  				Expect(missingSince.Valid).To(BeFalse())
  1014  			})
  1015  
  1016  			It("does not return an error", func() {
  1017  				Expect(err).ToNot(HaveOccurred())
  1018  			})
  1019  		})
  1020  
  1021  		Context("when the reported handles includes a container marked as missing", func() {
  1022  			BeforeEach(func() {
  1023  				handles = []string{"some-handle1", "some-handle2", "some-handle3"}
  1024  			})
  1025  
  1026  			It("should mark the previously missing container as not missing", func() {
  1027  				err = psql.Select("missing_since").From("containers").
  1028  					Where(sq.Eq{"handle": "some-handle1"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
  1029  				Expect(err).ToNot(HaveOccurred())
  1030  				Expect(missingSince.Valid).To(BeFalse())
  1031  
  1032  				err = psql.Select("missing_since").From("containers").
  1033  					Where(sq.Eq{"handle": "some-handle2"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
  1034  				Expect(err).ToNot(HaveOccurred())
  1035  				Expect(missingSince.Valid).To(BeFalse())
  1036  
  1037  				err = psql.Select("missing_since").From("containers").
  1038  					Where(sq.Eq{"handle": "some-handle3"}).RunWith(dbConn).QueryRow().Scan(&missingSince)
  1039  				Expect(err).ToNot(HaveOccurred())
  1040  				Expect(missingSince.Valid).To(BeFalse())
  1041  			})
  1042  
  1043  			It("does not return an error", func() {
  1044  				Expect(err).ToNot(HaveOccurred())
  1045  			})
  1046  		})
  1047  	})
  1048  
  1049  	Describe("DestroyUnknownContainers", func() {
  1050  		var (
  1051  			err                     error
  1052  			workerReportedHandles   []string
  1053  			numberUnknownContainers int
  1054  		)
  1055  
  1056  		BeforeEach(func() {
  1057  			result, err := psql.Insert("containers").SetMap(map[string]interface{}{
  1058  				"state":       atc.ContainerStateDestroying,
  1059  				"handle":      "some-handle1",
  1060  				"worker_name": defaultWorker.Name(),
  1061  			}).RunWith(dbConn).Exec()
  1062  
  1063  			Expect(err).ToNot(HaveOccurred())
  1064  			Expect(result.RowsAffected()).To(Equal(int64(1)))
  1065  
  1066  			result, err = psql.Insert("containers").SetMap(map[string]interface{}{
  1067  				"state":       atc.ContainerStateCreated,
  1068  				"handle":      "some-handle2",
  1069  				"worker_name": defaultWorker.Name(),
  1070  			}).RunWith(dbConn).Exec()
  1071  
  1072  			Expect(err).ToNot(HaveOccurred())
  1073  			Expect(result.RowsAffected()).To(Equal(int64(1)))
  1074  		})
  1075  
  1076  		JustBeforeEach(func() {
  1077  			numberUnknownContainers, err = containerRepository.DestroyUnknownContainers(defaultWorker.Name(), workerReportedHandles)
  1078  			Expect(err).ToNot(HaveOccurred())
  1079  		})
  1080  
  1081  		Context("when there are containers on the worker that are not in the db", func() {
  1082  			var destroyingContainerHandles []string
  1083  			BeforeEach(func() {
  1084  				workerReportedHandles = []string{"some-handle3", "some-handle4"}
  1085  				destroyingContainerHandles = append(workerReportedHandles, "some-handle1")
  1086  			})
  1087  
  1088  			It("adds new destroying containers to the database", func() {
  1089  				result, err := psql.Select("handle").
  1090  					From("containers").
  1091  					Where(sq.Eq{"state": atc.ContainerStateDestroying}).
  1092  					RunWith(dbConn).Query()
  1093  
  1094  				Expect(err).ToNot(HaveOccurred())
  1095  
  1096  				var handle string
  1097  				for result.Next() {
  1098  					err = result.Scan(&handle)
  1099  					Expect(err).ToNot(HaveOccurred())
  1100  					Expect(handle).Should(BeElementOf(destroyingContainerHandles))
  1101  				}
  1102  				Expect(numberUnknownContainers).To(Equal(2))
  1103  			})
  1104  
  1105  			It("does not affect containers in any other state", func() {
  1106  				rows, err := psql.Select("handle").
  1107  					From("containers").
  1108  					Where(sq.Eq{"state": atc.ContainerStateCreated}).
  1109  					RunWith(dbConn).Query()
  1110  
  1111  				Expect(err).ToNot(HaveOccurred())
  1112  
  1113  				var handle string
  1114  				var rowsAffected int
  1115  				for rows.Next() {
  1116  					err = rows.Scan(&handle)
  1117  					Expect(err).ToNot(HaveOccurred())
  1118  					Expect(handle).To(Equal("some-handle2"))
  1119  					rowsAffected++
  1120  				}
  1121  
  1122  				Expect(rowsAffected).To(Equal(1))
  1123  			})
  1124  		})
  1125  
  1126  		Context("when there are no unknown containers on the worker", func() {
  1127  			BeforeEach(func() {
  1128  				workerReportedHandles = []string{"some-handle1", "some-handle2"}
  1129  			})
  1130  
  1131  			It("should not try to destroy anything", func() {
  1132  				Expect(numberUnknownContainers).To(Equal(0))
  1133  
  1134  				rows, err := psql.Select("handle").
  1135  					From("containers").
  1136  					Where(sq.Eq{"state": atc.ContainerStateDestroying}).
  1137  					RunWith(dbConn).Query()
  1138  
  1139  				Expect(err).ToNot(HaveOccurred())
  1140  
  1141  				var handle string
  1142  				var rowsAffected int
  1143  				for rows.Next() {
  1144  					err = rows.Scan(&handle)
  1145  					Expect(err).ToNot(HaveOccurred())
  1146  					Expect(handle).To(Equal("some-handle1"))
  1147  					rowsAffected++
  1148  				}
  1149  
  1150  				Expect(rowsAffected).To(Equal(1))
  1151  			})
  1152  		})
  1153  	})
  1154  })