github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/resource_cache_lifecycle_test.go (about)

     1  package db_test
     2  
     3  import (
     4  	"fmt"
     5  	"time"
     6  
     7  	"github.com/pf-qiu/concourse/v6/atc"
     8  	"github.com/pf-qiu/concourse/v6/atc/db"
     9  	"github.com/pf-qiu/concourse/v6/atc/db/dbtest"
    10  
    11  	. "github.com/onsi/ginkgo"
    12  	. "github.com/onsi/gomega"
    13  )
    14  
    15  var _ = Describe("ResourceCacheLifecycle", func() {
    16  
    17  	var resourceCacheLifecycle db.ResourceCacheLifecycle
    18  
    19  	BeforeEach(func() {
    20  		resourceCacheLifecycle = db.NewResourceCacheLifecycle(dbConn)
    21  	})
    22  
    23  	Describe("CleanUpInvalidCaches", func() {
    24  		Context("the resource cache is used by a build", func() {
    25  			resourceCacheForOneOffBuild := func() (db.UsedResourceCache, db.Build) {
    26  				build, err := defaultTeam.CreateOneOffBuild()
    27  				Expect(err).ToNot(HaveOccurred())
    28  				return createResourceCacheWithUser(db.ForBuild(build.ID())), build
    29  			}
    30  
    31  			resourceCacheForJobBuild := func() (db.UsedResourceCache, db.Build) {
    32  				build, err := defaultJob.CreateBuild()
    33  				Expect(err).ToNot(HaveOccurred())
    34  				return createResourceCacheWithUser(db.ForBuild(build.ID())), build
    35  			}
    36  
    37  			Context("when its a one off build", func() {
    38  				It("doesn't delete the resource cache", func() {
    39  					_, _ = resourceCacheForOneOffBuild()
    40  
    41  					err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
    42  					Expect(err).ToNot(HaveOccurred())
    43  					Expect(countResourceCaches()).ToNot(BeZero())
    44  				})
    45  
    46  				Context("when the build goes away", func() {
    47  					BeforeEach(func() {
    48  						_, build := resourceCacheForOneOffBuild()
    49  
    50  						_, err := build.Delete()
    51  						Expect(err).ToNot(HaveOccurred())
    52  
    53  						Expect(countResourceCaches()).ToNot(BeZero())
    54  
    55  						err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
    56  						Expect(err).ToNot(HaveOccurred())
    57  					})
    58  
    59  					It("deletes the resource cache", func() {
    60  						Expect(countResourceCaches()).To(BeZero())
    61  					})
    62  				})
    63  
    64  				Context("when the cache is for a saved image resource version for a finished build", func() {
    65  					setBuildStatus := func(a db.BuildStatus) {
    66  						resourceCache, build := resourceCacheForOneOffBuild()
    67  
    68  						err := build.SaveImageResourceVersion(resourceCache)
    69  						Expect(err).ToNot(HaveOccurred())
    70  
    71  						err = build.SetInterceptible(false)
    72  						Expect(err).ToNot(HaveOccurred())
    73  
    74  						err = build.Finish(a)
    75  						Expect(err).ToNot(HaveOccurred())
    76  
    77  						err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger)
    78  						Expect(err).ToNot(HaveOccurred())
    79  					}
    80  
    81  					Context("when the build has succeeded", func() {
    82  						It("does not remove the image resource cache", func() {
    83  							setBuildStatus(db.BuildStatusSucceeded)
    84  							Expect(countResourceCaches()).ToNot(BeZero())
    85  
    86  							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
    87  							Expect(err).ToNot(HaveOccurred())
    88  
    89  							Expect(countResourceCaches()).ToNot(BeZero())
    90  						})
    91  					})
    92  
    93  					Context("when build has not succeeded", func() {
    94  						It("does not removes the image resource cache", func() {
    95  							setBuildStatus(db.BuildStatusFailed)
    96  							Expect(countResourceCaches()).ToNot(BeZero())
    97  
    98  							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
    99  							Expect(err).ToNot(HaveOccurred())
   100  
   101  							Expect(countResourceCaches()).ToNot(BeZero())
   102  						})
   103  					})
   104  				})
   105  			})
   106  
   107  			Context("when its a build of a job in a pipeline", func() {
   108  				Context("when the cache is for a saved image resource version for a finished build", func() {
   109  					setBuildStatus := func(a db.BuildStatus) (db.UsedResourceCache, db.Build) {
   110  						resourceCache, build := resourceCacheForJobBuild()
   111  						Expect(build.JobID()).ToNot(BeZero())
   112  
   113  						err := build.SaveImageResourceVersion(resourceCache)
   114  						Expect(err).ToNot(HaveOccurred())
   115  
   116  						err = build.SetInterceptible(false)
   117  						Expect(err).ToNot(HaveOccurred())
   118  
   119  						err = build.Finish(a)
   120  						Expect(err).ToNot(HaveOccurred())
   121  
   122  						err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger)
   123  						Expect(err).ToNot(HaveOccurred())
   124  						return resourceCache, build
   125  					}
   126  
   127  					Context("when the build has succeeded", func() {
   128  						It("does not remove the resource cache for the most recent build", func() {
   129  							setBuildStatus(db.BuildStatusSucceeded)
   130  							Expect(countResourceCaches()).To(Equal(1))
   131  
   132  							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   133  							Expect(err).ToNot(HaveOccurred())
   134  
   135  							Expect(countResourceCaches()).To(Equal(1))
   136  						})
   137  
   138  						It("removes resource caches for previous finished builds", func() {
   139  							resourceCache1, _ := setBuildStatus(db.BuildStatusFailed)
   140  							resourceCache2, _ := setBuildStatus(db.BuildStatusSucceeded)
   141  							Expect(resourceCache1.ID()).ToNot(Equal(resourceCache2.ID()))
   142  
   143  							Expect(countResourceCaches()).To(Equal(2))
   144  
   145  							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   146  							Expect(err).ToNot(HaveOccurred())
   147  
   148  							Expect(countResourceCaches()).To(Equal(1))
   149  						})
   150  						It("does not remove the resource caches from other jobs", func() {
   151  							By("creating a second pipeline")
   152  							secondPipeline, _, err := defaultTeam.SavePipeline(atc.PipelineRef{Name: "second-pipeline"}, atc.Config{
   153  								Jobs: atc.JobConfigs{
   154  									{
   155  										Name: "some-job",
   156  									},
   157  								},
   158  								Resources: atc.ResourceConfigs{
   159  									{
   160  										Name: "some-resource",
   161  										Type: "some-base-resource-type",
   162  										Source: atc.Source{
   163  											"some": "source",
   164  										},
   165  									},
   166  								},
   167  								ResourceTypes: atc.ResourceTypes{
   168  									{
   169  										Name: "some-type",
   170  										Type: "some-base-resource-type",
   171  										Source: atc.Source{
   172  											"some-type": "source",
   173  										},
   174  									},
   175  								},
   176  							}, db.ConfigVersion(0), false)
   177  							Expect(err).NotTo(HaveOccurred())
   178  
   179  							By("creating an image resource cache tied to the job in the second pipeline")
   180  							job, _, err := secondPipeline.Job("some-job")
   181  							Expect(err).ToNot(HaveOccurred())
   182  							build, err := job.CreateBuild()
   183  							Expect(err).ToNot(HaveOccurred())
   184  							resourceCache := createResourceCacheWithUser(db.ForBuild(build.ID()))
   185  
   186  							err = build.SaveImageResourceVersion(resourceCache)
   187  							Expect(err).ToNot(HaveOccurred())
   188  
   189  							err = build.SetInterceptible(false)
   190  							Expect(err).ToNot(HaveOccurred())
   191  
   192  							By("creating an image resource cached in the default pipeline")
   193  							setBuildStatus(db.BuildStatusSucceeded)
   194  
   195  							Expect(countResourceCaches()).To(Equal(2))
   196  						})
   197  					})
   198  
   199  					Context("when build has not succeeded", func() {
   200  						It("does not remove the image resource cache", func() {
   201  							setBuildStatus(db.BuildStatusFailed)
   202  							Expect(countResourceCaches()).ToNot(BeZero())
   203  
   204  							err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   205  							Expect(err).ToNot(HaveOccurred())
   206  
   207  							Expect(countResourceCaches()).ToNot(BeZero())
   208  						})
   209  					})
   210  				})
   211  			})
   212  		})
   213  
   214  		Context("the resource cache is used by a container", func() {
   215  			var (
   216  				container      db.CreatingContainer
   217  				containerOwner db.ContainerOwner
   218  			)
   219  
   220  			BeforeEach(func() {
   221  				var err error
   222  				resourceConfig, err := resourceConfigFactory.FindOrCreateResourceConfig(
   223  					"some-base-resource-type",
   224  					atc.Source{
   225  						"some": "source",
   226  					},
   227  					atc.VersionedResourceTypes{},
   228  				)
   229  				Expect(err).ToNot(HaveOccurred())
   230  
   231  				containerOwner = db.NewResourceConfigCheckSessionContainerOwner(
   232  					resourceConfig.ID(),
   233  					resourceConfig.OriginBaseResourceType().ID,
   234  					db.ContainerOwnerExpiries{},
   235  				)
   236  
   237  				container, err = defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{})
   238  				Expect(err).ToNot(HaveOccurred())
   239  
   240  				_ = createResourceCacheWithUser(db.ForContainer(container.ID()))
   241  			})
   242  
   243  			Context("and the container still exists", func() {
   244  				BeforeEach(func() {
   245  					err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   246  					Expect(err).ToNot(HaveOccurred())
   247  				})
   248  
   249  				It("doesn't delete the resource cache", func() {
   250  					Expect(countResourceCaches()).ToNot(BeZero())
   251  				})
   252  			})
   253  
   254  			Context("and the container has been deleted", func() {
   255  				BeforeEach(func() {
   256  					createdContainer, err := container.Created()
   257  					Expect(err).ToNot(HaveOccurred())
   258  
   259  					destroyingContainer, err := createdContainer.Destroying()
   260  					Expect(err).ToNot(HaveOccurred())
   261  
   262  					_, err = destroyingContainer.Destroy()
   263  					Expect(err).ToNot(HaveOccurred())
   264  
   265  					err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   266  					Expect(err).ToNot(HaveOccurred())
   267  				})
   268  
   269  				It("deletes the resource cache", func() {
   270  					Expect(countResourceCaches()).To(BeZero())
   271  				})
   272  			})
   273  		})
   274  
   275  		Context("when the cache is for a custom resource type", func() {
   276  			It("does not remove the cache if the type is still configured", func() {
   277  				_, err := resourceConfigFactory.FindOrCreateResourceConfig(
   278  					"some-type",
   279  					atc.Source{
   280  						"some": "source",
   281  					},
   282  					atc.VersionedResourceTypes{
   283  						atc.VersionedResourceType{
   284  							ResourceType: atc.ResourceType{
   285  								Name: "some-type",
   286  								Type: "some-base-resource-type",
   287  								Source: atc.Source{
   288  									"some": "source",
   289  								},
   290  							},
   291  							Version: atc.Version{"showme": "whatyougot"},
   292  						},
   293  					},
   294  				)
   295  				Expect(err).ToNot(HaveOccurred())
   296  
   297  				Expect(countResourceCaches()).ToNot(BeZero())
   298  				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   299  				Expect(err).ToNot(HaveOccurred())
   300  
   301  				Expect(countResourceCaches()).ToNot(BeZero())
   302  			})
   303  
   304  			It("removes the cache if the type is no longer configured", func() {
   305  				_, err := resourceConfigFactory.FindOrCreateResourceConfig(
   306  					"some-type",
   307  					atc.Source{
   308  						"some": "source",
   309  					},
   310  					atc.VersionedResourceTypes{
   311  						atc.VersionedResourceType{
   312  							ResourceType: atc.ResourceType{
   313  								Name: "some-type",
   314  								Type: "some-base-resource-type",
   315  								Source: atc.Source{
   316  									"some": "source",
   317  								},
   318  							},
   319  							Version: atc.Version{"showme": "whatyougot"},
   320  						},
   321  					},
   322  				)
   323  				Expect(err).ToNot(HaveOccurred())
   324  
   325  				err = resourceConfigCheckSessionLifecycle.CleanInactiveResourceConfigCheckSessions()
   326  				Expect(err).ToNot(HaveOccurred())
   327  
   328  				err = resourceConfigFactory.CleanUnreferencedConfigs(0)
   329  				Expect(err).ToNot(HaveOccurred())
   330  
   331  				Expect(countResourceCaches()).ToNot(BeZero())
   332  
   333  				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   334  				Expect(err).ToNot(HaveOccurred())
   335  
   336  				Expect(countResourceCaches()).To(BeZero())
   337  			})
   338  		})
   339  
   340  		Context("when the cache is for a resource version used as an input for the next build of a job", func() {
   341  			It("does not remove the cache", func() {
   342  				scenario := dbtest.Setup(
   343  					builder.WithPipeline(atc.Config{
   344  						Resources: atc.ResourceConfigs{
   345  							{
   346  								Name: "some-resource",
   347  								Type: "some-base-resource-type",
   348  								Source: atc.Source{
   349  									"some": "source",
   350  								},
   351  							},
   352  						},
   353  					}),
   354  					builder.WithResourceVersions("some-resource", atc.Version{"some": "version"}),
   355  				)
   356  
   357  				rc, found, err := resourceConfigFactory.FindResourceConfigByID(scenario.Resource("some-resource").ResourceConfigID())
   358  				Expect(found).To(BeTrue())
   359  				Expect(err).ToNot(HaveOccurred())
   360  
   361  				resourceConfigScope, err := rc.FindOrCreateScope(scenario.Resource("some-resource"))
   362  				Expect(err).ToNot(HaveOccurred())
   363  
   364  				containerOwner := db.NewResourceConfigCheckSessionContainerOwner(
   365  					resourceConfigScope.ResourceConfig().ID(),
   366  					resourceConfigScope.ResourceConfig().OriginBaseResourceType().ID,
   367  					db.ContainerOwnerExpiries{},
   368  				)
   369  
   370  				container, err := defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{})
   371  				Expect(err).ToNot(HaveOccurred())
   372  
   373  				_ = createResourceCacheWithUser(db.ForContainer(container.ID()))
   374  
   375  				resourceConfigVersion, found, err := resourceConfigScope.FindVersion(atc.Version{"some": "version"})
   376  				Expect(found).To(BeTrue())
   377  				Expect(err).ToNot(HaveOccurred())
   378  
   379  				err = defaultJob.SaveNextInputMapping(db.InputMapping{
   380  					"some-resource": db.InputResult{
   381  						Input: &db.AlgorithmInput{
   382  							AlgorithmVersion: db.AlgorithmVersion{
   383  								Version:    db.ResourceVersion(convertToMD5(atc.Version(resourceConfigVersion.Version()))),
   384  								ResourceID: scenario.Resource("some-resource").ID(),
   385  							},
   386  						},
   387  						PassedBuildIDs: []int{},
   388  					},
   389  				}, true)
   390  				Expect(err).ToNot(HaveOccurred())
   391  
   392  				createdContainer, err := container.Created()
   393  				Expect(err).ToNot(HaveOccurred())
   394  
   395  				destroyingContainer, err := createdContainer.Destroying()
   396  				Expect(err).ToNot(HaveOccurred())
   397  
   398  				_, err = destroyingContainer.Destroy()
   399  				Expect(err).ToNot(HaveOccurred())
   400  
   401  				Expect(countResourceCaches()).ToNot(BeZero())
   402  
   403  				err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle"))
   404  				Expect(err).ToNot(HaveOccurred())
   405  
   406  				Expect(countResourceCaches()).ToNot(BeZero())
   407  			})
   408  		})
   409  	})
   410  })
   411  
   412  func countResourceCaches() int {
   413  	var result int
   414  	err := psql.Select("count(*)").
   415  		From("resource_caches").
   416  		RunWith(dbConn).
   417  		QueryRow().
   418  		Scan(&result)
   419  	Expect(err).ToNot(HaveOccurred())
   420  
   421  	return result
   422  
   423  }
   424  
   425  func createResourceCacheWithUser(resourceCacheUser db.ResourceCacheUser) db.UsedResourceCache {
   426  	usedResourceCache, err := resourceCacheFactory.FindOrCreateResourceCache(
   427  		resourceCacheUser,
   428  		"some-base-resource-type",
   429  		atc.Version{"some": "version"},
   430  		atc.Source{
   431  			"some": "source",
   432  		},
   433  		atc.Params{"some": fmt.Sprintf("param-%d", time.Now().UnixNano())},
   434  		atc.VersionedResourceTypes{},
   435  	)
   436  	Expect(err).ToNot(HaveOccurred())
   437  
   438  	return usedResourceCache
   439  }