github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/resource_cache_lifecycle_test.go (about) 1 package db_test 2 3 import ( 4 "fmt" 5 "time" 6 7 "github.com/pf-qiu/concourse/v6/atc" 8 "github.com/pf-qiu/concourse/v6/atc/db" 9 "github.com/pf-qiu/concourse/v6/atc/db/dbtest" 10 11 . "github.com/onsi/ginkgo" 12 . "github.com/onsi/gomega" 13 ) 14 15 var _ = Describe("ResourceCacheLifecycle", func() { 16 17 var resourceCacheLifecycle db.ResourceCacheLifecycle 18 19 BeforeEach(func() { 20 resourceCacheLifecycle = db.NewResourceCacheLifecycle(dbConn) 21 }) 22 23 Describe("CleanUpInvalidCaches", func() { 24 Context("the resource cache is used by a build", func() { 25 resourceCacheForOneOffBuild := func() (db.UsedResourceCache, db.Build) { 26 build, err := defaultTeam.CreateOneOffBuild() 27 Expect(err).ToNot(HaveOccurred()) 28 return createResourceCacheWithUser(db.ForBuild(build.ID())), build 29 } 30 31 resourceCacheForJobBuild := func() (db.UsedResourceCache, db.Build) { 32 build, err := defaultJob.CreateBuild() 33 Expect(err).ToNot(HaveOccurred()) 34 return createResourceCacheWithUser(db.ForBuild(build.ID())), build 35 } 36 37 Context("when its a one off build", func() { 38 It("doesn't delete the resource cache", func() { 39 _, _ = resourceCacheForOneOffBuild() 40 41 err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 42 Expect(err).ToNot(HaveOccurred()) 43 Expect(countResourceCaches()).ToNot(BeZero()) 44 }) 45 46 Context("when the build goes away", func() { 47 BeforeEach(func() { 48 _, build := resourceCacheForOneOffBuild() 49 50 _, err := build.Delete() 51 Expect(err).ToNot(HaveOccurred()) 52 53 Expect(countResourceCaches()).ToNot(BeZero()) 54 55 err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 56 Expect(err).ToNot(HaveOccurred()) 57 }) 58 59 It("deletes the resource cache", func() { 60 Expect(countResourceCaches()).To(BeZero()) 61 }) 62 }) 63 64 Context("when the cache is for a saved image resource version for a finished build", func() { 65 setBuildStatus := func(a db.BuildStatus) { 66 resourceCache, build := resourceCacheForOneOffBuild() 67 68 err := build.SaveImageResourceVersion(resourceCache) 69 Expect(err).ToNot(HaveOccurred()) 70 71 err = build.SetInterceptible(false) 72 Expect(err).ToNot(HaveOccurred()) 73 74 err = build.Finish(a) 75 Expect(err).ToNot(HaveOccurred()) 76 77 err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger) 78 Expect(err).ToNot(HaveOccurred()) 79 } 80 81 Context("when the build has succeeded", func() { 82 It("does not remove the image resource cache", func() { 83 setBuildStatus(db.BuildStatusSucceeded) 84 Expect(countResourceCaches()).ToNot(BeZero()) 85 86 err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 87 Expect(err).ToNot(HaveOccurred()) 88 89 Expect(countResourceCaches()).ToNot(BeZero()) 90 }) 91 }) 92 93 Context("when build has not succeeded", func() { 94 It("does not removes the image resource cache", func() { 95 setBuildStatus(db.BuildStatusFailed) 96 Expect(countResourceCaches()).ToNot(BeZero()) 97 98 err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 99 Expect(err).ToNot(HaveOccurred()) 100 101 Expect(countResourceCaches()).ToNot(BeZero()) 102 }) 103 }) 104 }) 105 }) 106 107 Context("when its a build of a job in a pipeline", func() { 108 Context("when the cache is for a saved image resource version for a finished build", func() { 109 setBuildStatus := func(a db.BuildStatus) (db.UsedResourceCache, db.Build) { 110 resourceCache, build := resourceCacheForJobBuild() 111 Expect(build.JobID()).ToNot(BeZero()) 112 113 err := build.SaveImageResourceVersion(resourceCache) 114 Expect(err).ToNot(HaveOccurred()) 115 116 err = build.SetInterceptible(false) 117 Expect(err).ToNot(HaveOccurred()) 118 119 err = build.Finish(a) 120 Expect(err).ToNot(HaveOccurred()) 121 122 err = resourceCacheLifecycle.CleanUsesForFinishedBuilds(logger) 123 Expect(err).ToNot(HaveOccurred()) 124 return resourceCache, build 125 } 126 127 Context("when the build has succeeded", func() { 128 It("does not remove the resource cache for the most recent build", func() { 129 setBuildStatus(db.BuildStatusSucceeded) 130 Expect(countResourceCaches()).To(Equal(1)) 131 132 err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 133 Expect(err).ToNot(HaveOccurred()) 134 135 Expect(countResourceCaches()).To(Equal(1)) 136 }) 137 138 It("removes resource caches for previous finished builds", func() { 139 resourceCache1, _ := setBuildStatus(db.BuildStatusFailed) 140 resourceCache2, _ := setBuildStatus(db.BuildStatusSucceeded) 141 Expect(resourceCache1.ID()).ToNot(Equal(resourceCache2.ID())) 142 143 Expect(countResourceCaches()).To(Equal(2)) 144 145 err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 146 Expect(err).ToNot(HaveOccurred()) 147 148 Expect(countResourceCaches()).To(Equal(1)) 149 }) 150 It("does not remove the resource caches from other jobs", func() { 151 By("creating a second pipeline") 152 secondPipeline, _, err := defaultTeam.SavePipeline(atc.PipelineRef{Name: "second-pipeline"}, atc.Config{ 153 Jobs: atc.JobConfigs{ 154 { 155 Name: "some-job", 156 }, 157 }, 158 Resources: atc.ResourceConfigs{ 159 { 160 Name: "some-resource", 161 Type: "some-base-resource-type", 162 Source: atc.Source{ 163 "some": "source", 164 }, 165 }, 166 }, 167 ResourceTypes: atc.ResourceTypes{ 168 { 169 Name: "some-type", 170 Type: "some-base-resource-type", 171 Source: atc.Source{ 172 "some-type": "source", 173 }, 174 }, 175 }, 176 }, db.ConfigVersion(0), false) 177 Expect(err).NotTo(HaveOccurred()) 178 179 By("creating an image resource cache tied to the job in the second pipeline") 180 job, _, err := secondPipeline.Job("some-job") 181 Expect(err).ToNot(HaveOccurred()) 182 build, err := job.CreateBuild() 183 Expect(err).ToNot(HaveOccurred()) 184 resourceCache := createResourceCacheWithUser(db.ForBuild(build.ID())) 185 186 err = build.SaveImageResourceVersion(resourceCache) 187 Expect(err).ToNot(HaveOccurred()) 188 189 err = build.SetInterceptible(false) 190 Expect(err).ToNot(HaveOccurred()) 191 192 By("creating an image resource cached in the default pipeline") 193 setBuildStatus(db.BuildStatusSucceeded) 194 195 Expect(countResourceCaches()).To(Equal(2)) 196 }) 197 }) 198 199 Context("when build has not succeeded", func() { 200 It("does not remove the image resource cache", func() { 201 setBuildStatus(db.BuildStatusFailed) 202 Expect(countResourceCaches()).ToNot(BeZero()) 203 204 err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 205 Expect(err).ToNot(HaveOccurred()) 206 207 Expect(countResourceCaches()).ToNot(BeZero()) 208 }) 209 }) 210 }) 211 }) 212 }) 213 214 Context("the resource cache is used by a container", func() { 215 var ( 216 container db.CreatingContainer 217 containerOwner db.ContainerOwner 218 ) 219 220 BeforeEach(func() { 221 var err error 222 resourceConfig, err := resourceConfigFactory.FindOrCreateResourceConfig( 223 "some-base-resource-type", 224 atc.Source{ 225 "some": "source", 226 }, 227 atc.VersionedResourceTypes{}, 228 ) 229 Expect(err).ToNot(HaveOccurred()) 230 231 containerOwner = db.NewResourceConfigCheckSessionContainerOwner( 232 resourceConfig.ID(), 233 resourceConfig.OriginBaseResourceType().ID, 234 db.ContainerOwnerExpiries{}, 235 ) 236 237 container, err = defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{}) 238 Expect(err).ToNot(HaveOccurred()) 239 240 _ = createResourceCacheWithUser(db.ForContainer(container.ID())) 241 }) 242 243 Context("and the container still exists", func() { 244 BeforeEach(func() { 245 err := resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 246 Expect(err).ToNot(HaveOccurred()) 247 }) 248 249 It("doesn't delete the resource cache", func() { 250 Expect(countResourceCaches()).ToNot(BeZero()) 251 }) 252 }) 253 254 Context("and the container has been deleted", func() { 255 BeforeEach(func() { 256 createdContainer, err := container.Created() 257 Expect(err).ToNot(HaveOccurred()) 258 259 destroyingContainer, err := createdContainer.Destroying() 260 Expect(err).ToNot(HaveOccurred()) 261 262 _, err = destroyingContainer.Destroy() 263 Expect(err).ToNot(HaveOccurred()) 264 265 err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 266 Expect(err).ToNot(HaveOccurred()) 267 }) 268 269 It("deletes the resource cache", func() { 270 Expect(countResourceCaches()).To(BeZero()) 271 }) 272 }) 273 }) 274 275 Context("when the cache is for a custom resource type", func() { 276 It("does not remove the cache if the type is still configured", func() { 277 _, err := resourceConfigFactory.FindOrCreateResourceConfig( 278 "some-type", 279 atc.Source{ 280 "some": "source", 281 }, 282 atc.VersionedResourceTypes{ 283 atc.VersionedResourceType{ 284 ResourceType: atc.ResourceType{ 285 Name: "some-type", 286 Type: "some-base-resource-type", 287 Source: atc.Source{ 288 "some": "source", 289 }, 290 }, 291 Version: atc.Version{"showme": "whatyougot"}, 292 }, 293 }, 294 ) 295 Expect(err).ToNot(HaveOccurred()) 296 297 Expect(countResourceCaches()).ToNot(BeZero()) 298 err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 299 Expect(err).ToNot(HaveOccurred()) 300 301 Expect(countResourceCaches()).ToNot(BeZero()) 302 }) 303 304 It("removes the cache if the type is no longer configured", func() { 305 _, err := resourceConfigFactory.FindOrCreateResourceConfig( 306 "some-type", 307 atc.Source{ 308 "some": "source", 309 }, 310 atc.VersionedResourceTypes{ 311 atc.VersionedResourceType{ 312 ResourceType: atc.ResourceType{ 313 Name: "some-type", 314 Type: "some-base-resource-type", 315 Source: atc.Source{ 316 "some": "source", 317 }, 318 }, 319 Version: atc.Version{"showme": "whatyougot"}, 320 }, 321 }, 322 ) 323 Expect(err).ToNot(HaveOccurred()) 324 325 err = resourceConfigCheckSessionLifecycle.CleanInactiveResourceConfigCheckSessions() 326 Expect(err).ToNot(HaveOccurred()) 327 328 err = resourceConfigFactory.CleanUnreferencedConfigs(0) 329 Expect(err).ToNot(HaveOccurred()) 330 331 Expect(countResourceCaches()).ToNot(BeZero()) 332 333 err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 334 Expect(err).ToNot(HaveOccurred()) 335 336 Expect(countResourceCaches()).To(BeZero()) 337 }) 338 }) 339 340 Context("when the cache is for a resource version used as an input for the next build of a job", func() { 341 It("does not remove the cache", func() { 342 scenario := dbtest.Setup( 343 builder.WithPipeline(atc.Config{ 344 Resources: atc.ResourceConfigs{ 345 { 346 Name: "some-resource", 347 Type: "some-base-resource-type", 348 Source: atc.Source{ 349 "some": "source", 350 }, 351 }, 352 }, 353 }), 354 builder.WithResourceVersions("some-resource", atc.Version{"some": "version"}), 355 ) 356 357 rc, found, err := resourceConfigFactory.FindResourceConfigByID(scenario.Resource("some-resource").ResourceConfigID()) 358 Expect(found).To(BeTrue()) 359 Expect(err).ToNot(HaveOccurred()) 360 361 resourceConfigScope, err := rc.FindOrCreateScope(scenario.Resource("some-resource")) 362 Expect(err).ToNot(HaveOccurred()) 363 364 containerOwner := db.NewResourceConfigCheckSessionContainerOwner( 365 resourceConfigScope.ResourceConfig().ID(), 366 resourceConfigScope.ResourceConfig().OriginBaseResourceType().ID, 367 db.ContainerOwnerExpiries{}, 368 ) 369 370 container, err := defaultWorker.CreateContainer(containerOwner, db.ContainerMetadata{}) 371 Expect(err).ToNot(HaveOccurred()) 372 373 _ = createResourceCacheWithUser(db.ForContainer(container.ID())) 374 375 resourceConfigVersion, found, err := resourceConfigScope.FindVersion(atc.Version{"some": "version"}) 376 Expect(found).To(BeTrue()) 377 Expect(err).ToNot(HaveOccurred()) 378 379 err = defaultJob.SaveNextInputMapping(db.InputMapping{ 380 "some-resource": db.InputResult{ 381 Input: &db.AlgorithmInput{ 382 AlgorithmVersion: db.AlgorithmVersion{ 383 Version: db.ResourceVersion(convertToMD5(atc.Version(resourceConfigVersion.Version()))), 384 ResourceID: scenario.Resource("some-resource").ID(), 385 }, 386 }, 387 PassedBuildIDs: []int{}, 388 }, 389 }, true) 390 Expect(err).ToNot(HaveOccurred()) 391 392 createdContainer, err := container.Created() 393 Expect(err).ToNot(HaveOccurred()) 394 395 destroyingContainer, err := createdContainer.Destroying() 396 Expect(err).ToNot(HaveOccurred()) 397 398 _, err = destroyingContainer.Destroy() 399 Expect(err).ToNot(HaveOccurred()) 400 401 Expect(countResourceCaches()).ToNot(BeZero()) 402 403 err = resourceCacheLifecycle.CleanUpInvalidCaches(logger.Session("resource-cache-lifecycle")) 404 Expect(err).ToNot(HaveOccurred()) 405 406 Expect(countResourceCaches()).ToNot(BeZero()) 407 }) 408 }) 409 }) 410 }) 411 412 func countResourceCaches() int { 413 var result int 414 err := psql.Select("count(*)"). 415 From("resource_caches"). 416 RunWith(dbConn). 417 QueryRow(). 418 Scan(&result) 419 Expect(err).ToNot(HaveOccurred()) 420 421 return result 422 423 } 424 425 func createResourceCacheWithUser(resourceCacheUser db.ResourceCacheUser) db.UsedResourceCache { 426 usedResourceCache, err := resourceCacheFactory.FindOrCreateResourceCache( 427 resourceCacheUser, 428 "some-base-resource-type", 429 atc.Version{"some": "version"}, 430 atc.Source{ 431 "some": "source", 432 }, 433 atc.Params{"some": fmt.Sprintf("param-%d", time.Now().UnixNano())}, 434 atc.VersionedResourceTypes{}, 435 ) 436 Expect(err).ToNot(HaveOccurred()) 437 438 return usedResourceCache 439 }