github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/migration/migrate_job_configs_to_job_pipes_test.go (about) 1 package migration_test 2 3 import ( 4 "database/sql" 5 6 "github.com/pf-qiu/concourse/v6/atc" 7 . "github.com/onsi/ginkgo" 8 . "github.com/onsi/gomega" 9 ) 10 11 var _ = Describe("Migrate existing job configs into job pipes", func() { 12 const preMigrationVersion = 1579713198 13 const postMigrationVersion = 1579713199 14 15 var ( 16 db *sql.DB 17 ) 18 19 Context("Up", func() { 20 It("migrates the job configs for all the jobs into the job inputs table, job outputs table and the new config fields in the jobs table", func() { 21 db = postgresRunner.OpenDBAtVersion(preMigrationVersion) 22 23 _, err := db.Exec(` 24 INSERT INTO teams(id, name) VALUES 25 (1, 'some-team') 26 `) 27 Expect(err).NotTo(HaveOccurred()) 28 29 _, err = db.Exec(` 30 INSERT INTO pipelines(id, team_id, name, groups) VALUES 31 (1, 1, 'pipeline1', '[{"name":"group1","jobs":["job1","job2"]},{"name":"group2","jobs":["job2","job3"]}]'), 32 (2, 1, 'pipeline2', '[{"name":"group2","jobs":["job1"]}]') 33 `) 34 Expect(err).NotTo(HaveOccurred()) 35 36 _, err = db.Exec(` 37 INSERT INTO jobs(id, pipeline_id, name, active, config) VALUES 38 (1, 1, 'job1', true, '{"name":"job1","plan":[{"get":"resource-1"}]}'), 39 (2, 1, 'job2', true, '{"name":"job2","public":true,"disable_manual_trigger":true,"serial_groups":["serial-1","serial-2"],"plan":[{"get":"resource-1","passed":["job1"],"trigger":true,"version":"latest"},{"get":"resource-2"}]}'), 40 (3, 1, 'job3', true, '{"name":"job3","max_in_flight":5,"disable_manual_trigger":false,"plan":[{"get":"res1","resource":"resource-1","passed":["job1","job2"],"version":{"ver":"1"}},{"get":"resource-2","passed":["job2"],"trigger":false}]}'), 41 (4, 2, 'job1', true, '{"name":"job1","serial":true,"plan":[{"put":"resource-3"}]}'), 42 (5, 2, 'job2', true, '{"name":"job2","public":false,"serial_groups":["serial-1"],"plan":[{"get":"resource-1","version":"every"},{"put":"res-1","resource":"resource-1"}]}'), 43 (6, 2, 'job3', false, '{"name":"job3","serial_groups":["serial-1"],"plan":[{"get":"resource-2"},{"put":"resource-1"}]}') 44 `) 45 Expect(err).NotTo(HaveOccurred()) 46 47 _, err = db.Exec(` 48 INSERT INTO jobs_serial_groups(serial_group, job_id) VALUES 49 ('serial-group-1', 1), 50 ('serial-1', 2), 51 ('serial-5', 5) 52 `) 53 Expect(err).NotTo(HaveOccurred()) 54 55 var resource1ID, resource2ID, resource3ID, resource4ID int 56 57 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-1', 1, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource1ID) 58 Expect(err).NotTo(HaveOccurred()) 59 60 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-2', 1, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource2ID) 61 Expect(err).NotTo(HaveOccurred()) 62 63 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-3', 2, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource3ID) 64 Expect(err).NotTo(HaveOccurred()) 65 66 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-1', 2, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource4ID) 67 Expect(err).NotTo(HaveOccurred()) 68 69 db.Close() 70 71 db = postgresRunner.OpenDBAtVersion(postMigrationVersion) 72 73 type jobConfigFields struct { 74 jobID int 75 public bool 76 disableManualTrigger bool 77 maxInFlight int 78 } 79 80 rows, err := db.Query(`SELECT id, public, disable_manual_trigger, max_in_flight FROM jobs WHERE active = true`) 81 Expect(err).NotTo(HaveOccurred()) 82 83 var jobs []jobConfigFields 84 for rows.Next() { 85 job := jobConfigFields{} 86 87 err := rows.Scan(&job.jobID, &job.public, &job.disableManualTrigger, &job.maxInFlight) 88 Expect(err).NotTo(HaveOccurred()) 89 90 jobs = append(jobs, job) 91 } 92 93 expectedJobConfigFields := []jobConfigFields{ 94 { 95 jobID: 1, 96 }, 97 { 98 jobID: 2, 99 public: true, 100 disableManualTrigger: true, 101 maxInFlight: 1, 102 }, 103 { 104 jobID: 3, 105 disableManualTrigger: false, 106 maxInFlight: 5, 107 }, 108 { 109 jobID: 4, 110 maxInFlight: 1, 111 }, 112 { 113 jobID: 5, 114 public: false, 115 maxInFlight: 1, 116 }, 117 } 118 119 Expect(jobs).To(HaveLen(5)) 120 Expect(jobs).To(ConsistOf(expectedJobConfigFields)) 121 122 rows, err = db.Query(`SELECT name, job_id, resource_id, passed_job_id, trigger, version FROM job_inputs`) 123 Expect(err).NotTo(HaveOccurred()) 124 125 type jobInput struct { 126 name string 127 jobID int 128 resourceID int 129 passedJob int 130 trigger bool 131 version *atc.VersionConfig 132 } 133 134 var jobInputs []jobInput 135 for rows.Next() { 136 ji := jobInput{} 137 var passedJobID sql.NullInt64 138 var versionString sql.NullString 139 140 err := rows.Scan(&ji.name, &ji.jobID, &ji.resourceID, &passedJobID, &ji.trigger, &versionString) 141 Expect(err).NotTo(HaveOccurred()) 142 143 if versionString.Valid { 144 version := &atc.VersionConfig{} 145 err = version.UnmarshalJSON([]byte(versionString.String)) 146 Expect(err).NotTo(HaveOccurred()) 147 148 ji.version = version 149 } 150 151 if passedJobID.Valid { 152 ji.passedJob = int(passedJobID.Int64) 153 } 154 155 jobInputs = append(jobInputs, ji) 156 } 157 158 expectedJobInputs := []jobInput{ 159 { 160 name: "resource-1", 161 jobID: 1, 162 resourceID: resource1ID, 163 }, 164 { 165 name: "resource-1", 166 jobID: 2, 167 resourceID: resource1ID, 168 passedJob: 1, 169 trigger: true, 170 version: &atc.VersionConfig{Latest: true}, 171 }, 172 { 173 name: "resource-2", 174 jobID: 2, 175 resourceID: resource2ID, 176 }, 177 { 178 name: "res1", 179 jobID: 3, 180 resourceID: resource1ID, 181 passedJob: 1, 182 version: &atc.VersionConfig{Pinned: atc.Version{"ver": "1"}}, 183 }, 184 { 185 name: "res1", 186 jobID: 3, 187 resourceID: resource1ID, 188 passedJob: 2, 189 version: &atc.VersionConfig{Pinned: atc.Version{"ver": "1"}}, 190 }, 191 { 192 name: "resource-2", 193 jobID: 3, 194 resourceID: resource2ID, 195 passedJob: 2, 196 trigger: false, 197 }, 198 { 199 name: "resource-1", 200 jobID: 5, 201 resourceID: resource4ID, 202 version: &atc.VersionConfig{Every: true}, 203 }, 204 } 205 206 Expect(jobInputs).To(HaveLen(7)) 207 Expect(jobInputs).To(ConsistOf(expectedJobInputs)) 208 209 rows, err = db.Query(`SELECT name, job_id, resource_id FROM job_outputs`) 210 Expect(err).NotTo(HaveOccurred()) 211 212 type jobOutput struct { 213 name string 214 jobID int 215 resourceID int 216 } 217 218 var jobOutputs []jobOutput 219 for rows.Next() { 220 jo := jobOutput{} 221 222 err := rows.Scan(&jo.name, &jo.jobID, &jo.resourceID) 223 Expect(err).NotTo(HaveOccurred()) 224 225 jobOutputs = append(jobOutputs, jo) 226 } 227 228 expectedJobOutputs := []jobOutput{ 229 { 230 name: "resource-3", 231 jobID: 4, 232 resourceID: resource3ID, 233 }, 234 { 235 name: "res-1", 236 jobID: 5, 237 resourceID: resource4ID, 238 }, 239 } 240 241 Expect(jobOutputs).To(HaveLen(2)) 242 Expect(jobOutputs).To(ConsistOf(expectedJobOutputs)) 243 244 rows, err = db.Query(`SELECT job_id, serial_group FROM jobs_serial_groups`) 245 Expect(err).NotTo(HaveOccurred()) 246 247 type serialGroup struct { 248 jobID int 249 serialGroup string 250 } 251 252 var serialGroups []serialGroup 253 for rows.Next() { 254 sg := serialGroup{} 255 256 err := rows.Scan(&sg.jobID, &sg.serialGroup) 257 Expect(err).NotTo(HaveOccurred()) 258 259 serialGroups = append(serialGroups, sg) 260 } 261 262 _ = db.Close() 263 264 expectedSerialGroups := []serialGroup{ 265 { 266 jobID: 2, 267 serialGroup: "serial-1", 268 }, 269 { 270 jobID: 2, 271 serialGroup: "serial-2", 272 }, 273 { 274 jobID: 3, 275 serialGroup: "job3", 276 }, 277 { 278 jobID: 4, 279 serialGroup: "job1", 280 }, 281 { 282 jobID: 5, 283 serialGroup: "serial-1", 284 }, 285 } 286 287 Expect(serialGroups).To(HaveLen(5)) 288 Expect(serialGroups).To(ConsistOf(expectedSerialGroups)) 289 }) 290 }) 291 292 Context("Down", func() { 293 It("truncates the job inputs and outputs table", func() { 294 db = postgresRunner.OpenDBAtVersion(postMigrationVersion) 295 296 _, err := db.Exec(` 297 INSERT INTO teams(id, name) VALUES 298 (1, 'some-team') 299 `) 300 Expect(err).NotTo(HaveOccurred()) 301 302 _, err = db.Exec(` 303 INSERT INTO pipelines(id, team_id, name, groups) VALUES 304 (1, 1, 'pipeline1', '[{"name":"group1","jobs":["job1","job2"]},{"name":"group2","jobs":["job2","job3"]}]'), 305 (2, 1, 'pipeline2', '[{"name":"group2","jobs":["job1"]}]') 306 `) 307 Expect(err).NotTo(HaveOccurred()) 308 309 _, err = db.Exec(` 310 INSERT INTO jobs(id, pipeline_id, name, config) VALUES 311 (1, 1, 'job1', '{"name":"job1","plan":[{"get":"resource-1"}]}'), 312 (2, 1, 'job2', '{"name":"job2","plan":[{"get":"resource-1","passed":["job1"]},{"get":"resource-2"}]}'), 313 (3, 1, 'job3', '{"name":"job3","plan":[{"get":"res1","resource":"resource-1","passed":["job1","job2"]},{"get":"resource-2","passed":["job2"]}]}'), 314 (4, 2, 'job1', '{"name":"job1","plan":[{"put":"resource-3"}]}'), 315 (5, 2, 'job2', '{"name":"job2","plan":[{"get":"resource-1"}]}') 316 `) 317 Expect(err).NotTo(HaveOccurred()) 318 319 var resource1ID, resource2ID, resource3ID, resource4ID int 320 321 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-1', 1, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource1ID) 322 Expect(err).NotTo(HaveOccurred()) 323 324 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-2', 1, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource2ID) 325 Expect(err).NotTo(HaveOccurred()) 326 327 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-3', 2, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource3ID) 328 Expect(err).NotTo(HaveOccurred()) 329 330 err = db.QueryRow(`INSERT INTO resources(name, pipeline_id, config, active, type) VALUES('resource-1', 2, '{"type": "some-type"}', true, 'some-type') RETURNING id`).Scan(&resource4ID) 331 Expect(err).NotTo(HaveOccurred()) 332 333 _, err = db.Exec(` 334 INSERT INTO job_inputs(name, job_id, resource_id, passed_job_id, trigger, version) VALUES 335 ('resource-1', 1, 1, NULL, false, NULL), 336 ('resource-1', 2, 1, 1, false, 'latest'), 337 ('resource-2', 2, 2, NULL, true, NULL), 338 ('res1', 3, 1, 1, false, 'every'), 339 ('res1', 3, 1, 2, false, '{"ver":"1"}'), 340 ('resource-2', 3, 2, 2, true, NULL), 341 ('resource-1', 5, 4, NULL, false, NULL) 342 `) 343 Expect(err).NotTo(HaveOccurred()) 344 345 _, err = db.Exec(` 346 INSERT INTO job_outputs(name, job_id, resource_id) VALUES 347 ('resource-1', 1, 1), 348 ('resource-1', 2, 1), 349 ('resource-2', 2, 2) 350 `) 351 Expect(err).NotTo(HaveOccurred()) 352 353 db.Close() 354 355 db = postgresRunner.OpenDBAtVersion(preMigrationVersion) 356 357 var inputsRowNumber int 358 err = db.QueryRow(`SELECT COUNT(*) FROM job_inputs`).Scan(&inputsRowNumber) 359 Expect(err).NotTo(HaveOccurred()) 360 361 var outputsRowNumber int 362 err = db.QueryRow(`SELECT COUNT(*) FROM job_inputs`).Scan(&outputsRowNumber) 363 Expect(err).NotTo(HaveOccurred()) 364 365 _ = db.Close() 366 367 Expect(inputsRowNumber).To(BeZero()) 368 Expect(outputsRowNumber).To(BeZero()) 369 }) 370 }) 371 })