github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/job_factory.go (about) 1 package db 2 3 import ( 4 "database/sql" 5 "encoding/json" 6 "sort" 7 8 sq "github.com/Masterminds/squirrel" 9 "github.com/pf-qiu/concourse/v6/atc" 10 "github.com/pf-qiu/concourse/v6/atc/db/lock" 11 "github.com/lib/pq" 12 ) 13 14 //go:generate counterfeiter . JobFactory 15 16 // XXX: This job factory object is not really a job factory anymore. It is 17 // holding the responsibility for two very different things: constructing a 18 // dashboard object and also a scheduler job object. Figure out what this is 19 // trying to encapsulate or considering splitting this out! 20 type JobFactory interface { 21 VisibleJobs([]string) ([]atc.JobSummary, error) 22 AllActiveJobs() ([]atc.JobSummary, error) 23 JobsToSchedule() (SchedulerJobs, error) 24 } 25 26 type jobFactory struct { 27 conn Conn 28 lockFactory lock.LockFactory 29 } 30 31 func NewJobFactory(conn Conn, lockFactory lock.LockFactory) JobFactory { 32 return &jobFactory{ 33 conn: conn, 34 lockFactory: lockFactory, 35 } 36 } 37 38 type SchedulerJobs []SchedulerJob 39 40 type SchedulerJob struct { 41 Job 42 Resources SchedulerResources 43 ResourceTypes atc.VersionedResourceTypes 44 } 45 46 type SchedulerResources []SchedulerResource 47 48 type SchedulerResource struct { 49 Name string 50 Type string 51 Source atc.Source 52 } 53 54 func (r *SchedulerResource) ApplySourceDefaults(resourceTypes atc.VersionedResourceTypes) { 55 parentType, found := resourceTypes.Lookup(r.Type) 56 if found { 57 r.Source = parentType.Defaults.Merge(r.Source) 58 } else { 59 defaults, found := atc.FindBaseResourceTypeDefaults(r.Type) 60 if found { 61 r.Source = defaults.Merge(r.Source) 62 } 63 } 64 } 65 66 func (resources SchedulerResources) Lookup(name string) (*SchedulerResource, bool) { 67 for _, resource := range resources { 68 if resource.Name == name { 69 return &resource, true 70 } 71 } 72 73 return nil, false 74 } 75 76 func (j *jobFactory) JobsToSchedule() (SchedulerJobs, error) { 77 tx, err := j.conn.Begin() 78 if err != nil { 79 return nil, err 80 } 81 82 defer tx.Rollback() 83 84 rows, err := jobsQuery. 85 Where(sq.Expr("j.schedule_requested > j.last_scheduled")). 86 Where(sq.Eq{ 87 "j.active": true, 88 "j.paused": false, 89 "p.paused": false, 90 }). 91 RunWith(tx). 92 Query() 93 if err != nil { 94 return nil, err 95 } 96 97 jobs, err := scanJobs(j.conn, j.lockFactory, rows) 98 if err != nil { 99 return nil, err 100 } 101 102 var schedulerJobs SchedulerJobs 103 pipelineResourceTypes := make(map[int]ResourceTypes) 104 for _, job := range jobs { 105 rows, err := tx.Query(`WITH inputs AS ( 106 SELECT ji.resource_id from job_inputs ji where ji.job_id = $1 107 UNION 108 SELECT jo.resource_id from job_outputs jo where jo.job_id = $1 109 ) 110 SELECT r.name, r.type, r.config, r.nonce 111 From resources r 112 Join inputs i on i.resource_id = r.id`, job.ID()) 113 if err != nil { 114 return nil, err 115 } 116 117 var schedulerResources SchedulerResources 118 for rows.Next() { 119 var name, type_ string 120 var configBlob []byte 121 var nonce sql.NullString 122 123 err = rows.Scan(&name, &type_, &configBlob, &nonce) 124 if err != nil { 125 return nil, err 126 } 127 128 defer Close(rows) 129 130 es := j.conn.EncryptionStrategy() 131 132 var noncense *string 133 if nonce.Valid { 134 noncense = &nonce.String 135 } 136 137 decryptedConfig, err := es.Decrypt(string(configBlob), noncense) 138 if err != nil { 139 return nil, err 140 } 141 142 var config atc.ResourceConfig 143 err = json.Unmarshal(decryptedConfig, &config) 144 if err != nil { 145 return nil, err 146 } 147 148 schedulerResources = append(schedulerResources, SchedulerResource{ 149 Name: name, 150 Type: type_, 151 Source: config.Source, 152 }) 153 } 154 155 var resourceTypes ResourceTypes 156 var found bool 157 resourceTypes, found = pipelineResourceTypes[job.PipelineID()] 158 if !found { 159 rows, err := resourceTypesQuery. 160 Where(sq.Eq{"r.pipeline_id": job.PipelineID()}). 161 OrderBy("r.name"). 162 RunWith(tx). 163 Query() 164 if err != nil { 165 return nil, err 166 } 167 168 defer Close(rows) 169 170 for rows.Next() { 171 resourceType := newEmptyResourceType(j.conn, j.lockFactory) 172 err := scanResourceType(resourceType, rows) 173 if err != nil { 174 return nil, err 175 } 176 177 resourceTypes = append(resourceTypes, resourceType) 178 } 179 180 pipelineResourceTypes[job.PipelineID()] = resourceTypes 181 } 182 183 schedulerJobs = append(schedulerJobs, SchedulerJob{ 184 Job: job, 185 Resources: schedulerResources, 186 ResourceTypes: resourceTypes.Deserialize(), 187 }) 188 } 189 190 err = tx.Commit() 191 if err != nil { 192 return nil, err 193 } 194 195 return schedulerJobs, nil 196 } 197 198 func (j *jobFactory) VisibleJobs(teamNames []string) ([]atc.JobSummary, error) { 199 tx, err := j.conn.Begin() 200 if err != nil { 201 return nil, err 202 } 203 204 defer Rollback(tx) 205 206 dashboardFactory := newDashboardFactory(tx, sq.Or{ 207 sq.Eq{"tm.name": teamNames}, 208 sq.Eq{"p.public": true}, 209 }) 210 211 dashboard, err := dashboardFactory.buildDashboard() 212 if err != nil { 213 return nil, err 214 } 215 216 err = tx.Commit() 217 if err != nil { 218 return nil, err 219 } 220 221 return dashboard, nil 222 } 223 224 func (j *jobFactory) AllActiveJobs() ([]atc.JobSummary, error) { 225 tx, err := j.conn.Begin() 226 if err != nil { 227 return nil, err 228 } 229 230 defer Rollback(tx) 231 232 dashboardFactory := newDashboardFactory(tx, nil) 233 dashboard, err := dashboardFactory.buildDashboard() 234 if err != nil { 235 return nil, err 236 } 237 238 err = tx.Commit() 239 if err != nil { 240 return nil, err 241 } 242 243 return dashboard, nil 244 } 245 246 type dashboardFactory struct { 247 // Constraints that are used by the dashboard queries. For example, a job ID 248 // constraint so that the dashboard will only return the job I have access to 249 // see. 250 pred interface{} 251 252 tx Tx 253 } 254 255 func newDashboardFactory(tx Tx, pred interface{}) dashboardFactory { 256 return dashboardFactory{ 257 pred: pred, 258 tx: tx, 259 } 260 } 261 262 func (d dashboardFactory) buildDashboard() ([]atc.JobSummary, error) { 263 dashboard, err := d.constructJobsForDashboard() 264 if err != nil { 265 return nil, err 266 } 267 268 jobInputs, err := d.fetchJobInputs() 269 if err != nil { 270 return nil, err 271 } 272 273 jobOutputs, err := d.fetchJobOutputs() 274 if err != nil { 275 return nil, err 276 } 277 278 return d.combineJobInputsAndOutputsWithDashboardJobs(dashboard, jobInputs, jobOutputs), nil 279 } 280 281 func (d dashboardFactory) constructJobsForDashboard() ([]atc.JobSummary, error) { 282 rows, err := psql.Select("j.id", "j.name", "p.id", "p.name", "p.instance_vars", "j.paused", "j.has_new_inputs", "j.tags", "tm.name", 283 "l.id", "l.name", "l.status", "l.start_time", "l.end_time", 284 "n.id", "n.name", "n.status", "n.start_time", "n.end_time", 285 "t.id", "t.name", "t.status", "t.start_time", "t.end_time"). 286 From("jobs j"). 287 Join("pipelines p ON j.pipeline_id = p.id"). 288 Join("teams tm ON p.team_id = tm.id"). 289 LeftJoin("builds l on j.latest_completed_build_id = l.id"). 290 LeftJoin("builds n on j.next_build_id = n.id"). 291 LeftJoin("builds t on j.transition_build_id = t.id"). 292 Where(sq.Eq{ 293 "j.active": true, 294 }). 295 Where(d.pred). 296 OrderBy("j.id ASC"). 297 RunWith(d.tx). 298 Query() 299 if err != nil { 300 return nil, err 301 } 302 303 type nullableBuild struct { 304 id sql.NullInt64 305 name sql.NullString 306 jobName sql.NullString 307 status sql.NullString 308 startTime pq.NullTime 309 endTime pq.NullTime 310 } 311 312 var dashboard []atc.JobSummary 313 for rows.Next() { 314 var ( 315 f, n, t nullableBuild 316 317 pipelineInstanceVars sql.NullString 318 ) 319 320 j := atc.JobSummary{} 321 err = rows.Scan(&j.ID, &j.Name, &j.PipelineID, &j.PipelineName, &pipelineInstanceVars, &j.Paused, &j.HasNewInputs, pq.Array(&j.Groups), &j.TeamName, 322 &f.id, &f.name, &f.status, &f.startTime, &f.endTime, 323 &n.id, &n.name, &n.status, &n.startTime, &n.endTime, 324 &t.id, &t.name, &t.status, &t.startTime, &t.endTime) 325 if err != nil { 326 return nil, err 327 } 328 329 if pipelineInstanceVars.Valid { 330 err = json.Unmarshal([]byte(pipelineInstanceVars.String), &j.PipelineInstanceVars) 331 if err != nil { 332 return nil, err 333 } 334 } 335 336 if f.id.Valid { 337 j.FinishedBuild = &atc.BuildSummary{ 338 ID: int(f.id.Int64), 339 Name: f.name.String, 340 JobName: j.Name, 341 PipelineID: j.PipelineID, 342 PipelineName: j.PipelineName, 343 PipelineInstanceVars: j.PipelineInstanceVars, 344 TeamName: j.TeamName, 345 Status: atc.BuildStatus(f.status.String), 346 StartTime: f.startTime.Time.Unix(), 347 EndTime: f.endTime.Time.Unix(), 348 } 349 } 350 351 if n.id.Valid { 352 j.NextBuild = &atc.BuildSummary{ 353 ID: int(n.id.Int64), 354 Name: n.name.String, 355 JobName: j.Name, 356 PipelineID: j.PipelineID, 357 PipelineName: j.PipelineName, 358 PipelineInstanceVars: j.PipelineInstanceVars, 359 TeamName: j.TeamName, 360 Status: atc.BuildStatus(n.status.String), 361 StartTime: n.startTime.Time.Unix(), 362 EndTime: n.endTime.Time.Unix(), 363 } 364 } 365 366 if t.id.Valid { 367 j.TransitionBuild = &atc.BuildSummary{ 368 ID: int(t.id.Int64), 369 Name: t.name.String, 370 JobName: j.Name, 371 PipelineID: j.PipelineID, 372 PipelineName: j.PipelineName, 373 PipelineInstanceVars: j.PipelineInstanceVars, 374 TeamName: j.TeamName, 375 Status: atc.BuildStatus(t.status.String), 376 StartTime: t.startTime.Time.Unix(), 377 EndTime: t.endTime.Time.Unix(), 378 } 379 } 380 381 dashboard = append(dashboard, j) 382 } 383 384 return dashboard, nil 385 } 386 387 func (d dashboardFactory) fetchJobInputs() (map[int][]atc.JobInputSummary, error) { 388 rows, err := psql.Select("j.id", "i.name", "r.name", "array_agg(jp.name ORDER BY jp.id)", "i.trigger"). 389 From("job_inputs i"). 390 Join("jobs j ON j.id = i.job_id"). 391 Join("pipelines p ON p.id = j.pipeline_id"). 392 Join("teams tm ON tm.id = p.team_id"). 393 Join("resources r ON r.id = i.resource_id"). 394 LeftJoin("jobs jp ON jp.id = i.passed_job_id"). 395 Where(sq.Eq{ 396 "j.active": true, 397 }). 398 Where(d.pred). 399 GroupBy("i.name, j.id, r.name, i.trigger"). 400 OrderBy("j.id"). 401 RunWith(d.tx). 402 Query() 403 if err != nil { 404 return nil, err 405 } 406 407 jobInputs := make(map[int][]atc.JobInputSummary) 408 for rows.Next() { 409 var passedString []sql.NullString 410 var inputName, resourceName string 411 var jobID int 412 var trigger bool 413 414 err = rows.Scan(&jobID, &inputName, &resourceName, pq.Array(&passedString), &trigger) 415 if err != nil { 416 return nil, err 417 } 418 419 var passed []string 420 for _, s := range passedString { 421 if s.Valid { 422 passed = append(passed, s.String) 423 } 424 } 425 426 jobInputs[jobID] = append(jobInputs[jobID], atc.JobInputSummary{ 427 Name: inputName, 428 Resource: resourceName, 429 Trigger: trigger, 430 Passed: passed, 431 }) 432 } 433 434 return jobInputs, nil 435 } 436 437 func (d dashboardFactory) fetchJobOutputs() (map[int][]atc.JobOutputSummary, error) { 438 rows, err := psql.Select("o.name", "r.name", "o.job_id"). 439 From("job_outputs o"). 440 Join("jobs j ON j.id = o.job_id"). 441 Join("pipelines p ON p.id = j.pipeline_id"). 442 Join("teams tm ON tm.id = p.team_id"). 443 Join("resources r ON r.id = o.resource_id"). 444 Where(d.pred). 445 Where(sq.Eq{ 446 "j.active": true, 447 }). 448 OrderBy("j.id"). 449 RunWith(d.tx). 450 Query() 451 if err != nil { 452 return nil, err 453 } 454 455 jobOutputs := make(map[int][]atc.JobOutputSummary) 456 for rows.Next() { 457 var output atc.JobOutputSummary 458 var jobID int 459 err = rows.Scan(&output.Name, &output.Resource, &jobID) 460 if err != nil { 461 return nil, err 462 } 463 464 jobOutputs[jobID] = append(jobOutputs[jobID], output) 465 } 466 467 return jobOutputs, err 468 } 469 470 func (d dashboardFactory) combineJobInputsAndOutputsWithDashboardJobs(dashboard []atc.JobSummary, jobInputs map[int][]atc.JobInputSummary, jobOutputs map[int][]atc.JobOutputSummary) []atc.JobSummary { 471 var finalDashboard []atc.JobSummary 472 for _, job := range dashboard { 473 for _, input := range jobInputs[job.ID] { 474 job.Inputs = append(job.Inputs, input) 475 } 476 477 sort.Slice(job.Inputs, func(p, q int) bool { 478 return job.Inputs[p].Name < job.Inputs[q].Name 479 }) 480 481 for _, output := range jobOutputs[job.ID] { 482 job.Outputs = append(job.Outputs, output) 483 } 484 485 sort.Slice(job.Outputs, func(p, q int) bool { 486 return job.Outputs[p].Name < job.Outputs[q].Name 487 }) 488 489 finalDashboard = append(finalDashboard, job) 490 } 491 492 return finalDashboard 493 }