github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/pipeline.go (about) 1 package db 2 3 import ( 4 "database/sql" 5 "encoding/json" 6 "fmt" 7 "strconv" 8 "strings" 9 "time" 10 11 "code.cloudfoundry.org/lager" 12 13 sq "github.com/Masterminds/squirrel" 14 "github.com/pkg/errors" 15 16 "github.com/pf-qiu/concourse/v6/atc" 17 "github.com/pf-qiu/concourse/v6/atc/creds" 18 "github.com/pf-qiu/concourse/v6/atc/db/lock" 19 "github.com/pf-qiu/concourse/v6/atc/event" 20 "github.com/pf-qiu/concourse/v6/vars" 21 ) 22 23 type ErrResourceNotFound struct { 24 Name string 25 } 26 27 func (e ErrResourceNotFound) Error() string { 28 return fmt.Sprintf("resource '%s' not found", e.Name) 29 } 30 31 //go:generate counterfeiter . Pipeline 32 33 type Cause struct { 34 ResourceVersionID int `json:"resource_version_id"` 35 BuildID int `json:"build_id"` 36 } 37 38 type Pipeline interface { 39 ID() int 40 Name() string 41 TeamID() int 42 TeamName() string 43 InstanceVars() atc.InstanceVars 44 ParentJobID() int 45 ParentBuildID() int 46 Groups() atc.GroupConfigs 47 VarSources() atc.VarSourceConfigs 48 Display() *atc.DisplayConfig 49 ConfigVersion() ConfigVersion 50 Config() (atc.Config, error) 51 Public() bool 52 Paused() bool 53 Archived() bool 54 LastUpdated() time.Time 55 56 CheckPaused() (bool, error) 57 Reload() (bool, error) 58 59 Causality(versionedResourceID int) ([]Cause, error) 60 ResourceVersion(resourceConfigVersionID int) (atc.ResourceVersion, bool, error) 61 62 GetBuildsWithVersionAsInput(int, int) ([]Build, error) 63 GetBuildsWithVersionAsOutput(int, int) ([]Build, error) 64 Builds(page Page) ([]Build, Pagination, error) 65 66 CreateOneOffBuild() (Build, error) 67 CreateStartedBuild(plan atc.Plan) (Build, error) 68 69 BuildsWithTime(page Page) ([]Build, Pagination, error) 70 71 DeleteBuildEventsByBuildIDs(buildIDs []int) error 72 73 LoadDebugVersionsDB() (*atc.DebugVersionsDB, error) 74 75 Resource(name string) (Resource, bool, error) 76 ResourceByID(id int) (Resource, bool, error) 77 Resources() (Resources, error) 78 79 ResourceTypes() (ResourceTypes, error) 80 ResourceType(name string) (ResourceType, bool, error) 81 ResourceTypeByID(id int) (ResourceType, bool, error) 82 83 Job(name string) (Job, bool, error) 84 Jobs() (Jobs, error) 85 Dashboard() ([]atc.JobSummary, error) 86 87 Expose() error 88 Hide() error 89 90 Pause() error 91 Unpause() error 92 93 Archive() error 94 95 Destroy() error 96 Rename(string) error 97 98 Variables(lager.Logger, creds.Secrets, creds.VarSourcePool) (vars.Variables, error) 99 100 SetParentIDs(jobID, buildID int) error 101 } 102 103 type pipeline struct { 104 id int 105 name string 106 teamID int 107 teamName string 108 instanceVars atc.InstanceVars 109 parentJobID int 110 parentBuildID int 111 groups atc.GroupConfigs 112 varSources atc.VarSourceConfigs 113 display *atc.DisplayConfig 114 configVersion ConfigVersion 115 paused bool 116 public bool 117 archived bool 118 lastUpdated time.Time 119 120 conn Conn 121 lockFactory lock.LockFactory 122 } 123 124 // ConfigVersion is a sequence identifier used for compare-and-swap. 125 type ConfigVersion int 126 127 var pipelinesQuery = psql.Select(` 128 p.id, 129 p.name, 130 p.groups, 131 p.var_sources, 132 p.display, 133 p.nonce, 134 p.version, 135 p.team_id, 136 t.name, 137 p.paused, 138 p.public, 139 p.archived, 140 p.last_updated, 141 p.parent_job_id, 142 p.parent_build_id, 143 p.instance_vars 144 `). 145 From("pipelines p"). 146 LeftJoin("teams t ON p.team_id = t.id") 147 148 func newPipeline(conn Conn, lockFactory lock.LockFactory) *pipeline { 149 return &pipeline{ 150 conn: conn, 151 lockFactory: lockFactory, 152 } 153 } 154 155 func (p *pipeline) ID() int { return p.id } 156 func (p *pipeline) Name() string { return p.name } 157 func (p *pipeline) TeamID() int { return p.teamID } 158 func (p *pipeline) TeamName() string { return p.teamName } 159 func (p *pipeline) ParentJobID() int { return p.parentJobID } 160 func (p *pipeline) ParentBuildID() int { return p.parentBuildID } 161 func (p *pipeline) InstanceVars() atc.InstanceVars { return p.instanceVars } 162 func (p *pipeline) Groups() atc.GroupConfigs { return p.groups } 163 164 func (p *pipeline) VarSources() atc.VarSourceConfigs { return p.varSources } 165 func (p *pipeline) Display() *atc.DisplayConfig { return p.display } 166 func (p *pipeline) ConfigVersion() ConfigVersion { return p.configVersion } 167 func (p *pipeline) Public() bool { return p.public } 168 func (p *pipeline) Paused() bool { return p.paused } 169 func (p *pipeline) Archived() bool { return p.archived } 170 func (p *pipeline) LastUpdated() time.Time { return p.lastUpdated } 171 172 // IMPORTANT: This method is broken with the new resource config versions changes 173 func (p *pipeline) Causality(versionedResourceID int) ([]Cause, error) { 174 rows, err := p.conn.Query(` 175 WITH RECURSIVE causality(versioned_resource_id, build_id) AS ( 176 SELECT bi.versioned_resource_id, bi.build_id 177 FROM build_inputs bi 178 WHERE bi.versioned_resource_id = $1 179 UNION 180 SELECT bi.versioned_resource_id, bi.build_id 181 FROM causality t 182 INNER JOIN build_outputs bo ON bo.build_id = t.build_id 183 INNER JOIN build_inputs bi ON bi.versioned_resource_id = bo.versioned_resource_id 184 INNER JOIN builds b ON b.id = bi.build_id 185 AND NOT EXISTS ( 186 SELECT 1 187 FROM build_outputs obo 188 INNER JOIN builds ob ON ob.id = obo.build_id 189 WHERE obo.build_id < bi.build_id 190 AND ob.job_id = b.job_id 191 AND obo.versioned_resource_id = bi.versioned_resource_id 192 ) 193 ) 194 SELECT c.versioned_resource_id, c.build_id 195 FROM causality c 196 INNER JOIN builds b ON b.id = c.build_id 197 ORDER BY b.start_time ASC, c.versioned_resource_id ASC 198 `, versionedResourceID) 199 if err != nil { 200 return nil, err 201 } 202 203 var causality []Cause 204 for rows.Next() { 205 var vrID, buildID int 206 err := rows.Scan(&vrID, &buildID) 207 if err != nil { 208 return nil, err 209 } 210 211 causality = append(causality, Cause{ 212 ResourceVersionID: vrID, 213 BuildID: buildID, 214 }) 215 } 216 217 return causality, nil 218 } 219 220 func (p *pipeline) CheckPaused() (bool, error) { 221 var paused bool 222 223 err := psql.Select("paused"). 224 From("pipelines"). 225 Where(sq.Eq{"id": p.id}). 226 RunWith(p.conn). 227 QueryRow(). 228 Scan(&paused) 229 230 if err != nil { 231 return false, err 232 } 233 234 return paused, nil 235 } 236 func (p *pipeline) Reload() (bool, error) { 237 row := pipelinesQuery.Where(sq.Eq{"p.id": p.id}). 238 RunWith(p.conn). 239 QueryRow() 240 241 err := scanPipeline(p, row) 242 if err != nil { 243 if err == sql.ErrNoRows { 244 return false, nil 245 } 246 return false, err 247 } 248 249 return true, nil 250 } 251 252 func (p *pipeline) Config() (atc.Config, error) { 253 jobs, err := p.Jobs() 254 if err != nil { 255 return atc.Config{}, fmt.Errorf("failed to get jobs: %w", err) 256 } 257 258 resources, err := p.Resources() 259 if err != nil { 260 return atc.Config{}, fmt.Errorf("failed to get resources: %w", err) 261 } 262 263 resourceTypes, err := p.ResourceTypes() 264 if err != nil { 265 return atc.Config{}, fmt.Errorf("failed to get resources-types: %w", err) 266 } 267 268 jobConfigs, err := jobs.Configs() 269 if err != nil { 270 return atc.Config{}, fmt.Errorf("failed to get job configs: %w", err) 271 } 272 273 config := atc.Config{ 274 Groups: p.Groups(), 275 VarSources: p.VarSources(), 276 Resources: resources.Configs(), 277 ResourceTypes: resourceTypes.Configs(), 278 Jobs: jobConfigs, 279 Display: p.Display(), 280 } 281 282 return config, nil 283 } 284 285 func (p *pipeline) CreateJobBuild(jobName string) (Build, error) { 286 tx, err := p.conn.Begin() 287 if err != nil { 288 return nil, err 289 } 290 291 defer Rollback(tx) 292 293 buildName, jobID, err := getNewBuildNameForJob(tx, jobName, p.id) 294 if err != nil { 295 return nil, err 296 } 297 298 var buildID int 299 err = psql.Insert("builds"). 300 Columns("name", "job_id", "team_id", "status", "manually_triggered"). 301 Values(buildName, jobID, p.teamID, "pending", true). 302 Suffix("RETURNING id"). 303 RunWith(tx). 304 QueryRow(). 305 Scan(&buildID) 306 if err != nil { 307 return nil, err 308 } 309 310 build := newEmptyBuild(p.conn, p.lockFactory) 311 err = scanBuild(build, buildsQuery. 312 Where(sq.Eq{"b.id": buildID}). 313 RunWith(tx). 314 QueryRow(), 315 p.conn.EncryptionStrategy(), 316 ) 317 if err != nil { 318 return nil, err 319 } 320 321 err = createBuildEventSeq(tx, buildID) 322 if err != nil { 323 return nil, err 324 } 325 326 err = tx.Commit() 327 if err != nil { 328 return nil, err 329 } 330 331 return build, nil 332 } 333 334 // ResourceVersion is given a resource config version id and returns the 335 // resource version struct. This method is used by the API call 336 // GetResourceVersion to get all the attributes for that version of the 337 // resource. 338 func (p *pipeline) ResourceVersion(resourceConfigVersionID int) (atc.ResourceVersion, bool, error) { 339 rv := atc.ResourceVersion{} 340 var ( 341 versionBytes string 342 metadataBytes string 343 ) 344 345 enabled := ` 346 NOT EXISTS ( 347 SELECT 1 348 FROM resource_disabled_versions d, resources r 349 WHERE v.version_md5 = d.version_md5 350 AND r.resource_config_scope_id = v.resource_config_scope_id 351 AND r.id = d.resource_id 352 )` 353 354 err := psql.Select("v.id", "v.version", "v.metadata", enabled). 355 From("resource_config_versions v"). 356 Where(sq.Eq{ 357 "v.id": resourceConfigVersionID, 358 }). 359 RunWith(p.conn). 360 QueryRow(). 361 Scan(&rv.ID, &versionBytes, &metadataBytes, &rv.Enabled) 362 if err != nil { 363 if err == sql.ErrNoRows { 364 return atc.ResourceVersion{}, false, nil 365 } 366 367 return atc.ResourceVersion{}, false, err 368 } 369 370 err = json.Unmarshal([]byte(versionBytes), &rv.Version) 371 if err != nil { 372 return atc.ResourceVersion{}, false, err 373 } 374 375 err = json.Unmarshal([]byte(metadataBytes), &rv.Metadata) 376 if err != nil { 377 return atc.ResourceVersion{}, false, err 378 } 379 380 return rv, true, nil 381 } 382 383 func (p *pipeline) GetBuildsWithVersionAsInput(resourceID, resourceConfigVersionID int) ([]Build, error) { 384 rows, err := buildsQuery. 385 Join("build_resource_config_version_inputs bi ON bi.build_id = b.id"). 386 Join("resource_config_versions rcv ON rcv.version_md5 = bi.version_md5"). 387 Where(sq.Eq{ 388 "rcv.id": resourceConfigVersionID, 389 "bi.resource_id": resourceID, 390 }). 391 RunWith(p.conn). 392 Query() 393 if err != nil { 394 return nil, err 395 } 396 defer Close(rows) 397 398 builds := []Build{} 399 for rows.Next() { 400 build := newEmptyBuild(p.conn, p.lockFactory) 401 err = scanBuild(build, rows, p.conn.EncryptionStrategy()) 402 if err != nil { 403 return nil, err 404 } 405 builds = append(builds, build) 406 } 407 408 return builds, err 409 } 410 411 func (p *pipeline) GetBuildsWithVersionAsOutput(resourceID, resourceConfigVersionID int) ([]Build, error) { 412 rows, err := buildsQuery. 413 Join("build_resource_config_version_outputs bo ON bo.build_id = b.id"). 414 Join("resource_config_versions rcv ON rcv.version_md5 = bo.version_md5"). 415 Where(sq.Eq{ 416 "rcv.id": resourceConfigVersionID, 417 "bo.resource_id": resourceID, 418 }). 419 RunWith(p.conn). 420 Query() 421 if err != nil { 422 return nil, err 423 } 424 defer Close(rows) 425 426 builds := []Build{} 427 for rows.Next() { 428 build := newEmptyBuild(p.conn, p.lockFactory) 429 err = scanBuild(build, rows, p.conn.EncryptionStrategy()) 430 if err != nil { 431 return nil, err 432 } 433 434 builds = append(builds, build) 435 } 436 437 return builds, err 438 } 439 440 func (p *pipeline) Resource(name string) (Resource, bool, error) { 441 return p.resource(sq.Eq{ 442 "r.pipeline_id": p.id, 443 "r.name": name, 444 }) 445 } 446 447 func (p *pipeline) ResourceByID(id int) (Resource, bool, error) { 448 return p.resource(sq.Eq{ 449 "r.pipeline_id": p.id, 450 "r.id": id, 451 }) 452 } 453 454 func (p *pipeline) resource(where map[string]interface{}) (Resource, bool, error) { 455 row := resourcesQuery. 456 Where(where). 457 RunWith(p.conn). 458 QueryRow() 459 460 resource := newEmptyResource(p.conn, p.lockFactory) 461 err := scanResource(resource, row) 462 if err != nil { 463 if err == sql.ErrNoRows { 464 return nil, false, nil 465 } 466 467 return nil, false, err 468 } 469 470 return resource, true, nil 471 } 472 473 func (p *pipeline) Builds(page Page) ([]Build, Pagination, error) { 474 return getBuildsWithPagination( 475 buildsQuery.Where(sq.Eq{"b.pipeline_id": p.id}), minMaxIdQuery, page, p.conn, p.lockFactory) 476 } 477 478 func (p *pipeline) BuildsWithTime(page Page) ([]Build, Pagination, error) { 479 return getBuildsWithDates( 480 buildsQuery.Where(sq.Eq{"b.pipeline_id": p.id}), minMaxIdQuery, page, p.conn, p.lockFactory) 481 } 482 483 func (p *pipeline) Resources() (Resources, error) { 484 return resources(p.id, p.conn, p.lockFactory) 485 } 486 487 func (p *pipeline) ResourceTypes() (ResourceTypes, error) { 488 rows, err := resourceTypesQuery. 489 Where(sq.Eq{"r.pipeline_id": p.id}). 490 OrderBy("r.name"). 491 RunWith(p.conn). 492 Query() 493 if err != nil { 494 return nil, err 495 } 496 defer Close(rows) 497 498 resourceTypes := []ResourceType{} 499 500 for rows.Next() { 501 resourceType := newEmptyResourceType(p.conn, p.lockFactory) 502 err := scanResourceType(resourceType, rows) 503 if err != nil { 504 return nil, err 505 } 506 507 resourceTypes = append(resourceTypes, resourceType) 508 } 509 510 return resourceTypes, nil 511 } 512 513 func (p *pipeline) ResourceType(name string) (ResourceType, bool, error) { 514 return p.resourceType(sq.Eq{ 515 "r.pipeline_id": p.id, 516 "r.name": name, 517 }) 518 } 519 520 func (p *pipeline) ResourceTypeByID(id int) (ResourceType, bool, error) { 521 return p.resourceType(sq.Eq{ 522 "r.pipeline_id": p.id, 523 "r.id": id, 524 }) 525 } 526 527 func (p *pipeline) resourceType(where map[string]interface{}) (ResourceType, bool, error) { 528 row := resourceTypesQuery. 529 Where(where). 530 RunWith(p.conn). 531 QueryRow() 532 533 resourceType := newEmptyResourceType(p.conn, p.lockFactory) 534 err := scanResourceType(resourceType, row) 535 if err != nil { 536 if err == sql.ErrNoRows { 537 return nil, false, nil 538 } 539 540 return nil, false, err 541 } 542 543 return resourceType, true, nil 544 } 545 546 func (p *pipeline) Job(name string) (Job, bool, error) { 547 row := jobsQuery.Where(sq.Eq{ 548 "j.name": name, 549 "j.active": true, 550 "j.pipeline_id": p.id, 551 }).RunWith(p.conn).QueryRow() 552 553 job := newEmptyJob(p.conn, p.lockFactory) 554 err := scanJob(job, row) 555 556 if err != nil { 557 if err == sql.ErrNoRows { 558 return nil, false, nil 559 } 560 561 return nil, false, err 562 } 563 564 return job, true, nil 565 } 566 567 func (p *pipeline) Jobs() (Jobs, error) { 568 rows, err := jobsQuery. 569 Where(sq.Eq{ 570 "pipeline_id": p.id, 571 "active": true, 572 }). 573 OrderBy("j.id ASC"). 574 RunWith(p.conn). 575 Query() 576 if err != nil { 577 return nil, err 578 } 579 580 jobs, err := scanJobs(p.conn, p.lockFactory, rows) 581 return jobs, err 582 } 583 584 func (p *pipeline) Dashboard() ([]atc.JobSummary, error) { 585 tx, err := p.conn.Begin() 586 if err != nil { 587 return nil, err 588 } 589 590 defer Rollback(tx) 591 592 dashboardFactory := newDashboardFactory(tx, sq.Eq{ 593 "j.pipeline_id": p.id, 594 }) 595 596 dashboard, err := dashboardFactory.buildDashboard() 597 if err != nil { 598 return nil, err 599 } 600 601 err = tx.Commit() 602 if err != nil { 603 return nil, err 604 } 605 606 return dashboard, nil 607 } 608 609 func (p *pipeline) Pause() error { 610 _, err := psql.Update("pipelines"). 611 Set("paused", true). 612 Where(sq.Eq{ 613 "id": p.id, 614 }). 615 RunWith(p.conn). 616 Exec() 617 618 return err 619 } 620 621 func (p *pipeline) Unpause() error { 622 tx, err := p.conn.Begin() 623 if err != nil { 624 return err 625 } 626 627 defer Rollback(tx) 628 629 _, err = psql.Update("pipelines"). 630 Set("paused", false). 631 Where(sq.Eq{ 632 "id": p.id, 633 }). 634 RunWith(tx). 635 Exec() 636 637 if err != nil { 638 return err 639 } 640 641 err = requestScheduleForJobsInPipeline(tx, p.id) 642 if err != nil { 643 return err 644 } 645 646 return tx.Commit() 647 } 648 649 func (p *pipeline) Archive() error { 650 tx, err := p.conn.Begin() 651 if err != nil { 652 return err 653 } 654 655 defer Rollback(tx) 656 err = p.archive(tx) 657 if err != nil { 658 return err 659 } 660 661 return tx.Commit() 662 } 663 664 func (p *pipeline) archive(tx Tx) error { 665 _, err := psql.Update("pipelines"). 666 Set("archived", true). 667 Set("last_updated", sq.Expr("now()")). 668 Set("paused", true). 669 Set("version", 0). 670 Where(sq.Eq{ 671 "id": p.id, 672 }). 673 RunWith(tx). 674 Exec() 675 676 if err != nil { 677 return err 678 } 679 680 err = p.clearConfigForJobsInPipeline(tx) 681 if err != nil { 682 return err 683 } 684 685 err = p.clearConfigForResourcesInPipeline(tx) 686 if err != nil { 687 return err 688 } 689 690 return p.clearConfigForResourceTypesInPipeline(tx) 691 } 692 693 func (p *pipeline) Hide() error { 694 _, err := psql.Update("pipelines"). 695 Set("public", false). 696 Where(sq.Eq{ 697 "id": p.id, 698 }). 699 RunWith(p.conn). 700 Exec() 701 702 return err 703 } 704 705 func (p *pipeline) Expose() error { 706 _, err := psql.Update("pipelines"). 707 Set("public", true). 708 Where(sq.Eq{ 709 "id": p.id, 710 }). 711 RunWith(p.conn). 712 Exec() 713 714 return err 715 } 716 717 func (p *pipeline) Rename(name string) error { 718 _, err := psql.Update("pipelines"). 719 Set("name", name). 720 Where(sq.Eq{ 721 "id": p.id, 722 }). 723 RunWith(p.conn). 724 Exec() 725 726 return err 727 } 728 729 func (p *pipeline) Destroy() error { 730 tx, err := p.conn.Begin() 731 if err != nil { 732 return err 733 } 734 735 defer tx.Rollback() 736 737 _, err = psql.Delete("pipelines"). 738 Where(sq.Eq{ 739 "id": p.id, 740 }). 741 RunWith(tx). 742 Exec() 743 744 if err != nil { 745 return err 746 } 747 748 _, err = psql.Insert("deleted_pipelines"). 749 Columns("id"). 750 Values(p.id). 751 RunWith(tx). 752 Exec() 753 754 if err != nil { 755 return err 756 } 757 758 return tx.Commit() 759 } 760 761 func (p *pipeline) LoadDebugVersionsDB() (*atc.DebugVersionsDB, error) { 762 db := &atc.DebugVersionsDB{ 763 BuildOutputs: []atc.DebugBuildOutput{}, 764 BuildInputs: []atc.DebugBuildInput{}, 765 ResourceVersions: []atc.DebugResourceVersion{}, 766 BuildReruns: []atc.DebugBuildRerun{}, 767 Resources: []atc.DebugResource{}, 768 Jobs: []atc.DebugJob{}, 769 } 770 771 tx, err := p.conn.Begin() 772 if err != nil { 773 return nil, err 774 } 775 776 defer tx.Rollback() 777 778 rows, err := psql.Select("v.id, v.check_order, r.id, v.resource_config_scope_id, o.build_id, b.job_id"). 779 From("build_resource_config_version_outputs o"). 780 Join("builds b ON b.id = o.build_id"). 781 Join("resource_config_versions v ON v.version_md5 = o.version_md5"). 782 Join("resources r ON r.id = o.resource_id"). 783 Where(sq.Expr("r.resource_config_scope_id = v.resource_config_scope_id")). 784 Where(sq.Expr("(r.id, v.version_md5) NOT IN (SELECT resource_id, version_md5 from resource_disabled_versions)")). 785 Where(sq.Eq{ 786 "b.status": BuildStatusSucceeded, 787 "r.pipeline_id": p.id, 788 "r.active": true, 789 }). 790 RunWith(tx). 791 Query() 792 if err != nil { 793 return nil, err 794 } 795 796 defer Close(rows) 797 798 for rows.Next() { 799 var output atc.DebugBuildOutput 800 err = rows.Scan(&output.VersionID, &output.CheckOrder, &output.ResourceID, &output.ScopeID, &output.BuildID, &output.JobID) 801 if err != nil { 802 return nil, err 803 } 804 805 output.DebugResourceVersion.CheckOrder = output.CheckOrder 806 807 db.BuildOutputs = append(db.BuildOutputs, output) 808 } 809 810 rows, err = psql.Select("v.id, v.check_order, r.id, v.resource_config_scope_id, i.build_id, i.name, b.job_id, b.status = 'succeeded'"). 811 From("build_resource_config_version_inputs i"). 812 Join("builds b ON b.id = i.build_id"). 813 Join("resource_config_versions v ON v.version_md5 = i.version_md5"). 814 Join("resources r ON r.id = i.resource_id"). 815 Where(sq.Expr("r.resource_config_scope_id = v.resource_config_scope_id")). 816 Where(sq.Expr("(r.id, v.version_md5) NOT IN (SELECT resource_id, version_md5 from resource_disabled_versions)")). 817 Where(sq.Eq{ 818 "r.pipeline_id": p.id, 819 "r.active": true, 820 }). 821 RunWith(tx). 822 Query() 823 if err != nil { 824 return nil, err 825 } 826 827 defer Close(rows) 828 829 for rows.Next() { 830 var succeeded bool 831 832 var input atc.DebugBuildInput 833 err = rows.Scan(&input.VersionID, &input.CheckOrder, &input.ResourceID, &input.ScopeID, &input.BuildID, &input.InputName, &input.JobID, &succeeded) 834 if err != nil { 835 return nil, err 836 } 837 838 input.DebugResourceVersion.CheckOrder = input.CheckOrder 839 840 db.BuildInputs = append(db.BuildInputs, input) 841 842 if succeeded { 843 // implicit output 844 db.BuildOutputs = append(db.BuildOutputs, atc.DebugBuildOutput{ 845 DebugResourceVersion: input.DebugResourceVersion, 846 JobID: input.JobID, 847 BuildID: input.BuildID, 848 }) 849 } 850 } 851 852 rows, err = psql.Select("v.id, v.check_order, r.id, v.resource_config_scope_id"). 853 From("resource_config_versions v"). 854 Join("resources r ON r.resource_config_scope_id = v.resource_config_scope_id"). 855 LeftJoin("resource_disabled_versions d ON d.resource_id = r.id AND d.version_md5 = v.version_md5"). 856 Where(sq.Eq{ 857 "r.pipeline_id": p.id, 858 "r.active": true, 859 "d.resource_id": nil, 860 "d.version_md5": nil, 861 }). 862 RunWith(tx). 863 Query() 864 if err != nil { 865 return nil, err 866 } 867 868 defer Close(rows) 869 870 for rows.Next() { 871 var output atc.DebugResourceVersion 872 err = rows.Scan(&output.VersionID, &output.CheckOrder, &output.ResourceID, &output.ScopeID) 873 if err != nil { 874 return nil, err 875 } 876 877 db.ResourceVersions = append(db.ResourceVersions, output) 878 } 879 880 rows, err = psql.Select("j.id, b.id, b.rerun_of"). 881 From("builds b"). 882 Join("jobs j ON j.id = b.job_id"). 883 Where(sq.Eq{ 884 "j.active": true, 885 "b.pipeline_id": p.id, 886 }). 887 Where(sq.NotEq{ 888 "b.rerun_of": nil, 889 }). 890 RunWith(tx). 891 Query() 892 if err != nil { 893 return nil, err 894 } 895 896 defer Close(rows) 897 898 for rows.Next() { 899 var rerun atc.DebugBuildRerun 900 err = rows.Scan(&rerun.JobID, &rerun.BuildID, &rerun.RerunOf) 901 if err != nil { 902 return nil, err 903 } 904 905 db.BuildReruns = append(db.BuildReruns, rerun) 906 } 907 908 rows, err = psql.Select("j.name, j.id"). 909 From("jobs j"). 910 Where(sq.Eq{ 911 "j.pipeline_id": p.id, 912 "j.active": true, 913 }). 914 RunWith(tx). 915 Query() 916 if err != nil { 917 return nil, err 918 } 919 920 defer Close(rows) 921 922 for rows.Next() { 923 var job atc.DebugJob 924 err = rows.Scan(&job.Name, &job.ID) 925 if err != nil { 926 return nil, err 927 } 928 929 db.Jobs = append(db.Jobs, job) 930 } 931 932 rows, err = psql.Select("r.name, r.id, r.resource_config_scope_id"). 933 From("resources r"). 934 Where(sq.Eq{ 935 "r.pipeline_id": p.id, 936 "r.active": true, 937 }). 938 RunWith(tx). 939 Query() 940 if err != nil { 941 return nil, err 942 } 943 944 defer Close(rows) 945 946 for rows.Next() { 947 var scopeID sql.NullInt64 948 var resource atc.DebugResource 949 err = rows.Scan(&resource.Name, &resource.ID, &scopeID) 950 if err != nil { 951 return nil, err 952 } 953 954 if scopeID.Valid { 955 i := int(scopeID.Int64) 956 resource.ScopeID = &i 957 } 958 959 db.Resources = append(db.Resources, resource) 960 } 961 962 err = tx.Commit() 963 if err != nil { 964 return nil, err 965 } 966 967 return db, nil 968 } 969 970 func (p *pipeline) DeleteBuildEventsByBuildIDs(buildIDs []int) error { 971 if len(buildIDs) == 0 { 972 return nil 973 } 974 975 interfaceBuildIDs := make([]interface{}, len(buildIDs)) 976 for i, buildID := range buildIDs { 977 interfaceBuildIDs[i] = buildID 978 } 979 980 indexStrings := make([]string, len(buildIDs)) 981 for i := range indexStrings { 982 indexStrings[i] = "$" + strconv.Itoa(i+1) 983 } 984 985 tx, err := p.conn.Begin() 986 if err != nil { 987 return err 988 } 989 990 defer Rollback(tx) 991 992 _, err = tx.Exec(` 993 DELETE FROM build_events 994 WHERE build_id IN (`+strings.Join(indexStrings, ",")+`) 995 `, interfaceBuildIDs...) 996 if err != nil { 997 return err 998 } 999 1000 _, err = tx.Exec(` 1001 UPDATE builds 1002 SET reap_time = now() 1003 WHERE id IN (`+strings.Join(indexStrings, ",")+`) 1004 `, interfaceBuildIDs...) 1005 if err != nil { 1006 return err 1007 } 1008 1009 err = tx.Commit() 1010 return err 1011 } 1012 1013 func (p *pipeline) CreateOneOffBuild() (Build, error) { 1014 tx, err := p.conn.Begin() 1015 if err != nil { 1016 return nil, err 1017 } 1018 1019 defer Rollback(tx) 1020 1021 build := newEmptyBuild(p.conn, p.lockFactory) 1022 err = createBuild(tx, build, map[string]interface{}{ 1023 "name": sq.Expr("nextval('one_off_name')"), 1024 "pipeline_id": p.id, 1025 "team_id": p.teamID, 1026 "status": BuildStatusPending, 1027 }) 1028 if err != nil { 1029 return nil, err 1030 } 1031 1032 err = tx.Commit() 1033 if err != nil { 1034 return nil, err 1035 } 1036 1037 return build, nil 1038 } 1039 1040 func (p *pipeline) CreateStartedBuild(plan atc.Plan) (Build, error) { 1041 tx, err := p.conn.Begin() 1042 if err != nil { 1043 return nil, err 1044 } 1045 1046 defer Rollback(tx) 1047 1048 metadata, err := json.Marshal(plan) 1049 if err != nil { 1050 return nil, err 1051 } 1052 1053 encryptedPlan, nonce, err := p.conn.EncryptionStrategy().Encrypt(metadata) 1054 if err != nil { 1055 return nil, err 1056 } 1057 1058 build := newEmptyBuild(p.conn, p.lockFactory) 1059 err = createBuild(tx, build, map[string]interface{}{ 1060 "name": sq.Expr("nextval('one_off_name')"), 1061 "pipeline_id": p.id, 1062 "team_id": p.teamID, 1063 "status": BuildStatusStarted, 1064 "start_time": sq.Expr("now()"), 1065 "schema": schema, 1066 "private_plan": encryptedPlan, 1067 "public_plan": plan.Public(), 1068 "nonce": nonce, 1069 }) 1070 if err != nil { 1071 return nil, err 1072 } 1073 1074 err = build.saveEvent(tx, event.Status{ 1075 Status: atc.StatusStarted, 1076 Time: build.StartTime().Unix(), 1077 }) 1078 if err != nil { 1079 return nil, err 1080 } 1081 1082 err = tx.Commit() 1083 if err != nil { 1084 return nil, err 1085 } 1086 1087 if err = p.conn.Bus().Notify(buildStartedChannel()); err != nil { 1088 return nil, err 1089 } 1090 1091 if err = p.conn.Bus().Notify(buildEventsChannel(build.id)); err != nil { 1092 return nil, err 1093 } 1094 1095 return build, nil 1096 } 1097 1098 func (p *pipeline) getBuildsFrom(tx Tx, col string) (map[string]Build, error) { 1099 rows, err := buildsQuery. 1100 Where(sq.Eq{ 1101 "b.pipeline_id": p.id, 1102 }). 1103 Where(sq.Expr("j." + col + " = b.id")). 1104 RunWith(tx).Query() 1105 if err != nil { 1106 return nil, err 1107 } 1108 1109 defer Close(rows) 1110 1111 nextBuilds := make(map[string]Build) 1112 1113 for rows.Next() { 1114 build := newEmptyBuild(p.conn, p.lockFactory) 1115 err := scanBuild(build, rows, p.conn.EncryptionStrategy()) 1116 if err != nil { 1117 return nil, err 1118 } 1119 nextBuilds[build.JobName()] = build 1120 } 1121 1122 return nextBuilds, nil 1123 } 1124 1125 // Variables creates variables for this pipeline. If this pipeline has its own 1126 // var_sources, a vars.MultiVars containing all pipeline specific var_sources 1127 // plug the global variables, otherwise just return the global variables. 1128 func (p *pipeline) Variables(logger lager.Logger, globalSecrets creds.Secrets, varSourcePool creds.VarSourcePool) (vars.Variables, error) { 1129 globalVars := creds.NewVariables(globalSecrets, p.TeamName(), p.Name(), false) 1130 namedVarsMap := vars.NamedVariables{} 1131 1132 // It's safe to add NamedVariables to allVars via an array here, because 1133 // a map is passed by reference. 1134 allVars := vars.NewMultiVars([]vars.Variables{namedVarsMap, globalVars}) 1135 1136 orderedVarSources, err := p.varSources.OrderByDependency() 1137 if err != nil { 1138 return nil, err 1139 } 1140 1141 for _, cm := range orderedVarSources { 1142 factory := creds.ManagerFactories()[cm.Type] 1143 if factory == nil { 1144 return nil, fmt.Errorf("unknown credential manager type: %s", cm.Type) 1145 } 1146 1147 // Interpolate variables in pipeline credential manager's config 1148 newConfig, err := creds.NewParams(allVars, atc.Params{"config": cm.Config}).Evaluate() 1149 if err != nil { 1150 return nil, errors.Wrapf(err, "evaluate var_source '%s' error", cm.Name) 1151 } 1152 1153 config, ok := newConfig["config"].(map[string]interface{}) 1154 if !ok { 1155 return nil, fmt.Errorf("var_source '%s' invalid config", cm.Name) 1156 } 1157 secrets, err := varSourcePool.FindOrCreate(logger, config, factory) 1158 if err != nil { 1159 return nil, errors.Wrapf(err, "create var_source '%s' error", cm.Name) 1160 } 1161 namedVarsMap[cm.Name] = creds.NewVariables(secrets, p.TeamName(), p.Name(), true) 1162 } 1163 1164 // If there is no var_source from the pipeline, then just return the global 1165 // vars. 1166 if len(namedVarsMap) == 0 { 1167 return globalVars, nil 1168 } 1169 1170 return allVars, nil 1171 } 1172 1173 func (p *pipeline) SetParentIDs(jobID, buildID int) error { 1174 if jobID <= 0 || buildID <= 0 { 1175 return errors.New("job and build id cannot be negative or zero-value") 1176 } 1177 1178 tx, err := p.conn.Begin() 1179 if err != nil { 1180 return err 1181 } 1182 1183 defer Rollback(tx) 1184 1185 result, err := psql.Update("pipelines"). 1186 Set("parent_job_id", jobID). 1187 Set("parent_build_id", buildID). 1188 Where(sq.Eq{ 1189 "id": p.id, 1190 }). 1191 Where(sq.Or{sq.Lt{"parent_build_id": buildID}, sq.Eq{"parent_build_id": nil}}). 1192 RunWith(tx). 1193 Exec() 1194 1195 if err != nil { 1196 return err 1197 } 1198 1199 rows, err := result.RowsAffected() 1200 if err != nil { 1201 return err 1202 } 1203 if rows == 0 { 1204 return ErrSetByNewerBuild 1205 } 1206 1207 return tx.Commit() 1208 } 1209 1210 func getNewBuildNameForJob(tx Tx, jobName string, pipelineID int) (string, int, error) { 1211 var buildName string 1212 var jobID int 1213 err := tx.QueryRow(` 1214 UPDATE jobs 1215 SET build_number_seq = build_number_seq + 1 1216 WHERE name = $1 AND pipeline_id = $2 1217 RETURNING build_number_seq, id 1218 `, jobName, pipelineID).Scan(&buildName, &jobID) 1219 return buildName, jobID, err 1220 } 1221 1222 func resources(pipelineID int, conn Conn, lockFactory lock.LockFactory) (Resources, error) { 1223 rows, err := resourcesQuery. 1224 Where(sq.Eq{"r.pipeline_id": pipelineID}). 1225 OrderBy("r.name"). 1226 RunWith(conn). 1227 Query() 1228 if err != nil { 1229 return nil, err 1230 } 1231 defer Close(rows) 1232 1233 var resources Resources 1234 1235 for rows.Next() { 1236 newResource := newEmptyResource(conn, lockFactory) 1237 err := scanResource(newResource, rows) 1238 if err != nil { 1239 return nil, err 1240 } 1241 1242 resources = append(resources, newResource) 1243 } 1244 1245 return resources, nil 1246 } 1247 1248 // The SELECT query orders the jobs for updating to prevent deadlocking. 1249 // Updating multiple rows using a SELECT subquery does not preserve the same 1250 // order for the updates, which can lead to deadlocking. 1251 func requestScheduleForJobsInPipeline(tx Tx, pipelineID int) error { 1252 rows, err := psql.Select("id"). 1253 From("jobs"). 1254 Where(sq.Eq{ 1255 "pipeline_id": pipelineID, 1256 }). 1257 OrderBy("id DESC"). 1258 RunWith(tx). 1259 Query() 1260 if err != nil { 1261 return err 1262 } 1263 1264 var jobIDs []int 1265 for rows.Next() { 1266 var id int 1267 err = rows.Scan(&id) 1268 if err != nil { 1269 return err 1270 } 1271 1272 jobIDs = append(jobIDs, id) 1273 } 1274 1275 for _, jID := range jobIDs { 1276 _, err := psql.Update("jobs"). 1277 Set("schedule_requested", sq.Expr("now()")). 1278 Where(sq.Eq{ 1279 "id": jID, 1280 }). 1281 RunWith(tx). 1282 Exec() 1283 if err != nil { 1284 return err 1285 } 1286 } 1287 1288 return nil 1289 } 1290 1291 func (p *pipeline) clearConfigForJobsInPipeline(tx Tx) error { 1292 _, err := psql.Update("jobs"). 1293 Set("config", nil). 1294 Set("nonce", nil). 1295 Where(sq.Eq{"pipeline_id": p.id}). 1296 RunWith(tx). 1297 Exec() 1298 if err != nil { 1299 return err 1300 } 1301 1302 return nil 1303 } 1304 1305 func (p *pipeline) clearConfigForResourcesInPipeline(tx Tx) error { 1306 _, err := psql.Update("resources"). 1307 Set("config", nil). 1308 Set("nonce", nil). 1309 Where(sq.Eq{"pipeline_id": p.id}). 1310 RunWith(tx). 1311 Exec() 1312 if err != nil { 1313 return err 1314 } 1315 1316 return nil 1317 } 1318 1319 func (p *pipeline) clearConfigForResourceTypesInPipeline(tx Tx) error { 1320 _, err := psql.Update("resource_types"). 1321 Set("config", nil). 1322 Set("nonce", nil). 1323 Where(sq.Eq{"pipeline_id": p.id}). 1324 RunWith(tx). 1325 Exec() 1326 if err != nil { 1327 return err 1328 } 1329 1330 return nil 1331 }