github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/team.go (about) 1 package db 2 3 import ( 4 "database/sql" 5 "encoding/json" 6 "errors" 7 "fmt" 8 "time" 9 10 "code.cloudfoundry.org/lager" 11 12 sq "github.com/Masterminds/squirrel" 13 "github.com/gobwas/glob" 14 "github.com/lib/pq" 15 16 "github.com/pf-qiu/concourse/v6/atc" 17 "github.com/pf-qiu/concourse/v6/atc/creds" 18 "github.com/pf-qiu/concourse/v6/atc/db/lock" 19 "github.com/pf-qiu/concourse/v6/atc/event" 20 ) 21 22 var ErrConfigComparisonFailed = errors.New("comparison with existing config failed during save") 23 24 //go:generate counterfeiter . Team 25 26 type Team interface { 27 ID() int 28 Name() string 29 Admin() bool 30 31 Auth() atc.TeamAuth 32 33 Delete() error 34 Rename(string) error 35 36 SavePipeline( 37 pipelineRef atc.PipelineRef, 38 config atc.Config, 39 from ConfigVersion, 40 initiallyPaused bool, 41 ) (Pipeline, bool, error) 42 43 Pipeline(pipelineRef atc.PipelineRef) (Pipeline, bool, error) 44 Pipelines() ([]Pipeline, error) 45 PublicPipelines() ([]Pipeline, error) 46 OrderPipelines([]atc.PipelineRef) error 47 48 CreateOneOffBuild() (Build, error) 49 CreateStartedBuild(plan atc.Plan) (Build, error) 50 51 PrivateAndPublicBuilds(Page) ([]Build, Pagination, error) 52 Builds(page Page) ([]Build, Pagination, error) 53 BuildsWithTime(page Page) ([]Build, Pagination, error) 54 55 SaveWorker(atcWorker atc.Worker, ttl time.Duration) (Worker, error) 56 Workers() ([]Worker, error) 57 FindVolumeForWorkerArtifact(int) (CreatedVolume, bool, error) 58 59 Containers() ([]Container, error) 60 IsCheckContainer(string) (bool, error) 61 IsContainerWithinTeam(string, bool) (bool, error) 62 63 FindContainerByHandle(string) (Container, bool, error) 64 FindCheckContainers(lager.Logger, atc.PipelineRef, string, creds.Secrets, creds.VarSourcePool) ([]Container, map[int]time.Time, error) 65 FindContainersByMetadata(ContainerMetadata) ([]Container, error) 66 FindCreatedContainerByHandle(string) (CreatedContainer, bool, error) 67 FindWorkerForContainer(handle string) (Worker, bool, error) 68 FindWorkerForVolume(handle string) (Worker, bool, error) 69 70 UpdateProviderAuth(auth atc.TeamAuth) error 71 } 72 73 type team struct { 74 id int 75 conn Conn 76 lockFactory lock.LockFactory 77 78 name string 79 admin bool 80 81 auth atc.TeamAuth 82 } 83 84 func (t *team) ID() int { return t.id } 85 func (t *team) Name() string { return t.name } 86 func (t *team) Admin() bool { return t.admin } 87 88 func (t *team) Auth() atc.TeamAuth { return t.auth } 89 90 func (t *team) Delete() error { 91 _, err := psql.Delete("teams"). 92 Where(sq.Eq{ 93 "name": t.name, 94 }). 95 RunWith(t.conn). 96 Exec() 97 98 return err 99 } 100 101 func (t *team) Rename(name string) error { 102 _, err := psql.Update("teams"). 103 Set("name", name). 104 Where(sq.Eq{ 105 "id": t.id, 106 }). 107 RunWith(t.conn). 108 Exec() 109 110 return err 111 } 112 113 func (t *team) Workers() ([]Worker, error) { 114 return getWorkers(t.conn, workersQuery.Where(sq.Or{ 115 sq.Eq{"t.id": t.id}, 116 sq.Eq{"w.team_id": nil}, 117 })) 118 } 119 120 func (t *team) FindVolumeForWorkerArtifact(artifactID int) (CreatedVolume, bool, error) { 121 tx, err := t.conn.Begin() 122 if err != nil { 123 return nil, false, err 124 } 125 126 defer Rollback(tx) 127 128 artifact, found, err := getWorkerArtifact(tx, t.conn, artifactID) 129 if err != nil { 130 return nil, false, err 131 } 132 133 err = tx.Commit() 134 if err != nil { 135 return nil, false, err 136 } 137 138 if !found { 139 return nil, false, nil 140 } 141 142 return artifact.Volume(t.ID()) 143 } 144 145 func (t *team) FindWorkerForContainer(handle string) (Worker, bool, error) { 146 return getWorker(t.conn, workersQuery.Join("containers c ON c.worker_name = w.name").Where(sq.And{ 147 sq.Eq{"c.handle": handle}, 148 })) 149 } 150 151 func (t *team) FindWorkerForVolume(handle string) (Worker, bool, error) { 152 return getWorker(t.conn, workersQuery.Join("volumes v ON v.worker_name = w.name").Where(sq.And{ 153 sq.Eq{"v.handle": handle}, 154 })) 155 } 156 157 func (t *team) Containers() ([]Container, error) { 158 rows, err := selectContainers("c"). 159 Join("workers w ON c.worker_name = w.name"). 160 Join("resource_config_check_sessions rccs ON rccs.id = c.resource_config_check_session_id"). 161 Join("resources r ON r.resource_config_id = rccs.resource_config_id"). 162 Join("pipelines p ON p.id = r.pipeline_id"). 163 Where(sq.Eq{ 164 "p.team_id": t.id, 165 }). 166 Where(sq.Or{ 167 sq.Eq{ 168 "w.team_id": t.id, 169 }, sq.Eq{ 170 "w.team_id": nil, 171 }, 172 }). 173 Distinct(). 174 RunWith(t.conn). 175 Query() 176 if err != nil { 177 return nil, err 178 } 179 180 var containers []Container 181 containers, err = scanContainers(rows, t.conn, containers) 182 if err != nil { 183 return nil, err 184 } 185 186 rows, err = selectContainers("c"). 187 Join("workers w ON c.worker_name = w.name"). 188 Join("resource_config_check_sessions rccs ON rccs.id = c.resource_config_check_session_id"). 189 Join("resource_types rt ON rt.resource_config_id = rccs.resource_config_id"). 190 Join("pipelines p ON p.id = rt.pipeline_id"). 191 Where(sq.Eq{ 192 "p.team_id": t.id, 193 }). 194 Where(sq.Or{ 195 sq.Eq{ 196 "w.team_id": t.id, 197 }, sq.Eq{ 198 "w.team_id": nil, 199 }, 200 }). 201 Distinct(). 202 RunWith(t.conn). 203 Query() 204 if err != nil { 205 return nil, err 206 } 207 208 containers, err = scanContainers(rows, t.conn, containers) 209 if err != nil { 210 return nil, err 211 } 212 213 rows, err = selectContainers("c"). 214 Where(sq.Eq{ 215 "c.team_id": t.id, 216 }). 217 RunWith(t.conn). 218 Query() 219 if err != nil { 220 return nil, err 221 } 222 223 containers, err = scanContainers(rows, t.conn, containers) 224 if err != nil { 225 return nil, err 226 } 227 228 return containers, nil 229 } 230 231 func (t *team) IsCheckContainer(handle string) (bool, error) { 232 var ok int 233 err := psql.Select("1"). 234 From("containers"). 235 Where(sq.Eq{ 236 "handle": handle, 237 }). 238 Where(sq.NotEq{ 239 "resource_config_check_session_id": nil, 240 }). 241 RunWith(t.conn). 242 QueryRow(). 243 Scan(&ok) 244 if err != nil { 245 if err == sql.ErrNoRows { 246 return false, nil 247 } 248 249 return false, err 250 } 251 252 return true, nil 253 } 254 255 func (t *team) IsContainerWithinTeam(handle string, isCheck bool) (bool, error) { 256 var ok int 257 var err error 258 259 if isCheck { 260 err = psql.Select("1"). 261 From("resources r"). 262 Join("pipelines p ON p.id = r.pipeline_id"). 263 Join("resource_configs rc ON rc.id = r.resource_config_id"). 264 Join("resource_config_check_sessions rccs ON rccs.resource_config_id = rc.id"). 265 Join("containers c ON rccs.id = c.resource_config_check_session_id"). 266 Where(sq.Eq{ 267 "c.handle": handle, 268 "p.team_id": t.id, 269 }). 270 RunWith(t.conn). 271 QueryRow(). 272 Scan(&ok) 273 } else { 274 err = psql.Select("1"). 275 From("containers c"). 276 Where(sq.Eq{ 277 "c.team_id": t.id, 278 "c.handle": handle, 279 }). 280 RunWith(t.conn). 281 QueryRow(). 282 Scan(&ok) 283 } 284 if err != nil { 285 if err == sql.ErrNoRows { 286 return false, nil 287 } 288 return false, err 289 } 290 291 return true, nil 292 } 293 294 func (t *team) FindContainerByHandle( 295 handle string, 296 ) (Container, bool, error) { 297 creatingContainer, createdContainer, err := t.findContainer(sq.Eq{"handle": handle}) 298 if err != nil { 299 return nil, false, err 300 } 301 302 if creatingContainer != nil { 303 return creatingContainer, true, nil 304 } 305 306 if createdContainer != nil { 307 return createdContainer, true, nil 308 } 309 310 return nil, false, nil 311 } 312 313 func (t *team) FindContainersByMetadata(metadata ContainerMetadata) ([]Container, error) { 314 eq := sq.Eq(metadata.SQLMap()) 315 eq["team_id"] = t.id 316 317 rows, err := selectContainers(). 318 Where(eq). 319 RunWith(t.conn). 320 Query() 321 if err != nil { 322 return nil, err 323 } 324 325 var containers []Container 326 327 containers, err = scanContainers(rows, t.conn, containers) 328 if err != nil { 329 return nil, err 330 } 331 332 return containers, nil 333 } 334 335 func (t *team) FindCreatedContainerByHandle( 336 handle string, 337 ) (CreatedContainer, bool, error) { 338 _, createdContainer, err := t.findContainer(sq.Eq{"handle": handle}) 339 if err != nil { 340 return nil, false, err 341 } 342 343 if createdContainer != nil { 344 return createdContainer, true, nil 345 } 346 347 return nil, false, nil 348 } 349 350 func savePipeline( 351 tx Tx, 352 pipelineRef atc.PipelineRef, 353 config atc.Config, 354 from ConfigVersion, 355 initiallyPaused bool, 356 teamID int, 357 jobID sql.NullInt64, 358 buildID sql.NullInt64, 359 ) (int, bool, error) { 360 361 var instanceVars sql.NullString 362 if pipelineRef.InstanceVars != nil { 363 bytes, _ := json.Marshal(pipelineRef.InstanceVars) 364 instanceVars = sql.NullString{ 365 String: string(bytes), 366 Valid: true, 367 } 368 } 369 370 pipelineRefWhereClause := sq.Eq{ 371 "team_id": teamID, 372 "name": pipelineRef.Name, 373 "instance_vars": instanceVars, 374 } 375 376 var existingConfig bool 377 err := psql.Select("1"). 378 From("pipelines"). 379 Where(pipelineRefWhereClause). 380 Prefix("SELECT EXISTS (").Suffix(")"). 381 RunWith(tx). 382 QueryRow(). 383 Scan(&existingConfig) 384 if err != nil { 385 return 0, false, err 386 } 387 388 groupsPayload, err := json.Marshal(config.Groups) 389 if err != nil { 390 return 0, false, err 391 } 392 393 varSourcesPayload, err := json.Marshal(config.VarSources) 394 if err != nil { 395 return 0, false, err 396 } 397 398 encryptedVarSourcesPayload, nonce, err := tx.EncryptionStrategy().Encrypt(varSourcesPayload) 399 if err != nil { 400 return 0, false, err 401 } 402 403 displayPayload, err := json.Marshal(config.Display) 404 if err != nil { 405 return 0, false, err 406 } 407 408 var pipelineID int 409 if !existingConfig { 410 err = psql.Insert("pipelines"). 411 SetMap(map[string]interface{}{ 412 "name": pipelineRef.Name, 413 "groups": groupsPayload, 414 "var_sources": encryptedVarSourcesPayload, 415 "display": displayPayload, 416 "nonce": nonce, 417 "version": sq.Expr("nextval('config_version_seq')"), 418 "ordering": sq.Expr("currval('pipelines_id_seq')"), 419 "paused": initiallyPaused, 420 "last_updated": sq.Expr("now()"), 421 "team_id": teamID, 422 "parent_job_id": jobID, 423 "parent_build_id": buildID, 424 "instance_vars": instanceVars, 425 }). 426 Suffix("RETURNING id"). 427 RunWith(tx). 428 QueryRow().Scan(&pipelineID) 429 if err != nil { 430 return 0, false, err 431 } 432 } else { 433 434 q := psql.Update("pipelines"). 435 Set("archived", false). 436 Set("groups", groupsPayload). 437 Set("var_sources", encryptedVarSourcesPayload). 438 Set("display", displayPayload). 439 Set("nonce", nonce). 440 Set("version", sq.Expr("nextval('config_version_seq')")). 441 Set("last_updated", sq.Expr("now()")). 442 Set("parent_job_id", jobID). 443 Set("parent_build_id", buildID). 444 Where(sq.And{ 445 pipelineRefWhereClause, 446 sq.Eq{"version": from}, 447 }) 448 449 if buildID.Valid { 450 q = q.Where(sq.Or{sq.Lt{"parent_build_id": buildID}, sq.Eq{"parent_build_id": nil}}) 451 } 452 453 err := q.Suffix("RETURNING id"). 454 RunWith(tx). 455 QueryRow(). 456 Scan(&pipelineID) 457 if err != nil { 458 if err == sql.ErrNoRows { 459 var currentParentBuildID sql.NullInt64 460 err := psql.Select("parent_build_id"). 461 From("pipelines"). 462 Where(pipelineRefWhereClause). 463 RunWith(tx). 464 QueryRow(). 465 Scan(¤tParentBuildID) 466 if err != nil { 467 return 0, false, err 468 } 469 if currentParentBuildID.Valid && int(buildID.Int64) < int(currentParentBuildID.Int64) { 470 return 0, false, ErrSetByNewerBuild 471 } 472 return 0, false, ErrConfigComparisonFailed 473 } 474 475 return 0, false, err 476 } 477 478 err = resetDependentTableStates(tx, pipelineID) 479 if err != nil { 480 return 0, false, err 481 } 482 } 483 484 err = updateResourcesName(tx, config.Resources, pipelineID) 485 if err != nil { 486 return 0, false, err 487 } 488 489 resourceNameToID, err := saveResources(tx, config.Resources, pipelineID) 490 if err != nil { 491 return 0, false, err 492 } 493 494 _, err = psql.Update("resources"). 495 Set("resource_config_id", nil). 496 Where(sq.Eq{ 497 "pipeline_id": pipelineID, 498 "active": false, 499 }). 500 RunWith(tx). 501 Exec() 502 if err != nil { 503 return 0, false, err 504 } 505 506 err = saveResourceTypes(tx, config.ResourceTypes, pipelineID) 507 if err != nil { 508 return 0, false, err 509 } 510 511 err = updateJobsName(tx, config.Jobs, pipelineID) 512 if err != nil { 513 return 0, false, err 514 } 515 516 jobNameToID, err := saveJobsAndSerialGroups(tx, config.Jobs, config.Groups, pipelineID) 517 if err != nil { 518 return 0, false, err 519 } 520 521 err = removeUnusedWorkerTaskCaches(tx, pipelineID, config.Jobs) 522 if err != nil { 523 return 0, false, err 524 } 525 526 err = insertJobPipes(tx, config.Jobs, resourceNameToID, jobNameToID, pipelineID) 527 if err != nil { 528 return 0, false, err 529 } 530 531 err = requestScheduleForJobsInPipeline(tx, pipelineID) 532 if err != nil { 533 return 0, false, err 534 } 535 536 return pipelineID, !existingConfig, nil 537 } 538 539 func (t *team) SavePipeline( 540 pipelineRef atc.PipelineRef, 541 config atc.Config, 542 from ConfigVersion, 543 initiallyPaused bool, 544 ) (Pipeline, bool, error) { 545 tx, err := t.conn.Begin() 546 if err != nil { 547 return nil, false, err 548 } 549 550 defer Rollback(tx) 551 552 nullID := sql.NullInt64{Valid: false} 553 pipelineID, isNewPipeline, err := savePipeline(tx, pipelineRef, config, from, initiallyPaused, t.id, nullID, nullID) 554 if err != nil { 555 return nil, false, err 556 } 557 558 pipeline := newPipeline(t.conn, t.lockFactory) 559 560 err = scanPipeline( 561 pipeline, 562 pipelinesQuery. 563 Where(sq.Eq{"p.id": pipelineID}). 564 RunWith(tx). 565 QueryRow(), 566 ) 567 if err != nil { 568 return nil, false, err 569 } 570 571 err = tx.Commit() 572 if err != nil { 573 return nil, false, err 574 } 575 576 return pipeline, isNewPipeline, nil 577 } 578 579 func (t *team) Pipeline(pipelineRef atc.PipelineRef) (Pipeline, bool, error) { 580 pipeline := newPipeline(t.conn, t.lockFactory) 581 582 var instanceVars sql.NullString 583 if pipelineRef.InstanceVars != nil { 584 bytes, _ := json.Marshal(pipelineRef.InstanceVars) 585 instanceVars = sql.NullString{ 586 String: string(bytes), 587 Valid: true, 588 } 589 } 590 591 err := scanPipeline( 592 pipeline, 593 pipelinesQuery. 594 Where(sq.Eq{ 595 "p.team_id": t.id, 596 "p.name": pipelineRef.Name, 597 "p.instance_vars": instanceVars, 598 }). 599 RunWith(t.conn). 600 QueryRow(), 601 ) 602 603 if err != nil { 604 if err == sql.ErrNoRows { 605 return nil, false, nil 606 } 607 return nil, false, err 608 } 609 610 return pipeline, true, nil 611 } 612 613 func (t *team) Pipelines() ([]Pipeline, error) { 614 rows, err := pipelinesQuery. 615 Where(sq.Eq{ 616 "team_id": t.id, 617 }). 618 OrderBy("ordering"). 619 RunWith(t.conn). 620 Query() 621 if err != nil { 622 return nil, err 623 } 624 625 pipelines, err := scanPipelines(t.conn, t.lockFactory, rows) 626 if err != nil { 627 return nil, err 628 } 629 630 return pipelines, nil 631 } 632 633 func (t *team) PublicPipelines() ([]Pipeline, error) { 634 rows, err := pipelinesQuery. 635 Where(sq.Eq{ 636 "team_id": t.id, 637 "public": true, 638 }). 639 OrderBy("t.name ASC", "ordering ASC"). 640 RunWith(t.conn). 641 Query() 642 if err != nil { 643 return nil, err 644 } 645 646 pipelines, err := scanPipelines(t.conn, t.lockFactory, rows) 647 if err != nil { 648 return nil, err 649 } 650 651 return pipelines, nil 652 } 653 654 func (t *team) OrderPipelines(pipelineRefs []atc.PipelineRef) error { 655 tx, err := t.conn.Begin() 656 if err != nil { 657 return err 658 } 659 660 defer Rollback(tx) 661 662 for i, pipelineRef := range pipelineRefs { 663 var instanceVars sql.NullString 664 if pipelineRef.InstanceVars != nil { 665 bytes, _ := json.Marshal(pipelineRef.InstanceVars) 666 instanceVars = sql.NullString{ 667 String: string(bytes), 668 Valid: true, 669 } 670 } 671 672 pipelineUpdate, err := psql.Update("pipelines"). 673 Set("ordering", i). 674 Where(sq.Eq{ 675 "team_id": t.id, 676 "name": pipelineRef.Name, 677 "instance_vars": instanceVars, 678 }). 679 RunWith(tx). 680 Exec() 681 if err != nil { 682 return err 683 } 684 updatedPipelines, err := pipelineUpdate.RowsAffected() 685 if err != nil { 686 return err 687 } 688 if updatedPipelines == 0 { 689 return fmt.Errorf("pipeline %s does not exist", pipelineRef.String()) 690 } 691 } 692 693 return tx.Commit() 694 } 695 696 // XXX: This is only begin used by tests, replace all tests to CreateBuild on a job 697 func (t *team) CreateOneOffBuild() (Build, error) { 698 tx, err := t.conn.Begin() 699 if err != nil { 700 return nil, err 701 } 702 703 defer Rollback(tx) 704 705 build := newEmptyBuild(t.conn, t.lockFactory) 706 err = createBuild(tx, build, map[string]interface{}{ 707 "name": sq.Expr("nextval('one_off_name')"), 708 "team_id": t.id, 709 "status": BuildStatusPending, 710 }) 711 if err != nil { 712 return nil, err 713 } 714 715 err = tx.Commit() 716 if err != nil { 717 return nil, err 718 } 719 720 return build, nil 721 } 722 723 func (t *team) CreateStartedBuild(plan atc.Plan) (Build, error) { 724 tx, err := t.conn.Begin() 725 if err != nil { 726 return nil, err 727 } 728 729 defer Rollback(tx) 730 731 metadata, err := json.Marshal(plan) 732 if err != nil { 733 return nil, err 734 } 735 736 encryptedPlan, nonce, err := t.conn.EncryptionStrategy().Encrypt(metadata) 737 if err != nil { 738 return nil, err 739 } 740 741 build := newEmptyBuild(t.conn, t.lockFactory) 742 err = createBuild(tx, build, map[string]interface{}{ 743 "name": sq.Expr("nextval('one_off_name')"), 744 "team_id": t.id, 745 "status": BuildStatusStarted, 746 "start_time": sq.Expr("now()"), 747 "schema": schema, 748 "private_plan": encryptedPlan, 749 "public_plan": plan.Public(), 750 "nonce": nonce, 751 }) 752 if err != nil { 753 return nil, err 754 } 755 756 err = build.saveEvent(tx, event.Status{ 757 Status: atc.StatusStarted, 758 Time: build.StartTime().Unix(), 759 }) 760 if err != nil { 761 return nil, err 762 } 763 764 err = tx.Commit() 765 if err != nil { 766 return nil, err 767 } 768 769 if err = t.conn.Bus().Notify(buildStartedChannel()); err != nil { 770 return nil, err 771 } 772 773 if err = t.conn.Bus().Notify(buildEventsChannel(build.id)); err != nil { 774 return nil, err 775 } 776 777 return build, nil 778 } 779 780 func (t *team) PrivateAndPublicBuilds(page Page) ([]Build, Pagination, error) { 781 newBuildsQuery := buildsQuery. 782 Where(sq.Or{sq.Eq{"p.public": true}, sq.Eq{"t.id": t.id}}) 783 784 return getBuildsWithPagination(newBuildsQuery, minMaxIdQuery, page, t.conn, t.lockFactory) 785 } 786 787 func (t *team) BuildsWithTime(page Page) ([]Build, Pagination, error) { 788 return getBuildsWithDates(buildsQuery.Where(sq.Eq{"t.id": t.id}), minMaxIdQuery, page, t.conn, t.lockFactory) 789 } 790 791 func (t *team) Builds(page Page) ([]Build, Pagination, error) { 792 return getBuildsWithPagination(buildsQuery.Where(sq.Eq{"t.id": t.id}), minMaxIdQuery, page, t.conn, t.lockFactory) 793 } 794 795 func (t *team) SaveWorker(atcWorker atc.Worker, ttl time.Duration) (Worker, error) { 796 tx, err := t.conn.Begin() 797 if err != nil { 798 return nil, err 799 } 800 801 defer Rollback(tx) 802 803 savedWorker, err := saveWorker(tx, atcWorker, &t.id, ttl, t.conn) 804 if err != nil { 805 return nil, err 806 } 807 808 err = tx.Commit() 809 if err != nil { 810 return nil, err 811 } 812 813 return savedWorker, nil 814 } 815 816 func (t *team) UpdateProviderAuth(auth atc.TeamAuth) error { 817 tx, err := t.conn.Begin() 818 if err != nil { 819 return err 820 } 821 defer Rollback(tx) 822 823 jsonEncodedProviderAuth, err := json.Marshal(auth) 824 if err != nil { 825 return err 826 } 827 828 query := ` 829 UPDATE teams 830 SET auth = $1, legacy_auth = NULL, nonce = NULL 831 WHERE id = $2 832 RETURNING id, name, admin, auth, nonce 833 ` 834 err = t.queryTeam(tx, query, jsonEncodedProviderAuth, t.id) 835 if err != nil { 836 return err 837 } 838 839 return tx.Commit() 840 } 841 842 func (t *team) FindCheckContainers(logger lager.Logger, pipelineRef atc.PipelineRef, resourceName string, secretManager creds.Secrets, varSourcePool creds.VarSourcePool) ([]Container, map[int]time.Time, error) { 843 pipeline, found, err := t.Pipeline(pipelineRef) 844 if err != nil { 845 return nil, nil, err 846 } 847 if !found { 848 return nil, nil, nil 849 } 850 851 resource, found, err := pipeline.Resource(resourceName) 852 if err != nil { 853 return nil, nil, err 854 } 855 if !found { 856 return nil, nil, nil 857 } 858 859 pipelineResourceTypes, err := pipeline.ResourceTypes() 860 if err != nil { 861 return nil, nil, err 862 } 863 864 variables, err := pipeline.Variables(logger, secretManager, varSourcePool) 865 if err != nil { 866 return nil, nil, err 867 } 868 869 versionedResourceTypes := pipelineResourceTypes.Deserialize() 870 871 source, err := creds.NewSource(variables, resource.Source()).Evaluate() 872 if err != nil { 873 return nil, nil, err 874 } 875 876 resourceTypes, err := creds.NewVersionedResourceTypes(variables, versionedResourceTypes).Evaluate() 877 if err != nil { 878 return nil, nil, err 879 } 880 881 resourceConfigFactory := NewResourceConfigFactory(t.conn, t.lockFactory) 882 resourceConfig, err := resourceConfigFactory.FindOrCreateResourceConfig( 883 resource.Type(), 884 source, 885 resourceTypes, 886 ) 887 if err != nil { 888 return nil, nil, err 889 } 890 891 rows, err := selectContainers("c"). 892 Join("resource_config_check_sessions rccs ON rccs.id = c.resource_config_check_session_id"). 893 Where(sq.Eq{ 894 "rccs.resource_config_id": resourceConfig.ID(), 895 }). 896 Distinct(). 897 RunWith(t.conn). 898 Query() 899 if err != nil { 900 return nil, nil, err 901 } 902 903 var containers []Container 904 905 containers, err = scanContainers(rows, t.conn, containers) 906 if err != nil { 907 return nil, nil, err 908 } 909 910 rows, err = psql.Select("c.id", "rccs.expires_at"). 911 From("containers c"). 912 Join("resource_config_check_sessions rccs ON rccs.id = c.resource_config_check_session_id"). 913 Where(sq.Eq{ 914 "rccs.resource_config_id": resourceConfig.ID(), 915 }). 916 Distinct(). 917 RunWith(t.conn). 918 Query() 919 if err != nil { 920 return nil, nil, err 921 } 922 923 defer Close(rows) 924 925 checkContainersExpiresAt := make(map[int]time.Time) 926 for rows.Next() { 927 var ( 928 id int 929 expiresAt pq.NullTime 930 ) 931 932 err = rows.Scan(&id, &expiresAt) 933 if err != nil { 934 return nil, nil, err 935 } 936 937 checkContainersExpiresAt[id] = expiresAt.Time 938 } 939 940 return containers, checkContainersExpiresAt, nil 941 } 942 943 type UpdateName struct { 944 OldName string 945 NewName string 946 } 947 948 func updateJobsName(tx Tx, jobs []atc.JobConfig, pipelineID int) error { 949 jobsToUpdate := []UpdateName{} 950 951 for _, job := range jobs { 952 if job.OldName != "" && job.OldName != job.Name { 953 var count int 954 err := psql.Select("COUNT(*) as count"). 955 From("jobs"). 956 Where(sq.Eq{ 957 "name": job.OldName, 958 "pipeline_id": pipelineID}). 959 RunWith(tx). 960 QueryRow(). 961 Scan(&count) 962 if err != nil { 963 return err 964 } 965 966 if count != 0 { 967 jobsToUpdate = append(jobsToUpdate, UpdateName{ 968 OldName: job.OldName, 969 NewName: job.Name, 970 }) 971 } 972 } 973 } 974 975 newMap := make(map[int]bool) 976 for _, updateNames := range jobsToUpdate { 977 isCyclic := checkCyclic(jobsToUpdate, updateNames.OldName, newMap) 978 if isCyclic { 979 return errors.New("job name swapping is not supported at this time") 980 } 981 } 982 983 jobsToUpdate = sortUpdateNames(jobsToUpdate) 984 985 for _, updateName := range jobsToUpdate { 986 _, err := psql.Delete("jobs"). 987 Where(sq.Eq{ 988 "name": updateName.NewName, 989 "pipeline_id": pipelineID, 990 "active": false}). 991 RunWith(tx). 992 Exec() 993 if err != nil { 994 return err 995 } 996 997 _, err = psql.Update("jobs"). 998 Set("name", updateName.NewName). 999 Where(sq.Eq{"name": updateName.OldName, "pipeline_id": pipelineID}). 1000 RunWith(tx). 1001 Exec() 1002 if err != nil { 1003 return err 1004 } 1005 } 1006 1007 return nil 1008 } 1009 1010 func updateResourcesName(tx Tx, resources []atc.ResourceConfig, pipelineID int) error { 1011 resourcesToUpdate := []UpdateName{} 1012 1013 for _, res := range resources { 1014 if res.OldName != "" && res.OldName != res.Name { 1015 var count int 1016 err := psql.Select("COUNT(*) as count"). 1017 From("resources"). 1018 Where(sq.Eq{ 1019 "name": res.OldName, 1020 "pipeline_id": pipelineID}). 1021 RunWith(tx). 1022 QueryRow(). 1023 Scan(&count) 1024 if err != nil { 1025 return err 1026 } 1027 1028 if count != 0 { 1029 resourcesToUpdate = append(resourcesToUpdate, UpdateName{ 1030 OldName: res.OldName, 1031 NewName: res.Name, 1032 }) 1033 } 1034 } 1035 } 1036 1037 newMap := make(map[int]bool) 1038 for _, updateNames := range resourcesToUpdate { 1039 isCyclic := checkCyclic(resourcesToUpdate, updateNames.OldName, newMap) 1040 if isCyclic { 1041 return errors.New("resource name swapping is not supported at this time") 1042 } 1043 } 1044 1045 resourcesToUpdate = sortUpdateNames(resourcesToUpdate) 1046 1047 for _, updateName := range resourcesToUpdate { 1048 _, err := psql.Delete("resources"). 1049 Where(sq.Eq{ 1050 "name": updateName.NewName, 1051 "pipeline_id": pipelineID}). 1052 RunWith(tx). 1053 Exec() 1054 if err != nil { 1055 return err 1056 } 1057 1058 _, err = psql.Update("resources"). 1059 Set("name", updateName.NewName). 1060 Where(sq.Eq{"name": updateName.OldName, "pipeline_id": pipelineID}). 1061 RunWith(tx). 1062 Exec() 1063 if err != nil { 1064 return err 1065 } 1066 } 1067 1068 return nil 1069 } 1070 1071 func checkCyclic(updateNames []UpdateName, curr string, visited map[int]bool) bool { 1072 for i, updateName := range updateNames { 1073 if updateName.NewName == curr && !visited[i] { 1074 visited[i] = true 1075 checkCyclic(updateNames, updateName.OldName, visited) 1076 } else if updateName.NewName == curr && visited[i] && curr != updateName.OldName { 1077 return true 1078 } 1079 } 1080 1081 return false 1082 } 1083 1084 func sortUpdateNames(updateNames []UpdateName) []UpdateName { 1085 newMap := make(map[string]int) 1086 for i, updateName := range updateNames { 1087 newMap[updateName.NewName] = i + 1 1088 1089 if newMap[updateName.OldName] != 0 { 1090 index := newMap[updateName.OldName] - 1 1091 1092 tempName := updateNames[index] 1093 updateNames[index] = updateName 1094 updateNames[i] = tempName 1095 1096 return sortUpdateNames(updateNames) 1097 } 1098 } 1099 1100 return updateNames 1101 } 1102 1103 func saveJob(tx Tx, job atc.JobConfig, pipelineID int, groups []string) (int, error) { 1104 configPayload, err := json.Marshal(job) 1105 if err != nil { 1106 return 0, err 1107 } 1108 1109 es := tx.EncryptionStrategy() 1110 encryptedPayload, nonce, err := es.Encrypt(configPayload) 1111 if err != nil { 1112 return 0, err 1113 } 1114 1115 var jobID int 1116 err = psql.Insert("jobs"). 1117 Columns("name", "pipeline_id", "config", "public", "max_in_flight", "disable_manual_trigger", "interruptible", "active", "nonce", "tags"). 1118 Values(job.Name, pipelineID, encryptedPayload, job.Public, job.MaxInFlight(), job.DisableManualTrigger, job.Interruptible, true, nonce, pq.Array(groups)). 1119 Suffix("ON CONFLICT (name, pipeline_id) DO UPDATE SET config = EXCLUDED.config, public = EXCLUDED.public, max_in_flight = EXCLUDED.max_in_flight, disable_manual_trigger = EXCLUDED.disable_manual_trigger, interruptible = EXCLUDED.interruptible, active = EXCLUDED.active, nonce = EXCLUDED.nonce, tags = EXCLUDED.tags"). 1120 Suffix("RETURNING id"). 1121 RunWith(tx). 1122 QueryRow(). 1123 Scan(&jobID) 1124 if err != nil { 1125 return 0, err 1126 } 1127 1128 return jobID, nil 1129 } 1130 1131 func registerSerialGroup(tx Tx, serialGroup string, jobID int) error { 1132 _, err := psql.Insert("jobs_serial_groups"). 1133 Columns("serial_group", "job_id"). 1134 Values(serialGroup, jobID). 1135 RunWith(tx). 1136 Exec() 1137 return err 1138 } 1139 1140 func saveResource(tx Tx, resource atc.ResourceConfig, pipelineID int) (int, error) { 1141 configPayload, err := json.Marshal(resource) 1142 if err != nil { 1143 return 0, err 1144 } 1145 1146 es := tx.EncryptionStrategy() 1147 encryptedPayload, nonce, err := es.Encrypt(configPayload) 1148 if err != nil { 1149 return 0, err 1150 } 1151 1152 var resourceID int 1153 err = psql.Insert("resources"). 1154 Columns("name", "pipeline_id", "config", "active", "nonce", "type"). 1155 Values(resource.Name, pipelineID, encryptedPayload, true, nonce, resource.Type). 1156 Suffix("ON CONFLICT (name, pipeline_id) DO UPDATE SET config = EXCLUDED.config, active = EXCLUDED.active, nonce = EXCLUDED.nonce, type = EXCLUDED.type"). 1157 Suffix("RETURNING id"). 1158 RunWith(tx). 1159 QueryRow(). 1160 Scan(&resourceID) 1161 if err != nil { 1162 return 0, err 1163 } 1164 1165 _, err = psql.Delete("resource_pins"). 1166 Where(sq.Eq{ 1167 "resource_id": resourceID, 1168 "config": true, 1169 }). 1170 RunWith(tx). 1171 Exec() 1172 if err != nil { 1173 return 0, err 1174 } 1175 1176 if resource.Version != nil { 1177 version, err := json.Marshal(resource.Version) 1178 if err != nil { 1179 return 0, err 1180 } 1181 1182 _, err = psql.Insert("resource_pins"). 1183 Columns("resource_id", "version", "comment_text", "config"). 1184 Values(resourceID, version, "", true). 1185 Suffix("ON CONFLICT (resource_id) DO UPDATE SET version = EXCLUDED.version, comment_text = EXCLUDED.comment_text, config = true"). 1186 RunWith(tx). 1187 Exec() 1188 if err != nil { 1189 return 0, err 1190 } 1191 } 1192 1193 return resourceID, nil 1194 } 1195 1196 func saveResourceType(tx Tx, resourceType atc.ResourceType, pipelineID int) error { 1197 configPayload, err := json.Marshal(resourceType) 1198 if err != nil { 1199 return err 1200 } 1201 1202 es := tx.EncryptionStrategy() 1203 encryptedPayload, nonce, err := es.Encrypt(configPayload) 1204 if err != nil { 1205 return err 1206 } 1207 1208 _, err = psql.Insert("resource_types"). 1209 Columns("name", "pipeline_id", "config", "active", "nonce", "type"). 1210 Values(resourceType.Name, pipelineID, encryptedPayload, true, nonce, resourceType.Type). 1211 Suffix("ON CONFLICT (name, pipeline_id) DO UPDATE SET config = EXCLUDED.config, active = EXCLUDED.active, nonce = EXCLUDED.nonce, type = EXCLUDED.type"). 1212 RunWith(tx). 1213 Exec() 1214 1215 return err 1216 } 1217 1218 func checkIfRowsUpdated(tx Tx, query string, params ...interface{}) (bool, error) { 1219 result, err := tx.Exec(query, params...) 1220 if err != nil { 1221 return false, err 1222 } 1223 1224 rows, err := result.RowsAffected() 1225 if err != nil { 1226 return false, err 1227 } 1228 1229 if rows == 0 { 1230 return false, nil 1231 } 1232 1233 return true, nil 1234 } 1235 1236 func swallowUniqueiolation(err error) error { 1237 if err != nil { 1238 if pgErr, ok := err.(*pq.Error); ok { 1239 if pgErr.Code.Class().Name() == "integrity_constraint_violation" { 1240 return nil 1241 } 1242 } 1243 1244 return err 1245 } 1246 1247 return nil 1248 } 1249 1250 func (t *team) findContainer(whereClause sq.Sqlizer) (CreatingContainer, CreatedContainer, error) { 1251 creating, created, destroying, _, err := scanContainer( 1252 selectContainers(). 1253 Where(whereClause). 1254 RunWith(t.conn). 1255 QueryRow(), 1256 t.conn, 1257 ) 1258 if err != nil { 1259 if err == sql.ErrNoRows { 1260 return nil, nil, nil 1261 } 1262 return nil, nil, err 1263 } 1264 1265 if destroying != nil { 1266 return nil, nil, nil 1267 } 1268 1269 return creating, created, nil 1270 } 1271 1272 func scanPipeline(p *pipeline, scan scannable) error { 1273 var ( 1274 groups sql.NullString 1275 varSources sql.NullString 1276 display sql.NullString 1277 nonce sql.NullString 1278 nonceStr *string 1279 lastUpdated pq.NullTime 1280 parentJobID sql.NullInt64 1281 parentBuildID sql.NullInt64 1282 instanceVars sql.NullString 1283 ) 1284 err := scan.Scan(&p.id, &p.name, &groups, &varSources, &display, &nonce, &p.configVersion, &p.teamID, &p.teamName, &p.paused, &p.public, &p.archived, &lastUpdated, &parentJobID, &parentBuildID, &instanceVars) 1285 if err != nil { 1286 return err 1287 } 1288 1289 p.lastUpdated = lastUpdated.Time 1290 p.parentJobID = int(parentJobID.Int64) 1291 p.parentBuildID = int(parentBuildID.Int64) 1292 1293 if groups.Valid { 1294 var pipelineGroups atc.GroupConfigs 1295 err = json.Unmarshal([]byte(groups.String), &pipelineGroups) 1296 if err != nil { 1297 return err 1298 } 1299 1300 p.groups = pipelineGroups 1301 } 1302 1303 if nonce.Valid { 1304 nonceStr = &nonce.String 1305 } 1306 1307 if display.Valid { 1308 var displayConfig *atc.DisplayConfig 1309 err = json.Unmarshal([]byte(display.String), &displayConfig) 1310 if err != nil { 1311 return err 1312 } 1313 1314 p.display = displayConfig 1315 } 1316 1317 if varSources.Valid { 1318 var pipelineVarSources atc.VarSourceConfigs 1319 decryptedVarSource, err := p.conn.EncryptionStrategy().Decrypt(varSources.String, nonceStr) 1320 if err != nil { 1321 return err 1322 } 1323 err = json.Unmarshal([]byte(decryptedVarSource), &pipelineVarSources) 1324 if err != nil { 1325 return err 1326 } 1327 1328 p.varSources = pipelineVarSources 1329 } 1330 1331 if instanceVars.Valid { 1332 err = json.Unmarshal([]byte(instanceVars.String), &p.instanceVars) 1333 if err != nil { 1334 return err 1335 } 1336 } 1337 1338 return nil 1339 } 1340 1341 func scanPipelines(conn Conn, lockFactory lock.LockFactory, rows *sql.Rows) ([]Pipeline, error) { 1342 defer Close(rows) 1343 1344 pipelines := []Pipeline{} 1345 1346 for rows.Next() { 1347 pipeline := newPipeline(conn, lockFactory) 1348 1349 err := scanPipeline(pipeline, rows) 1350 if err != nil { 1351 return nil, err 1352 } 1353 1354 pipelines = append(pipelines, pipeline) 1355 } 1356 1357 return pipelines, nil 1358 } 1359 1360 func scanContainers(rows *sql.Rows, conn Conn, initContainers []Container) ([]Container, error) { 1361 containers := initContainers 1362 1363 defer Close(rows) 1364 1365 for rows.Next() { 1366 creating, created, destroying, _, err := scanContainer(rows, conn) 1367 if err != nil { 1368 return []Container{}, err 1369 } 1370 1371 if creating != nil { 1372 containers = append(containers, creating) 1373 } 1374 1375 if created != nil { 1376 containers = append(containers, created) 1377 } 1378 1379 if destroying != nil { 1380 containers = append(containers, destroying) 1381 } 1382 } 1383 1384 return containers, nil 1385 } 1386 1387 func (t *team) queryTeam(tx Tx, query string, params ...interface{}) error { 1388 var providerAuth, nonce sql.NullString 1389 1390 err := tx.QueryRow(query, params...).Scan( 1391 &t.id, 1392 &t.name, 1393 &t.admin, 1394 &providerAuth, 1395 &nonce, 1396 ) 1397 if err != nil { 1398 return err 1399 } 1400 1401 if providerAuth.Valid { 1402 var auth atc.TeamAuth 1403 err = json.Unmarshal([]byte(providerAuth.String), &auth) 1404 if err != nil { 1405 return err 1406 } 1407 t.auth = auth 1408 } 1409 1410 return nil 1411 } 1412 1413 func resetDependentTableStates(tx Tx, pipelineID int) error { 1414 _, err := psql.Delete("jobs_serial_groups"). 1415 Where(sq.Expr(`job_id in ( 1416 SELECT j.id 1417 FROM jobs j 1418 WHERE j.pipeline_id = $1 1419 )`, pipelineID)). 1420 RunWith(tx). 1421 Exec() 1422 if err != nil { 1423 return err 1424 } 1425 1426 tableNames := []string{"jobs", "resources", "resource_types"} 1427 for _, table := range tableNames { 1428 err = inactivateTableForPipeline(tx, pipelineID, table) 1429 if err != nil { 1430 return err 1431 } 1432 } 1433 return err 1434 } 1435 1436 func inactivateTableForPipeline(tx Tx, pipelineID int, tableName string) error { 1437 _, err := psql.Update(tableName). 1438 Set("active", false). 1439 Where(sq.Eq{ 1440 "pipeline_id": pipelineID, 1441 }). 1442 RunWith(tx). 1443 Exec() 1444 return err 1445 } 1446 1447 func saveResources(tx Tx, resources atc.ResourceConfigs, pipelineID int) (map[string]int, error) { 1448 resourceNameToID := make(map[string]int) 1449 for _, resource := range resources { 1450 resourceID, err := saveResource(tx, resource, pipelineID) 1451 if err != nil { 1452 return nil, err 1453 } 1454 1455 resourceNameToID[resource.Name] = resourceID 1456 } 1457 1458 return resourceNameToID, nil 1459 } 1460 1461 func saveResourceTypes(tx Tx, resourceTypes atc.ResourceTypes, pipelineID int) error { 1462 for _, resourceType := range resourceTypes { 1463 err := saveResourceType(tx, resourceType, pipelineID) 1464 if err != nil { 1465 return err 1466 } 1467 } 1468 1469 return nil 1470 } 1471 1472 func saveJobsAndSerialGroups(tx Tx, jobs atc.JobConfigs, groups atc.GroupConfigs, pipelineID int) (map[string]int, error) { 1473 jobGroups := make(map[string][]string) 1474 for _, group := range groups { 1475 for _, jobGlob := range group.Jobs { 1476 for _, job := range jobs { 1477 if g, err := glob.Compile(jobGlob); err == nil && g.Match(job.Name) { 1478 jobGroups[job.Name] = append(jobGroups[job.Name], group.Name) 1479 } 1480 } 1481 } 1482 } 1483 1484 jobNameToID := make(map[string]int) 1485 for _, job := range jobs { 1486 jobID, err := saveJob(tx, job, pipelineID, jobGroups[job.Name]) 1487 if err != nil { 1488 return nil, err 1489 } 1490 1491 jobNameToID[job.Name] = jobID 1492 1493 if len(job.SerialGroups) != 0 { 1494 for _, sg := range job.SerialGroups { 1495 err = registerSerialGroup(tx, sg, jobID) 1496 if err != nil { 1497 return nil, err 1498 } 1499 } 1500 } else { 1501 if job.Serial || job.RawMaxInFlight > 0 { 1502 err = registerSerialGroup(tx, job.Name, jobID) 1503 if err != nil { 1504 return nil, err 1505 } 1506 } 1507 } 1508 } 1509 1510 return jobNameToID, nil 1511 } 1512 1513 func insertJobPipes(tx Tx, jobConfigs atc.JobConfigs, resourceNameToID map[string]int, jobNameToID map[string]int, pipelineID int) error { 1514 _, err := psql.Delete("job_inputs"). 1515 Where(sq.Expr(`job_id in ( 1516 SELECT j.id 1517 FROM jobs j 1518 WHERE j.pipeline_id = $1 1519 )`, pipelineID)). 1520 RunWith(tx). 1521 Exec() 1522 if err != nil { 1523 return err 1524 } 1525 1526 _, err = psql.Delete("job_outputs"). 1527 Where(sq.Expr(`job_id in ( 1528 SELECT j.id 1529 FROM jobs j 1530 WHERE j.pipeline_id = $1 1531 )`, pipelineID)). 1532 RunWith(tx). 1533 Exec() 1534 if err != nil { 1535 return err 1536 } 1537 1538 for _, jobConfig := range jobConfigs { 1539 err := jobConfig.StepConfig().Visit(atc.StepRecursor{ 1540 OnGet: func(step *atc.GetStep) error { 1541 return insertJobInput(tx, step, jobConfig.Name, resourceNameToID, jobNameToID) 1542 }, 1543 OnPut: func(step *atc.PutStep) error { 1544 return insertJobOutput(tx, step, jobConfig.Name, resourceNameToID, jobNameToID) 1545 }, 1546 }) 1547 if err != nil { 1548 return err 1549 } 1550 } 1551 1552 return nil 1553 } 1554 1555 func insertJobInput(tx Tx, step *atc.GetStep, jobName string, resourceNameToID map[string]int, jobNameToID map[string]int) error { 1556 if len(step.Passed) != 0 { 1557 for _, passedJob := range step.Passed { 1558 var version sql.NullString 1559 if step.Version != nil { 1560 versionJSON, err := step.Version.MarshalJSON() 1561 if err != nil { 1562 return err 1563 } 1564 1565 version = sql.NullString{Valid: true, String: string(versionJSON)} 1566 } 1567 1568 _, err := psql.Insert("job_inputs"). 1569 Columns("name", "job_id", "resource_id", "passed_job_id", "trigger", "version"). 1570 Values(step.Name, jobNameToID[jobName], resourceNameToID[step.ResourceName()], jobNameToID[passedJob], step.Trigger, version). 1571 RunWith(tx). 1572 Exec() 1573 if err != nil { 1574 return err 1575 } 1576 } 1577 } else { 1578 var version sql.NullString 1579 if step.Version != nil { 1580 versionJSON, err := step.Version.MarshalJSON() 1581 if err != nil { 1582 return err 1583 } 1584 1585 version = sql.NullString{Valid: true, String: string(versionJSON)} 1586 } 1587 1588 _, err := psql.Insert("job_inputs"). 1589 Columns("name", "job_id", "resource_id", "trigger", "version"). 1590 Values(step.Name, jobNameToID[jobName], resourceNameToID[step.ResourceName()], step.Trigger, version). 1591 RunWith(tx). 1592 Exec() 1593 if err != nil { 1594 return err 1595 } 1596 } 1597 1598 return nil 1599 } 1600 1601 func insertJobOutput(tx Tx, step *atc.PutStep, jobName string, resourceNameToID map[string]int, jobNameToID map[string]int) error { 1602 _, err := psql.Insert("job_outputs"). 1603 Columns("name", "job_id", "resource_id"). 1604 Values(step.Name, jobNameToID[jobName], resourceNameToID[step.ResourceName()]). 1605 RunWith(tx). 1606 Exec() 1607 if err != nil { 1608 return err 1609 } 1610 1611 return nil 1612 }