github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/pipeline.go (about)

     1  package db
     2  
     3  import (
     4  	"database/sql"
     5  	"encoding/json"
     6  	"fmt"
     7  	"strconv"
     8  	"strings"
     9  	"time"
    10  
    11  	"code.cloudfoundry.org/lager"
    12  
    13  	sq "github.com/Masterminds/squirrel"
    14  	"github.com/pkg/errors"
    15  
    16  	"github.com/pf-qiu/concourse/v6/atc"
    17  	"github.com/pf-qiu/concourse/v6/atc/creds"
    18  	"github.com/pf-qiu/concourse/v6/atc/db/lock"
    19  	"github.com/pf-qiu/concourse/v6/atc/event"
    20  	"github.com/pf-qiu/concourse/v6/vars"
    21  )
    22  
    23  type ErrResourceNotFound struct {
    24  	Name string
    25  }
    26  
    27  func (e ErrResourceNotFound) Error() string {
    28  	return fmt.Sprintf("resource '%s' not found", e.Name)
    29  }
    30  
    31  //go:generate counterfeiter . Pipeline
    32  
    33  type Cause struct {
    34  	ResourceVersionID int `json:"resource_version_id"`
    35  	BuildID           int `json:"build_id"`
    36  }
    37  
    38  type Pipeline interface {
    39  	ID() int
    40  	Name() string
    41  	TeamID() int
    42  	TeamName() string
    43  	InstanceVars() atc.InstanceVars
    44  	ParentJobID() int
    45  	ParentBuildID() int
    46  	Groups() atc.GroupConfigs
    47  	VarSources() atc.VarSourceConfigs
    48  	Display() *atc.DisplayConfig
    49  	ConfigVersion() ConfigVersion
    50  	Config() (atc.Config, error)
    51  	Public() bool
    52  	Paused() bool
    53  	Archived() bool
    54  	LastUpdated() time.Time
    55  
    56  	CheckPaused() (bool, error)
    57  	Reload() (bool, error)
    58  
    59  	Causality(versionedResourceID int) ([]Cause, error)
    60  	ResourceVersion(resourceConfigVersionID int) (atc.ResourceVersion, bool, error)
    61  
    62  	GetBuildsWithVersionAsInput(int, int) ([]Build, error)
    63  	GetBuildsWithVersionAsOutput(int, int) ([]Build, error)
    64  	Builds(page Page) ([]Build, Pagination, error)
    65  
    66  	CreateOneOffBuild() (Build, error)
    67  	CreateStartedBuild(plan atc.Plan) (Build, error)
    68  
    69  	BuildsWithTime(page Page) ([]Build, Pagination, error)
    70  
    71  	DeleteBuildEventsByBuildIDs(buildIDs []int) error
    72  
    73  	LoadDebugVersionsDB() (*atc.DebugVersionsDB, error)
    74  
    75  	Resource(name string) (Resource, bool, error)
    76  	ResourceByID(id int) (Resource, bool, error)
    77  	Resources() (Resources, error)
    78  
    79  	ResourceTypes() (ResourceTypes, error)
    80  	ResourceType(name string) (ResourceType, bool, error)
    81  	ResourceTypeByID(id int) (ResourceType, bool, error)
    82  
    83  	Job(name string) (Job, bool, error)
    84  	Jobs() (Jobs, error)
    85  	Dashboard() ([]atc.JobSummary, error)
    86  
    87  	Expose() error
    88  	Hide() error
    89  
    90  	Pause() error
    91  	Unpause() error
    92  
    93  	Archive() error
    94  
    95  	Destroy() error
    96  	Rename(string) error
    97  
    98  	Variables(lager.Logger, creds.Secrets, creds.VarSourcePool) (vars.Variables, error)
    99  
   100  	SetParentIDs(jobID, buildID int) error
   101  }
   102  
   103  type pipeline struct {
   104  	id            int
   105  	name          string
   106  	teamID        int
   107  	teamName      string
   108  	instanceVars  atc.InstanceVars
   109  	parentJobID   int
   110  	parentBuildID int
   111  	groups        atc.GroupConfigs
   112  	varSources    atc.VarSourceConfigs
   113  	display       *atc.DisplayConfig
   114  	configVersion ConfigVersion
   115  	paused        bool
   116  	public        bool
   117  	archived      bool
   118  	lastUpdated   time.Time
   119  
   120  	conn        Conn
   121  	lockFactory lock.LockFactory
   122  }
   123  
   124  // ConfigVersion is a sequence identifier used for compare-and-swap.
   125  type ConfigVersion int
   126  
   127  var pipelinesQuery = psql.Select(`
   128  		p.id,
   129  		p.name,
   130  		p.groups,
   131  		p.var_sources,
   132  		p.display,
   133  		p.nonce,
   134  		p.version,
   135  		p.team_id,
   136  		t.name,
   137  		p.paused,
   138  		p.public,
   139  		p.archived,
   140  		p.last_updated,
   141  		p.parent_job_id,
   142  		p.parent_build_id,
   143  		p.instance_vars
   144  	`).
   145  	From("pipelines p").
   146  	LeftJoin("teams t ON p.team_id = t.id")
   147  
   148  func newPipeline(conn Conn, lockFactory lock.LockFactory) *pipeline {
   149  	return &pipeline{
   150  		conn:        conn,
   151  		lockFactory: lockFactory,
   152  	}
   153  }
   154  
   155  func (p *pipeline) ID() int                        { return p.id }
   156  func (p *pipeline) Name() string                   { return p.name }
   157  func (p *pipeline) TeamID() int                    { return p.teamID }
   158  func (p *pipeline) TeamName() string               { return p.teamName }
   159  func (p *pipeline) ParentJobID() int               { return p.parentJobID }
   160  func (p *pipeline) ParentBuildID() int             { return p.parentBuildID }
   161  func (p *pipeline) InstanceVars() atc.InstanceVars { return p.instanceVars }
   162  func (p *pipeline) Groups() atc.GroupConfigs       { return p.groups }
   163  
   164  func (p *pipeline) VarSources() atc.VarSourceConfigs { return p.varSources }
   165  func (p *pipeline) Display() *atc.DisplayConfig      { return p.display }
   166  func (p *pipeline) ConfigVersion() ConfigVersion     { return p.configVersion }
   167  func (p *pipeline) Public() bool                     { return p.public }
   168  func (p *pipeline) Paused() bool                     { return p.paused }
   169  func (p *pipeline) Archived() bool                   { return p.archived }
   170  func (p *pipeline) LastUpdated() time.Time           { return p.lastUpdated }
   171  
   172  // IMPORTANT: This method is broken with the new resource config versions changes
   173  func (p *pipeline) Causality(versionedResourceID int) ([]Cause, error) {
   174  	rows, err := p.conn.Query(`
   175  		WITH RECURSIVE causality(versioned_resource_id, build_id) AS (
   176  				SELECT bi.versioned_resource_id, bi.build_id
   177  				FROM build_inputs bi
   178  				WHERE bi.versioned_resource_id = $1
   179  			UNION
   180  				SELECT bi.versioned_resource_id, bi.build_id
   181  				FROM causality t
   182  				INNER JOIN build_outputs bo ON bo.build_id = t.build_id
   183  				INNER JOIN build_inputs bi ON bi.versioned_resource_id = bo.versioned_resource_id
   184  				INNER JOIN builds b ON b.id = bi.build_id
   185  				AND NOT EXISTS (
   186  					SELECT 1
   187  					FROM build_outputs obo
   188  					INNER JOIN builds ob ON ob.id = obo.build_id
   189  					WHERE obo.build_id < bi.build_id
   190  					AND ob.job_id = b.job_id
   191  					AND obo.versioned_resource_id = bi.versioned_resource_id
   192  				)
   193  		)
   194  		SELECT c.versioned_resource_id, c.build_id
   195  		FROM causality c
   196  		INNER JOIN builds b ON b.id = c.build_id
   197  		ORDER BY b.start_time ASC, c.versioned_resource_id ASC
   198  	`, versionedResourceID)
   199  	if err != nil {
   200  		return nil, err
   201  	}
   202  
   203  	var causality []Cause
   204  	for rows.Next() {
   205  		var vrID, buildID int
   206  		err := rows.Scan(&vrID, &buildID)
   207  		if err != nil {
   208  			return nil, err
   209  		}
   210  
   211  		causality = append(causality, Cause{
   212  			ResourceVersionID: vrID,
   213  			BuildID:           buildID,
   214  		})
   215  	}
   216  
   217  	return causality, nil
   218  }
   219  
   220  func (p *pipeline) CheckPaused() (bool, error) {
   221  	var paused bool
   222  
   223  	err := psql.Select("paused").
   224  		From("pipelines").
   225  		Where(sq.Eq{"id": p.id}).
   226  		RunWith(p.conn).
   227  		QueryRow().
   228  		Scan(&paused)
   229  
   230  	if err != nil {
   231  		return false, err
   232  	}
   233  
   234  	return paused, nil
   235  }
   236  func (p *pipeline) Reload() (bool, error) {
   237  	row := pipelinesQuery.Where(sq.Eq{"p.id": p.id}).
   238  		RunWith(p.conn).
   239  		QueryRow()
   240  
   241  	err := scanPipeline(p, row)
   242  	if err != nil {
   243  		if err == sql.ErrNoRows {
   244  			return false, nil
   245  		}
   246  		return false, err
   247  	}
   248  
   249  	return true, nil
   250  }
   251  
   252  func (p *pipeline) Config() (atc.Config, error) {
   253  	jobs, err := p.Jobs()
   254  	if err != nil {
   255  		return atc.Config{}, fmt.Errorf("failed to get jobs: %w", err)
   256  	}
   257  
   258  	resources, err := p.Resources()
   259  	if err != nil {
   260  		return atc.Config{}, fmt.Errorf("failed to get resources: %w", err)
   261  	}
   262  
   263  	resourceTypes, err := p.ResourceTypes()
   264  	if err != nil {
   265  		return atc.Config{}, fmt.Errorf("failed to get resources-types: %w", err)
   266  	}
   267  
   268  	jobConfigs, err := jobs.Configs()
   269  	if err != nil {
   270  		return atc.Config{}, fmt.Errorf("failed to get job configs: %w", err)
   271  	}
   272  
   273  	config := atc.Config{
   274  		Groups:        p.Groups(),
   275  		VarSources:    p.VarSources(),
   276  		Resources:     resources.Configs(),
   277  		ResourceTypes: resourceTypes.Configs(),
   278  		Jobs:          jobConfigs,
   279  		Display:       p.Display(),
   280  	}
   281  
   282  	return config, nil
   283  }
   284  
   285  func (p *pipeline) CreateJobBuild(jobName string) (Build, error) {
   286  	tx, err := p.conn.Begin()
   287  	if err != nil {
   288  		return nil, err
   289  	}
   290  
   291  	defer Rollback(tx)
   292  
   293  	buildName, jobID, err := getNewBuildNameForJob(tx, jobName, p.id)
   294  	if err != nil {
   295  		return nil, err
   296  	}
   297  
   298  	var buildID int
   299  	err = psql.Insert("builds").
   300  		Columns("name", "job_id", "team_id", "status", "manually_triggered").
   301  		Values(buildName, jobID, p.teamID, "pending", true).
   302  		Suffix("RETURNING id").
   303  		RunWith(tx).
   304  		QueryRow().
   305  		Scan(&buildID)
   306  	if err != nil {
   307  		return nil, err
   308  	}
   309  
   310  	build := newEmptyBuild(p.conn, p.lockFactory)
   311  	err = scanBuild(build, buildsQuery.
   312  		Where(sq.Eq{"b.id": buildID}).
   313  		RunWith(tx).
   314  		QueryRow(),
   315  		p.conn.EncryptionStrategy(),
   316  	)
   317  	if err != nil {
   318  		return nil, err
   319  	}
   320  
   321  	err = createBuildEventSeq(tx, buildID)
   322  	if err != nil {
   323  		return nil, err
   324  	}
   325  
   326  	err = tx.Commit()
   327  	if err != nil {
   328  		return nil, err
   329  	}
   330  
   331  	return build, nil
   332  }
   333  
   334  // ResourceVersion is given a resource config version id and returns the
   335  // resource version struct. This method is used by the API call
   336  // GetResourceVersion to get all the attributes for that version of the
   337  // resource.
   338  func (p *pipeline) ResourceVersion(resourceConfigVersionID int) (atc.ResourceVersion, bool, error) {
   339  	rv := atc.ResourceVersion{}
   340  	var (
   341  		versionBytes  string
   342  		metadataBytes string
   343  	)
   344  
   345  	enabled := `
   346  		NOT EXISTS (
   347  			SELECT 1
   348  			FROM resource_disabled_versions d, resources r
   349  			WHERE v.version_md5 = d.version_md5
   350  			AND r.resource_config_scope_id = v.resource_config_scope_id
   351  			AND r.id = d.resource_id
   352  		)`
   353  
   354  	err := psql.Select("v.id", "v.version", "v.metadata", enabled).
   355  		From("resource_config_versions v").
   356  		Where(sq.Eq{
   357  			"v.id": resourceConfigVersionID,
   358  		}).
   359  		RunWith(p.conn).
   360  		QueryRow().
   361  		Scan(&rv.ID, &versionBytes, &metadataBytes, &rv.Enabled)
   362  	if err != nil {
   363  		if err == sql.ErrNoRows {
   364  			return atc.ResourceVersion{}, false, nil
   365  		}
   366  
   367  		return atc.ResourceVersion{}, false, err
   368  	}
   369  
   370  	err = json.Unmarshal([]byte(versionBytes), &rv.Version)
   371  	if err != nil {
   372  		return atc.ResourceVersion{}, false, err
   373  	}
   374  
   375  	err = json.Unmarshal([]byte(metadataBytes), &rv.Metadata)
   376  	if err != nil {
   377  		return atc.ResourceVersion{}, false, err
   378  	}
   379  
   380  	return rv, true, nil
   381  }
   382  
   383  func (p *pipeline) GetBuildsWithVersionAsInput(resourceID, resourceConfigVersionID int) ([]Build, error) {
   384  	rows, err := buildsQuery.
   385  		Join("build_resource_config_version_inputs bi ON bi.build_id = b.id").
   386  		Join("resource_config_versions rcv ON rcv.version_md5 = bi.version_md5").
   387  		Where(sq.Eq{
   388  			"rcv.id":         resourceConfigVersionID,
   389  			"bi.resource_id": resourceID,
   390  		}).
   391  		RunWith(p.conn).
   392  		Query()
   393  	if err != nil {
   394  		return nil, err
   395  	}
   396  	defer Close(rows)
   397  
   398  	builds := []Build{}
   399  	for rows.Next() {
   400  		build := newEmptyBuild(p.conn, p.lockFactory)
   401  		err = scanBuild(build, rows, p.conn.EncryptionStrategy())
   402  		if err != nil {
   403  			return nil, err
   404  		}
   405  		builds = append(builds, build)
   406  	}
   407  
   408  	return builds, err
   409  }
   410  
   411  func (p *pipeline) GetBuildsWithVersionAsOutput(resourceID, resourceConfigVersionID int) ([]Build, error) {
   412  	rows, err := buildsQuery.
   413  		Join("build_resource_config_version_outputs bo ON bo.build_id = b.id").
   414  		Join("resource_config_versions rcv ON rcv.version_md5 = bo.version_md5").
   415  		Where(sq.Eq{
   416  			"rcv.id":         resourceConfigVersionID,
   417  			"bo.resource_id": resourceID,
   418  		}).
   419  		RunWith(p.conn).
   420  		Query()
   421  	if err != nil {
   422  		return nil, err
   423  	}
   424  	defer Close(rows)
   425  
   426  	builds := []Build{}
   427  	for rows.Next() {
   428  		build := newEmptyBuild(p.conn, p.lockFactory)
   429  		err = scanBuild(build, rows, p.conn.EncryptionStrategy())
   430  		if err != nil {
   431  			return nil, err
   432  		}
   433  
   434  		builds = append(builds, build)
   435  	}
   436  
   437  	return builds, err
   438  }
   439  
   440  func (p *pipeline) Resource(name string) (Resource, bool, error) {
   441  	return p.resource(sq.Eq{
   442  		"r.pipeline_id": p.id,
   443  		"r.name":        name,
   444  	})
   445  }
   446  
   447  func (p *pipeline) ResourceByID(id int) (Resource, bool, error) {
   448  	return p.resource(sq.Eq{
   449  		"r.pipeline_id": p.id,
   450  		"r.id":          id,
   451  	})
   452  }
   453  
   454  func (p *pipeline) resource(where map[string]interface{}) (Resource, bool, error) {
   455  	row := resourcesQuery.
   456  		Where(where).
   457  		RunWith(p.conn).
   458  		QueryRow()
   459  
   460  	resource := newEmptyResource(p.conn, p.lockFactory)
   461  	err := scanResource(resource, row)
   462  	if err != nil {
   463  		if err == sql.ErrNoRows {
   464  			return nil, false, nil
   465  		}
   466  
   467  		return nil, false, err
   468  	}
   469  
   470  	return resource, true, nil
   471  }
   472  
   473  func (p *pipeline) Builds(page Page) ([]Build, Pagination, error) {
   474  	return getBuildsWithPagination(
   475  		buildsQuery.Where(sq.Eq{"b.pipeline_id": p.id}), minMaxIdQuery, page, p.conn, p.lockFactory)
   476  }
   477  
   478  func (p *pipeline) BuildsWithTime(page Page) ([]Build, Pagination, error) {
   479  	return getBuildsWithDates(
   480  		buildsQuery.Where(sq.Eq{"b.pipeline_id": p.id}), minMaxIdQuery, page, p.conn, p.lockFactory)
   481  }
   482  
   483  func (p *pipeline) Resources() (Resources, error) {
   484  	return resources(p.id, p.conn, p.lockFactory)
   485  }
   486  
   487  func (p *pipeline) ResourceTypes() (ResourceTypes, error) {
   488  	rows, err := resourceTypesQuery.
   489  		Where(sq.Eq{"r.pipeline_id": p.id}).
   490  		OrderBy("r.name").
   491  		RunWith(p.conn).
   492  		Query()
   493  	if err != nil {
   494  		return nil, err
   495  	}
   496  	defer Close(rows)
   497  
   498  	resourceTypes := []ResourceType{}
   499  
   500  	for rows.Next() {
   501  		resourceType := newEmptyResourceType(p.conn, p.lockFactory)
   502  		err := scanResourceType(resourceType, rows)
   503  		if err != nil {
   504  			return nil, err
   505  		}
   506  
   507  		resourceTypes = append(resourceTypes, resourceType)
   508  	}
   509  
   510  	return resourceTypes, nil
   511  }
   512  
   513  func (p *pipeline) ResourceType(name string) (ResourceType, bool, error) {
   514  	return p.resourceType(sq.Eq{
   515  		"r.pipeline_id": p.id,
   516  		"r.name":        name,
   517  	})
   518  }
   519  
   520  func (p *pipeline) ResourceTypeByID(id int) (ResourceType, bool, error) {
   521  	return p.resourceType(sq.Eq{
   522  		"r.pipeline_id": p.id,
   523  		"r.id":          id,
   524  	})
   525  }
   526  
   527  func (p *pipeline) resourceType(where map[string]interface{}) (ResourceType, bool, error) {
   528  	row := resourceTypesQuery.
   529  		Where(where).
   530  		RunWith(p.conn).
   531  		QueryRow()
   532  
   533  	resourceType := newEmptyResourceType(p.conn, p.lockFactory)
   534  	err := scanResourceType(resourceType, row)
   535  	if err != nil {
   536  		if err == sql.ErrNoRows {
   537  			return nil, false, nil
   538  		}
   539  
   540  		return nil, false, err
   541  	}
   542  
   543  	return resourceType, true, nil
   544  }
   545  
   546  func (p *pipeline) Job(name string) (Job, bool, error) {
   547  	row := jobsQuery.Where(sq.Eq{
   548  		"j.name":        name,
   549  		"j.active":      true,
   550  		"j.pipeline_id": p.id,
   551  	}).RunWith(p.conn).QueryRow()
   552  
   553  	job := newEmptyJob(p.conn, p.lockFactory)
   554  	err := scanJob(job, row)
   555  
   556  	if err != nil {
   557  		if err == sql.ErrNoRows {
   558  			return nil, false, nil
   559  		}
   560  
   561  		return nil, false, err
   562  	}
   563  
   564  	return job, true, nil
   565  }
   566  
   567  func (p *pipeline) Jobs() (Jobs, error) {
   568  	rows, err := jobsQuery.
   569  		Where(sq.Eq{
   570  			"pipeline_id": p.id,
   571  			"active":      true,
   572  		}).
   573  		OrderBy("j.id ASC").
   574  		RunWith(p.conn).
   575  		Query()
   576  	if err != nil {
   577  		return nil, err
   578  	}
   579  
   580  	jobs, err := scanJobs(p.conn, p.lockFactory, rows)
   581  	return jobs, err
   582  }
   583  
   584  func (p *pipeline) Dashboard() ([]atc.JobSummary, error) {
   585  	tx, err := p.conn.Begin()
   586  	if err != nil {
   587  		return nil, err
   588  	}
   589  
   590  	defer Rollback(tx)
   591  
   592  	dashboardFactory := newDashboardFactory(tx, sq.Eq{
   593  		"j.pipeline_id": p.id,
   594  	})
   595  
   596  	dashboard, err := dashboardFactory.buildDashboard()
   597  	if err != nil {
   598  		return nil, err
   599  	}
   600  
   601  	err = tx.Commit()
   602  	if err != nil {
   603  		return nil, err
   604  	}
   605  
   606  	return dashboard, nil
   607  }
   608  
   609  func (p *pipeline) Pause() error {
   610  	_, err := psql.Update("pipelines").
   611  		Set("paused", true).
   612  		Where(sq.Eq{
   613  			"id": p.id,
   614  		}).
   615  		RunWith(p.conn).
   616  		Exec()
   617  
   618  	return err
   619  }
   620  
   621  func (p *pipeline) Unpause() error {
   622  	tx, err := p.conn.Begin()
   623  	if err != nil {
   624  		return err
   625  	}
   626  
   627  	defer Rollback(tx)
   628  
   629  	_, err = psql.Update("pipelines").
   630  		Set("paused", false).
   631  		Where(sq.Eq{
   632  			"id": p.id,
   633  		}).
   634  		RunWith(tx).
   635  		Exec()
   636  
   637  	if err != nil {
   638  		return err
   639  	}
   640  
   641  	err = requestScheduleForJobsInPipeline(tx, p.id)
   642  	if err != nil {
   643  		return err
   644  	}
   645  
   646  	return tx.Commit()
   647  }
   648  
   649  func (p *pipeline) Archive() error {
   650  	tx, err := p.conn.Begin()
   651  	if err != nil {
   652  		return err
   653  	}
   654  
   655  	defer Rollback(tx)
   656  	err = p.archive(tx)
   657  	if err != nil {
   658  		return err
   659  	}
   660  
   661  	return tx.Commit()
   662  }
   663  
   664  func (p *pipeline) archive(tx Tx) error {
   665  	_, err := psql.Update("pipelines").
   666  		Set("archived", true).
   667  		Set("last_updated", sq.Expr("now()")).
   668  		Set("paused", true).
   669  		Set("version", 0).
   670  		Where(sq.Eq{
   671  			"id": p.id,
   672  		}).
   673  		RunWith(tx).
   674  		Exec()
   675  
   676  	if err != nil {
   677  		return err
   678  	}
   679  
   680  	err = p.clearConfigForJobsInPipeline(tx)
   681  	if err != nil {
   682  		return err
   683  	}
   684  
   685  	err = p.clearConfigForResourcesInPipeline(tx)
   686  	if err != nil {
   687  		return err
   688  	}
   689  
   690  	return p.clearConfigForResourceTypesInPipeline(tx)
   691  }
   692  
   693  func (p *pipeline) Hide() error {
   694  	_, err := psql.Update("pipelines").
   695  		Set("public", false).
   696  		Where(sq.Eq{
   697  			"id": p.id,
   698  		}).
   699  		RunWith(p.conn).
   700  		Exec()
   701  
   702  	return err
   703  }
   704  
   705  func (p *pipeline) Expose() error {
   706  	_, err := psql.Update("pipelines").
   707  		Set("public", true).
   708  		Where(sq.Eq{
   709  			"id": p.id,
   710  		}).
   711  		RunWith(p.conn).
   712  		Exec()
   713  
   714  	return err
   715  }
   716  
   717  func (p *pipeline) Rename(name string) error {
   718  	_, err := psql.Update("pipelines").
   719  		Set("name", name).
   720  		Where(sq.Eq{
   721  			"id": p.id,
   722  		}).
   723  		RunWith(p.conn).
   724  		Exec()
   725  
   726  	return err
   727  }
   728  
   729  func (p *pipeline) Destroy() error {
   730  	tx, err := p.conn.Begin()
   731  	if err != nil {
   732  		return err
   733  	}
   734  
   735  	defer tx.Rollback()
   736  
   737  	_, err = psql.Delete("pipelines").
   738  		Where(sq.Eq{
   739  			"id": p.id,
   740  		}).
   741  		RunWith(tx).
   742  		Exec()
   743  
   744  	if err != nil {
   745  		return err
   746  	}
   747  
   748  	_, err = psql.Insert("deleted_pipelines").
   749  		Columns("id").
   750  		Values(p.id).
   751  		RunWith(tx).
   752  		Exec()
   753  
   754  	if err != nil {
   755  		return err
   756  	}
   757  
   758  	return tx.Commit()
   759  }
   760  
   761  func (p *pipeline) LoadDebugVersionsDB() (*atc.DebugVersionsDB, error) {
   762  	db := &atc.DebugVersionsDB{
   763  		BuildOutputs:     []atc.DebugBuildOutput{},
   764  		BuildInputs:      []atc.DebugBuildInput{},
   765  		ResourceVersions: []atc.DebugResourceVersion{},
   766  		BuildReruns:      []atc.DebugBuildRerun{},
   767  		Resources:        []atc.DebugResource{},
   768  		Jobs:             []atc.DebugJob{},
   769  	}
   770  
   771  	tx, err := p.conn.Begin()
   772  	if err != nil {
   773  		return nil, err
   774  	}
   775  
   776  	defer tx.Rollback()
   777  
   778  	rows, err := psql.Select("v.id, v.check_order, r.id, v.resource_config_scope_id, o.build_id, b.job_id").
   779  		From("build_resource_config_version_outputs o").
   780  		Join("builds b ON b.id = o.build_id").
   781  		Join("resource_config_versions v ON v.version_md5 = o.version_md5").
   782  		Join("resources r ON r.id = o.resource_id").
   783  		Where(sq.Expr("r.resource_config_scope_id = v.resource_config_scope_id")).
   784  		Where(sq.Expr("(r.id, v.version_md5) NOT IN (SELECT resource_id, version_md5 from resource_disabled_versions)")).
   785  		Where(sq.Eq{
   786  			"b.status":      BuildStatusSucceeded,
   787  			"r.pipeline_id": p.id,
   788  			"r.active":      true,
   789  		}).
   790  		RunWith(tx).
   791  		Query()
   792  	if err != nil {
   793  		return nil, err
   794  	}
   795  
   796  	defer Close(rows)
   797  
   798  	for rows.Next() {
   799  		var output atc.DebugBuildOutput
   800  		err = rows.Scan(&output.VersionID, &output.CheckOrder, &output.ResourceID, &output.ScopeID, &output.BuildID, &output.JobID)
   801  		if err != nil {
   802  			return nil, err
   803  		}
   804  
   805  		output.DebugResourceVersion.CheckOrder = output.CheckOrder
   806  
   807  		db.BuildOutputs = append(db.BuildOutputs, output)
   808  	}
   809  
   810  	rows, err = psql.Select("v.id, v.check_order, r.id, v.resource_config_scope_id, i.build_id, i.name, b.job_id, b.status = 'succeeded'").
   811  		From("build_resource_config_version_inputs i").
   812  		Join("builds b ON b.id = i.build_id").
   813  		Join("resource_config_versions v ON v.version_md5 = i.version_md5").
   814  		Join("resources r ON r.id = i.resource_id").
   815  		Where(sq.Expr("r.resource_config_scope_id = v.resource_config_scope_id")).
   816  		Where(sq.Expr("(r.id, v.version_md5) NOT IN (SELECT resource_id, version_md5 from resource_disabled_versions)")).
   817  		Where(sq.Eq{
   818  			"r.pipeline_id": p.id,
   819  			"r.active":      true,
   820  		}).
   821  		RunWith(tx).
   822  		Query()
   823  	if err != nil {
   824  		return nil, err
   825  	}
   826  
   827  	defer Close(rows)
   828  
   829  	for rows.Next() {
   830  		var succeeded bool
   831  
   832  		var input atc.DebugBuildInput
   833  		err = rows.Scan(&input.VersionID, &input.CheckOrder, &input.ResourceID, &input.ScopeID, &input.BuildID, &input.InputName, &input.JobID, &succeeded)
   834  		if err != nil {
   835  			return nil, err
   836  		}
   837  
   838  		input.DebugResourceVersion.CheckOrder = input.CheckOrder
   839  
   840  		db.BuildInputs = append(db.BuildInputs, input)
   841  
   842  		if succeeded {
   843  			// implicit output
   844  			db.BuildOutputs = append(db.BuildOutputs, atc.DebugBuildOutput{
   845  				DebugResourceVersion: input.DebugResourceVersion,
   846  				JobID:                input.JobID,
   847  				BuildID:              input.BuildID,
   848  			})
   849  		}
   850  	}
   851  
   852  	rows, err = psql.Select("v.id, v.check_order, r.id, v.resource_config_scope_id").
   853  		From("resource_config_versions v").
   854  		Join("resources r ON r.resource_config_scope_id = v.resource_config_scope_id").
   855  		LeftJoin("resource_disabled_versions d ON d.resource_id = r.id AND d.version_md5 = v.version_md5").
   856  		Where(sq.Eq{
   857  			"r.pipeline_id": p.id,
   858  			"r.active":      true,
   859  			"d.resource_id": nil,
   860  			"d.version_md5": nil,
   861  		}).
   862  		RunWith(tx).
   863  		Query()
   864  	if err != nil {
   865  		return nil, err
   866  	}
   867  
   868  	defer Close(rows)
   869  
   870  	for rows.Next() {
   871  		var output atc.DebugResourceVersion
   872  		err = rows.Scan(&output.VersionID, &output.CheckOrder, &output.ResourceID, &output.ScopeID)
   873  		if err != nil {
   874  			return nil, err
   875  		}
   876  
   877  		db.ResourceVersions = append(db.ResourceVersions, output)
   878  	}
   879  
   880  	rows, err = psql.Select("j.id, b.id, b.rerun_of").
   881  		From("builds b").
   882  		Join("jobs j ON j.id = b.job_id").
   883  		Where(sq.Eq{
   884  			"j.active":      true,
   885  			"b.pipeline_id": p.id,
   886  		}).
   887  		Where(sq.NotEq{
   888  			"b.rerun_of": nil,
   889  		}).
   890  		RunWith(tx).
   891  		Query()
   892  	if err != nil {
   893  		return nil, err
   894  	}
   895  
   896  	defer Close(rows)
   897  
   898  	for rows.Next() {
   899  		var rerun atc.DebugBuildRerun
   900  		err = rows.Scan(&rerun.JobID, &rerun.BuildID, &rerun.RerunOf)
   901  		if err != nil {
   902  			return nil, err
   903  		}
   904  
   905  		db.BuildReruns = append(db.BuildReruns, rerun)
   906  	}
   907  
   908  	rows, err = psql.Select("j.name, j.id").
   909  		From("jobs j").
   910  		Where(sq.Eq{
   911  			"j.pipeline_id": p.id,
   912  			"j.active":      true,
   913  		}).
   914  		RunWith(tx).
   915  		Query()
   916  	if err != nil {
   917  		return nil, err
   918  	}
   919  
   920  	defer Close(rows)
   921  
   922  	for rows.Next() {
   923  		var job atc.DebugJob
   924  		err = rows.Scan(&job.Name, &job.ID)
   925  		if err != nil {
   926  			return nil, err
   927  		}
   928  
   929  		db.Jobs = append(db.Jobs, job)
   930  	}
   931  
   932  	rows, err = psql.Select("r.name, r.id, r.resource_config_scope_id").
   933  		From("resources r").
   934  		Where(sq.Eq{
   935  			"r.pipeline_id": p.id,
   936  			"r.active":      true,
   937  		}).
   938  		RunWith(tx).
   939  		Query()
   940  	if err != nil {
   941  		return nil, err
   942  	}
   943  
   944  	defer Close(rows)
   945  
   946  	for rows.Next() {
   947  		var scopeID sql.NullInt64
   948  		var resource atc.DebugResource
   949  		err = rows.Scan(&resource.Name, &resource.ID, &scopeID)
   950  		if err != nil {
   951  			return nil, err
   952  		}
   953  
   954  		if scopeID.Valid {
   955  			i := int(scopeID.Int64)
   956  			resource.ScopeID = &i
   957  		}
   958  
   959  		db.Resources = append(db.Resources, resource)
   960  	}
   961  
   962  	err = tx.Commit()
   963  	if err != nil {
   964  		return nil, err
   965  	}
   966  
   967  	return db, nil
   968  }
   969  
   970  func (p *pipeline) DeleteBuildEventsByBuildIDs(buildIDs []int) error {
   971  	if len(buildIDs) == 0 {
   972  		return nil
   973  	}
   974  
   975  	interfaceBuildIDs := make([]interface{}, len(buildIDs))
   976  	for i, buildID := range buildIDs {
   977  		interfaceBuildIDs[i] = buildID
   978  	}
   979  
   980  	indexStrings := make([]string, len(buildIDs))
   981  	for i := range indexStrings {
   982  		indexStrings[i] = "$" + strconv.Itoa(i+1)
   983  	}
   984  
   985  	tx, err := p.conn.Begin()
   986  	if err != nil {
   987  		return err
   988  	}
   989  
   990  	defer Rollback(tx)
   991  
   992  	_, err = tx.Exec(`
   993     DELETE FROM build_events
   994  	 WHERE build_id IN (`+strings.Join(indexStrings, ",")+`)
   995  	 `, interfaceBuildIDs...)
   996  	if err != nil {
   997  		return err
   998  	}
   999  
  1000  	_, err = tx.Exec(`
  1001  		UPDATE builds
  1002  		SET reap_time = now()
  1003  		WHERE id IN (`+strings.Join(indexStrings, ",")+`)
  1004  	`, interfaceBuildIDs...)
  1005  	if err != nil {
  1006  		return err
  1007  	}
  1008  
  1009  	err = tx.Commit()
  1010  	return err
  1011  }
  1012  
  1013  func (p *pipeline) CreateOneOffBuild() (Build, error) {
  1014  	tx, err := p.conn.Begin()
  1015  	if err != nil {
  1016  		return nil, err
  1017  	}
  1018  
  1019  	defer Rollback(tx)
  1020  
  1021  	build := newEmptyBuild(p.conn, p.lockFactory)
  1022  	err = createBuild(tx, build, map[string]interface{}{
  1023  		"name":        sq.Expr("nextval('one_off_name')"),
  1024  		"pipeline_id": p.id,
  1025  		"team_id":     p.teamID,
  1026  		"status":      BuildStatusPending,
  1027  	})
  1028  	if err != nil {
  1029  		return nil, err
  1030  	}
  1031  
  1032  	err = tx.Commit()
  1033  	if err != nil {
  1034  		return nil, err
  1035  	}
  1036  
  1037  	return build, nil
  1038  }
  1039  
  1040  func (p *pipeline) CreateStartedBuild(plan atc.Plan) (Build, error) {
  1041  	tx, err := p.conn.Begin()
  1042  	if err != nil {
  1043  		return nil, err
  1044  	}
  1045  
  1046  	defer Rollback(tx)
  1047  
  1048  	metadata, err := json.Marshal(plan)
  1049  	if err != nil {
  1050  		return nil, err
  1051  	}
  1052  
  1053  	encryptedPlan, nonce, err := p.conn.EncryptionStrategy().Encrypt(metadata)
  1054  	if err != nil {
  1055  		return nil, err
  1056  	}
  1057  
  1058  	build := newEmptyBuild(p.conn, p.lockFactory)
  1059  	err = createBuild(tx, build, map[string]interface{}{
  1060  		"name":         sq.Expr("nextval('one_off_name')"),
  1061  		"pipeline_id":  p.id,
  1062  		"team_id":      p.teamID,
  1063  		"status":       BuildStatusStarted,
  1064  		"start_time":   sq.Expr("now()"),
  1065  		"schema":       schema,
  1066  		"private_plan": encryptedPlan,
  1067  		"public_plan":  plan.Public(),
  1068  		"nonce":        nonce,
  1069  	})
  1070  	if err != nil {
  1071  		return nil, err
  1072  	}
  1073  
  1074  	err = build.saveEvent(tx, event.Status{
  1075  		Status: atc.StatusStarted,
  1076  		Time:   build.StartTime().Unix(),
  1077  	})
  1078  	if err != nil {
  1079  		return nil, err
  1080  	}
  1081  
  1082  	err = tx.Commit()
  1083  	if err != nil {
  1084  		return nil, err
  1085  	}
  1086  
  1087  	if err = p.conn.Bus().Notify(buildStartedChannel()); err != nil {
  1088  		return nil, err
  1089  	}
  1090  
  1091  	if err = p.conn.Bus().Notify(buildEventsChannel(build.id)); err != nil {
  1092  		return nil, err
  1093  	}
  1094  
  1095  	return build, nil
  1096  }
  1097  
  1098  func (p *pipeline) getBuildsFrom(tx Tx, col string) (map[string]Build, error) {
  1099  	rows, err := buildsQuery.
  1100  		Where(sq.Eq{
  1101  			"b.pipeline_id": p.id,
  1102  		}).
  1103  		Where(sq.Expr("j." + col + " = b.id")).
  1104  		RunWith(tx).Query()
  1105  	if err != nil {
  1106  		return nil, err
  1107  	}
  1108  
  1109  	defer Close(rows)
  1110  
  1111  	nextBuilds := make(map[string]Build)
  1112  
  1113  	for rows.Next() {
  1114  		build := newEmptyBuild(p.conn, p.lockFactory)
  1115  		err := scanBuild(build, rows, p.conn.EncryptionStrategy())
  1116  		if err != nil {
  1117  			return nil, err
  1118  		}
  1119  		nextBuilds[build.JobName()] = build
  1120  	}
  1121  
  1122  	return nextBuilds, nil
  1123  }
  1124  
  1125  // Variables creates variables for this pipeline. If this pipeline has its own
  1126  // var_sources, a vars.MultiVars containing all pipeline specific var_sources
  1127  // plug the global variables, otherwise just return the global variables.
  1128  func (p *pipeline) Variables(logger lager.Logger, globalSecrets creds.Secrets, varSourcePool creds.VarSourcePool) (vars.Variables, error) {
  1129  	globalVars := creds.NewVariables(globalSecrets, p.TeamName(), p.Name(), false)
  1130  	namedVarsMap := vars.NamedVariables{}
  1131  
  1132  	// It's safe to add NamedVariables to allVars via an array here, because
  1133  	// a map is passed by reference.
  1134  	allVars := vars.NewMultiVars([]vars.Variables{namedVarsMap, globalVars})
  1135  
  1136  	orderedVarSources, err := p.varSources.OrderByDependency()
  1137  	if err != nil {
  1138  		return nil, err
  1139  	}
  1140  
  1141  	for _, cm := range orderedVarSources {
  1142  		factory := creds.ManagerFactories()[cm.Type]
  1143  		if factory == nil {
  1144  			return nil, fmt.Errorf("unknown credential manager type: %s", cm.Type)
  1145  		}
  1146  
  1147  		// Interpolate variables in pipeline credential manager's config
  1148  		newConfig, err := creds.NewParams(allVars, atc.Params{"config": cm.Config}).Evaluate()
  1149  		if err != nil {
  1150  			return nil, errors.Wrapf(err, "evaluate var_source '%s' error", cm.Name)
  1151  		}
  1152  
  1153  		config, ok := newConfig["config"].(map[string]interface{})
  1154  		if !ok {
  1155  			return nil, fmt.Errorf("var_source '%s' invalid config", cm.Name)
  1156  		}
  1157  		secrets, err := varSourcePool.FindOrCreate(logger, config, factory)
  1158  		if err != nil {
  1159  			return nil, errors.Wrapf(err, "create var_source '%s' error", cm.Name)
  1160  		}
  1161  		namedVarsMap[cm.Name] = creds.NewVariables(secrets, p.TeamName(), p.Name(), true)
  1162  	}
  1163  
  1164  	// If there is no var_source from the pipeline, then just return the global
  1165  	// vars.
  1166  	if len(namedVarsMap) == 0 {
  1167  		return globalVars, nil
  1168  	}
  1169  
  1170  	return allVars, nil
  1171  }
  1172  
  1173  func (p *pipeline) SetParentIDs(jobID, buildID int) error {
  1174  	if jobID <= 0 || buildID <= 0 {
  1175  		return errors.New("job and build id cannot be negative or zero-value")
  1176  	}
  1177  
  1178  	tx, err := p.conn.Begin()
  1179  	if err != nil {
  1180  		return err
  1181  	}
  1182  
  1183  	defer Rollback(tx)
  1184  
  1185  	result, err := psql.Update("pipelines").
  1186  		Set("parent_job_id", jobID).
  1187  		Set("parent_build_id", buildID).
  1188  		Where(sq.Eq{
  1189  			"id": p.id,
  1190  		}).
  1191  		Where(sq.Or{sq.Lt{"parent_build_id": buildID}, sq.Eq{"parent_build_id": nil}}).
  1192  		RunWith(tx).
  1193  		Exec()
  1194  
  1195  	if err != nil {
  1196  		return err
  1197  	}
  1198  
  1199  	rows, err := result.RowsAffected()
  1200  	if err != nil {
  1201  		return err
  1202  	}
  1203  	if rows == 0 {
  1204  		return ErrSetByNewerBuild
  1205  	}
  1206  
  1207  	return tx.Commit()
  1208  }
  1209  
  1210  func getNewBuildNameForJob(tx Tx, jobName string, pipelineID int) (string, int, error) {
  1211  	var buildName string
  1212  	var jobID int
  1213  	err := tx.QueryRow(`
  1214  		UPDATE jobs
  1215  		SET build_number_seq = build_number_seq + 1
  1216  		WHERE name = $1 AND pipeline_id = $2
  1217  		RETURNING build_number_seq, id
  1218  	`, jobName, pipelineID).Scan(&buildName, &jobID)
  1219  	return buildName, jobID, err
  1220  }
  1221  
  1222  func resources(pipelineID int, conn Conn, lockFactory lock.LockFactory) (Resources, error) {
  1223  	rows, err := resourcesQuery.
  1224  		Where(sq.Eq{"r.pipeline_id": pipelineID}).
  1225  		OrderBy("r.name").
  1226  		RunWith(conn).
  1227  		Query()
  1228  	if err != nil {
  1229  		return nil, err
  1230  	}
  1231  	defer Close(rows)
  1232  
  1233  	var resources Resources
  1234  
  1235  	for rows.Next() {
  1236  		newResource := newEmptyResource(conn, lockFactory)
  1237  		err := scanResource(newResource, rows)
  1238  		if err != nil {
  1239  			return nil, err
  1240  		}
  1241  
  1242  		resources = append(resources, newResource)
  1243  	}
  1244  
  1245  	return resources, nil
  1246  }
  1247  
  1248  // The SELECT query orders the jobs for updating to prevent deadlocking.
  1249  // Updating multiple rows using a SELECT subquery does not preserve the same
  1250  // order for the updates, which can lead to deadlocking.
  1251  func requestScheduleForJobsInPipeline(tx Tx, pipelineID int) error {
  1252  	rows, err := psql.Select("id").
  1253  		From("jobs").
  1254  		Where(sq.Eq{
  1255  			"pipeline_id": pipelineID,
  1256  		}).
  1257  		OrderBy("id DESC").
  1258  		RunWith(tx).
  1259  		Query()
  1260  	if err != nil {
  1261  		return err
  1262  	}
  1263  
  1264  	var jobIDs []int
  1265  	for rows.Next() {
  1266  		var id int
  1267  		err = rows.Scan(&id)
  1268  		if err != nil {
  1269  			return err
  1270  		}
  1271  
  1272  		jobIDs = append(jobIDs, id)
  1273  	}
  1274  
  1275  	for _, jID := range jobIDs {
  1276  		_, err := psql.Update("jobs").
  1277  			Set("schedule_requested", sq.Expr("now()")).
  1278  			Where(sq.Eq{
  1279  				"id": jID,
  1280  			}).
  1281  			RunWith(tx).
  1282  			Exec()
  1283  		if err != nil {
  1284  			return err
  1285  		}
  1286  	}
  1287  
  1288  	return nil
  1289  }
  1290  
  1291  func (p *pipeline) clearConfigForJobsInPipeline(tx Tx) error {
  1292  	_, err := psql.Update("jobs").
  1293  		Set("config", nil).
  1294  		Set("nonce", nil).
  1295  		Where(sq.Eq{"pipeline_id": p.id}).
  1296  		RunWith(tx).
  1297  		Exec()
  1298  	if err != nil {
  1299  		return err
  1300  	}
  1301  
  1302  	return nil
  1303  }
  1304  
  1305  func (p *pipeline) clearConfigForResourcesInPipeline(tx Tx) error {
  1306  	_, err := psql.Update("resources").
  1307  		Set("config", nil).
  1308  		Set("nonce", nil).
  1309  		Where(sq.Eq{"pipeline_id": p.id}).
  1310  		RunWith(tx).
  1311  		Exec()
  1312  	if err != nil {
  1313  		return err
  1314  	}
  1315  
  1316  	return nil
  1317  }
  1318  
  1319  func (p *pipeline) clearConfigForResourceTypesInPipeline(tx Tx) error {
  1320  	_, err := psql.Update("resource_types").
  1321  		Set("config", nil).
  1322  		Set("nonce", nil).
  1323  		Where(sq.Eq{"pipeline_id": p.id}).
  1324  		RunWith(tx).
  1325  		Exec()
  1326  	if err != nil {
  1327  		return err
  1328  	}
  1329  
  1330  	return nil
  1331  }