github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/db/job_factory.go (about)

     1  package db
     2  
     3  import (
     4  	"database/sql"
     5  	"encoding/json"
     6  	"sort"
     7  
     8  	sq "github.com/Masterminds/squirrel"
     9  	"github.com/pf-qiu/concourse/v6/atc"
    10  	"github.com/pf-qiu/concourse/v6/atc/db/lock"
    11  	"github.com/lib/pq"
    12  )
    13  
    14  //go:generate counterfeiter . JobFactory
    15  
    16  // XXX: This job factory object is not really a job factory anymore. It is
    17  // holding the responsibility for two very different things: constructing a
    18  // dashboard object and also a scheduler job object. Figure out what this is
    19  // trying to encapsulate or considering splitting this out!
    20  type JobFactory interface {
    21  	VisibleJobs([]string) ([]atc.JobSummary, error)
    22  	AllActiveJobs() ([]atc.JobSummary, error)
    23  	JobsToSchedule() (SchedulerJobs, error)
    24  }
    25  
    26  type jobFactory struct {
    27  	conn        Conn
    28  	lockFactory lock.LockFactory
    29  }
    30  
    31  func NewJobFactory(conn Conn, lockFactory lock.LockFactory) JobFactory {
    32  	return &jobFactory{
    33  		conn:        conn,
    34  		lockFactory: lockFactory,
    35  	}
    36  }
    37  
    38  type SchedulerJobs []SchedulerJob
    39  
    40  type SchedulerJob struct {
    41  	Job
    42  	Resources     SchedulerResources
    43  	ResourceTypes atc.VersionedResourceTypes
    44  }
    45  
    46  type SchedulerResources []SchedulerResource
    47  
    48  type SchedulerResource struct {
    49  	Name   string
    50  	Type   string
    51  	Source atc.Source
    52  }
    53  
    54  func (r *SchedulerResource) ApplySourceDefaults(resourceTypes atc.VersionedResourceTypes) {
    55  	parentType, found := resourceTypes.Lookup(r.Type)
    56  	if found {
    57  		r.Source = parentType.Defaults.Merge(r.Source)
    58  	} else {
    59  		defaults, found := atc.FindBaseResourceTypeDefaults(r.Type)
    60  		if found {
    61  			r.Source = defaults.Merge(r.Source)
    62  		}
    63  	}
    64  }
    65  
    66  func (resources SchedulerResources) Lookup(name string) (*SchedulerResource, bool) {
    67  	for _, resource := range resources {
    68  		if resource.Name == name {
    69  			return &resource, true
    70  		}
    71  	}
    72  
    73  	return nil, false
    74  }
    75  
    76  func (j *jobFactory) JobsToSchedule() (SchedulerJobs, error) {
    77  	tx, err := j.conn.Begin()
    78  	if err != nil {
    79  		return nil, err
    80  	}
    81  
    82  	defer tx.Rollback()
    83  
    84  	rows, err := jobsQuery.
    85  		Where(sq.Expr("j.schedule_requested > j.last_scheduled")).
    86  		Where(sq.Eq{
    87  			"j.active": true,
    88  			"j.paused": false,
    89  			"p.paused": false,
    90  		}).
    91  		RunWith(tx).
    92  		Query()
    93  	if err != nil {
    94  		return nil, err
    95  	}
    96  
    97  	jobs, err := scanJobs(j.conn, j.lockFactory, rows)
    98  	if err != nil {
    99  		return nil, err
   100  	}
   101  
   102  	var schedulerJobs SchedulerJobs
   103  	pipelineResourceTypes := make(map[int]ResourceTypes)
   104  	for _, job := range jobs {
   105  		rows, err := tx.Query(`WITH inputs AS (
   106  				SELECT ji.resource_id from job_inputs ji where ji.job_id = $1
   107  				UNION
   108  				SELECT jo.resource_id from job_outputs jo where jo.job_id = $1
   109  			)
   110  			SELECT r.name, r.type, r.config, r.nonce
   111  			From resources r
   112  			Join inputs i on i.resource_id = r.id`, job.ID())
   113  		if err != nil {
   114  			return nil, err
   115  		}
   116  
   117  		var schedulerResources SchedulerResources
   118  		for rows.Next() {
   119  			var name, type_ string
   120  			var configBlob []byte
   121  			var nonce sql.NullString
   122  
   123  			err = rows.Scan(&name, &type_, &configBlob, &nonce)
   124  			if err != nil {
   125  				return nil, err
   126  			}
   127  
   128  			defer Close(rows)
   129  
   130  			es := j.conn.EncryptionStrategy()
   131  
   132  			var noncense *string
   133  			if nonce.Valid {
   134  				noncense = &nonce.String
   135  			}
   136  
   137  			decryptedConfig, err := es.Decrypt(string(configBlob), noncense)
   138  			if err != nil {
   139  				return nil, err
   140  			}
   141  
   142  			var config atc.ResourceConfig
   143  			err = json.Unmarshal(decryptedConfig, &config)
   144  			if err != nil {
   145  				return nil, err
   146  			}
   147  
   148  			schedulerResources = append(schedulerResources, SchedulerResource{
   149  				Name:   name,
   150  				Type:   type_,
   151  				Source: config.Source,
   152  			})
   153  		}
   154  
   155  		var resourceTypes ResourceTypes
   156  		var found bool
   157  		resourceTypes, found = pipelineResourceTypes[job.PipelineID()]
   158  		if !found {
   159  			rows, err := resourceTypesQuery.
   160  				Where(sq.Eq{"r.pipeline_id": job.PipelineID()}).
   161  				OrderBy("r.name").
   162  				RunWith(tx).
   163  				Query()
   164  			if err != nil {
   165  				return nil, err
   166  			}
   167  
   168  			defer Close(rows)
   169  
   170  			for rows.Next() {
   171  				resourceType := newEmptyResourceType(j.conn, j.lockFactory)
   172  				err := scanResourceType(resourceType, rows)
   173  				if err != nil {
   174  					return nil, err
   175  				}
   176  
   177  				resourceTypes = append(resourceTypes, resourceType)
   178  			}
   179  
   180  			pipelineResourceTypes[job.PipelineID()] = resourceTypes
   181  		}
   182  
   183  		schedulerJobs = append(schedulerJobs, SchedulerJob{
   184  			Job:           job,
   185  			Resources:     schedulerResources,
   186  			ResourceTypes: resourceTypes.Deserialize(),
   187  		})
   188  	}
   189  
   190  	err = tx.Commit()
   191  	if err != nil {
   192  		return nil, err
   193  	}
   194  
   195  	return schedulerJobs, nil
   196  }
   197  
   198  func (j *jobFactory) VisibleJobs(teamNames []string) ([]atc.JobSummary, error) {
   199  	tx, err := j.conn.Begin()
   200  	if err != nil {
   201  		return nil, err
   202  	}
   203  
   204  	defer Rollback(tx)
   205  
   206  	dashboardFactory := newDashboardFactory(tx, sq.Or{
   207  		sq.Eq{"tm.name": teamNames},
   208  		sq.Eq{"p.public": true},
   209  	})
   210  
   211  	dashboard, err := dashboardFactory.buildDashboard()
   212  	if err != nil {
   213  		return nil, err
   214  	}
   215  
   216  	err = tx.Commit()
   217  	if err != nil {
   218  		return nil, err
   219  	}
   220  
   221  	return dashboard, nil
   222  }
   223  
   224  func (j *jobFactory) AllActiveJobs() ([]atc.JobSummary, error) {
   225  	tx, err := j.conn.Begin()
   226  	if err != nil {
   227  		return nil, err
   228  	}
   229  
   230  	defer Rollback(tx)
   231  
   232  	dashboardFactory := newDashboardFactory(tx, nil)
   233  	dashboard, err := dashboardFactory.buildDashboard()
   234  	if err != nil {
   235  		return nil, err
   236  	}
   237  
   238  	err = tx.Commit()
   239  	if err != nil {
   240  		return nil, err
   241  	}
   242  
   243  	return dashboard, nil
   244  }
   245  
   246  type dashboardFactory struct {
   247  	// Constraints that are used by the dashboard queries. For example, a job ID
   248  	// constraint so that the dashboard will only return the job I have access to
   249  	// see.
   250  	pred interface{}
   251  
   252  	tx Tx
   253  }
   254  
   255  func newDashboardFactory(tx Tx, pred interface{}) dashboardFactory {
   256  	return dashboardFactory{
   257  		pred: pred,
   258  		tx:   tx,
   259  	}
   260  }
   261  
   262  func (d dashboardFactory) buildDashboard() ([]atc.JobSummary, error) {
   263  	dashboard, err := d.constructJobsForDashboard()
   264  	if err != nil {
   265  		return nil, err
   266  	}
   267  
   268  	jobInputs, err := d.fetchJobInputs()
   269  	if err != nil {
   270  		return nil, err
   271  	}
   272  
   273  	jobOutputs, err := d.fetchJobOutputs()
   274  	if err != nil {
   275  		return nil, err
   276  	}
   277  
   278  	return d.combineJobInputsAndOutputsWithDashboardJobs(dashboard, jobInputs, jobOutputs), nil
   279  }
   280  
   281  func (d dashboardFactory) constructJobsForDashboard() ([]atc.JobSummary, error) {
   282  	rows, err := psql.Select("j.id", "j.name", "p.id", "p.name", "p.instance_vars", "j.paused", "j.has_new_inputs", "j.tags", "tm.name",
   283  		"l.id", "l.name", "l.status", "l.start_time", "l.end_time",
   284  		"n.id", "n.name", "n.status", "n.start_time", "n.end_time",
   285  		"t.id", "t.name", "t.status", "t.start_time", "t.end_time").
   286  		From("jobs j").
   287  		Join("pipelines p ON j.pipeline_id = p.id").
   288  		Join("teams tm ON p.team_id = tm.id").
   289  		LeftJoin("builds l on j.latest_completed_build_id = l.id").
   290  		LeftJoin("builds n on j.next_build_id = n.id").
   291  		LeftJoin("builds t on j.transition_build_id = t.id").
   292  		Where(sq.Eq{
   293  			"j.active": true,
   294  		}).
   295  		Where(d.pred).
   296  		OrderBy("j.id ASC").
   297  		RunWith(d.tx).
   298  		Query()
   299  	if err != nil {
   300  		return nil, err
   301  	}
   302  
   303  	type nullableBuild struct {
   304  		id        sql.NullInt64
   305  		name      sql.NullString
   306  		jobName   sql.NullString
   307  		status    sql.NullString
   308  		startTime pq.NullTime
   309  		endTime   pq.NullTime
   310  	}
   311  
   312  	var dashboard []atc.JobSummary
   313  	for rows.Next() {
   314  		var (
   315  			f, n, t nullableBuild
   316  
   317  			pipelineInstanceVars sql.NullString
   318  		)
   319  
   320  		j := atc.JobSummary{}
   321  		err = rows.Scan(&j.ID, &j.Name, &j.PipelineID, &j.PipelineName, &pipelineInstanceVars, &j.Paused, &j.HasNewInputs, pq.Array(&j.Groups), &j.TeamName,
   322  			&f.id, &f.name, &f.status, &f.startTime, &f.endTime,
   323  			&n.id, &n.name, &n.status, &n.startTime, &n.endTime,
   324  			&t.id, &t.name, &t.status, &t.startTime, &t.endTime)
   325  		if err != nil {
   326  			return nil, err
   327  		}
   328  
   329  		if pipelineInstanceVars.Valid {
   330  			err = json.Unmarshal([]byte(pipelineInstanceVars.String), &j.PipelineInstanceVars)
   331  			if err != nil {
   332  				return nil, err
   333  			}
   334  		}
   335  
   336  		if f.id.Valid {
   337  			j.FinishedBuild = &atc.BuildSummary{
   338  				ID:                   int(f.id.Int64),
   339  				Name:                 f.name.String,
   340  				JobName:              j.Name,
   341  				PipelineID:           j.PipelineID,
   342  				PipelineName:         j.PipelineName,
   343  				PipelineInstanceVars: j.PipelineInstanceVars,
   344  				TeamName:             j.TeamName,
   345  				Status:               atc.BuildStatus(f.status.String),
   346  				StartTime:            f.startTime.Time.Unix(),
   347  				EndTime:              f.endTime.Time.Unix(),
   348  			}
   349  		}
   350  
   351  		if n.id.Valid {
   352  			j.NextBuild = &atc.BuildSummary{
   353  				ID:                   int(n.id.Int64),
   354  				Name:                 n.name.String,
   355  				JobName:              j.Name,
   356  				PipelineID:           j.PipelineID,
   357  				PipelineName:         j.PipelineName,
   358  				PipelineInstanceVars: j.PipelineInstanceVars,
   359  				TeamName:             j.TeamName,
   360  				Status:               atc.BuildStatus(n.status.String),
   361  				StartTime:            n.startTime.Time.Unix(),
   362  				EndTime:              n.endTime.Time.Unix(),
   363  			}
   364  		}
   365  
   366  		if t.id.Valid {
   367  			j.TransitionBuild = &atc.BuildSummary{
   368  				ID:                   int(t.id.Int64),
   369  				Name:                 t.name.String,
   370  				JobName:              j.Name,
   371  				PipelineID:           j.PipelineID,
   372  				PipelineName:         j.PipelineName,
   373  				PipelineInstanceVars: j.PipelineInstanceVars,
   374  				TeamName:             j.TeamName,
   375  				Status:               atc.BuildStatus(t.status.String),
   376  				StartTime:            t.startTime.Time.Unix(),
   377  				EndTime:              t.endTime.Time.Unix(),
   378  			}
   379  		}
   380  
   381  		dashboard = append(dashboard, j)
   382  	}
   383  
   384  	return dashboard, nil
   385  }
   386  
   387  func (d dashboardFactory) fetchJobInputs() (map[int][]atc.JobInputSummary, error) {
   388  	rows, err := psql.Select("j.id", "i.name", "r.name", "array_agg(jp.name ORDER BY jp.id)", "i.trigger").
   389  		From("job_inputs i").
   390  		Join("jobs j ON j.id = i.job_id").
   391  		Join("pipelines p ON p.id = j.pipeline_id").
   392  		Join("teams tm ON tm.id = p.team_id").
   393  		Join("resources r ON r.id = i.resource_id").
   394  		LeftJoin("jobs jp ON jp.id = i.passed_job_id").
   395  		Where(sq.Eq{
   396  			"j.active": true,
   397  		}).
   398  		Where(d.pred).
   399  		GroupBy("i.name, j.id, r.name, i.trigger").
   400  		OrderBy("j.id").
   401  		RunWith(d.tx).
   402  		Query()
   403  	if err != nil {
   404  		return nil, err
   405  	}
   406  
   407  	jobInputs := make(map[int][]atc.JobInputSummary)
   408  	for rows.Next() {
   409  		var passedString []sql.NullString
   410  		var inputName, resourceName string
   411  		var jobID int
   412  		var trigger bool
   413  
   414  		err = rows.Scan(&jobID, &inputName, &resourceName, pq.Array(&passedString), &trigger)
   415  		if err != nil {
   416  			return nil, err
   417  		}
   418  
   419  		var passed []string
   420  		for _, s := range passedString {
   421  			if s.Valid {
   422  				passed = append(passed, s.String)
   423  			}
   424  		}
   425  
   426  		jobInputs[jobID] = append(jobInputs[jobID], atc.JobInputSummary{
   427  			Name:     inputName,
   428  			Resource: resourceName,
   429  			Trigger:  trigger,
   430  			Passed:   passed,
   431  		})
   432  	}
   433  
   434  	return jobInputs, nil
   435  }
   436  
   437  func (d dashboardFactory) fetchJobOutputs() (map[int][]atc.JobOutputSummary, error) {
   438  	rows, err := psql.Select("o.name", "r.name", "o.job_id").
   439  		From("job_outputs o").
   440  		Join("jobs j ON j.id = o.job_id").
   441  		Join("pipelines p ON p.id = j.pipeline_id").
   442  		Join("teams tm ON tm.id = p.team_id").
   443  		Join("resources r ON r.id = o.resource_id").
   444  		Where(d.pred).
   445  		Where(sq.Eq{
   446  			"j.active": true,
   447  		}).
   448  		OrderBy("j.id").
   449  		RunWith(d.tx).
   450  		Query()
   451  	if err != nil {
   452  		return nil, err
   453  	}
   454  
   455  	jobOutputs := make(map[int][]atc.JobOutputSummary)
   456  	for rows.Next() {
   457  		var output atc.JobOutputSummary
   458  		var jobID int
   459  		err = rows.Scan(&output.Name, &output.Resource, &jobID)
   460  		if err != nil {
   461  			return nil, err
   462  		}
   463  
   464  		jobOutputs[jobID] = append(jobOutputs[jobID], output)
   465  	}
   466  
   467  	return jobOutputs, err
   468  }
   469  
   470  func (d dashboardFactory) combineJobInputsAndOutputsWithDashboardJobs(dashboard []atc.JobSummary, jobInputs map[int][]atc.JobInputSummary, jobOutputs map[int][]atc.JobOutputSummary) []atc.JobSummary {
   471  	var finalDashboard []atc.JobSummary
   472  	for _, job := range dashboard {
   473  		for _, input := range jobInputs[job.ID] {
   474  			job.Inputs = append(job.Inputs, input)
   475  		}
   476  
   477  		sort.Slice(job.Inputs, func(p, q int) bool {
   478  			return job.Inputs[p].Name < job.Inputs[q].Name
   479  		})
   480  
   481  		for _, output := range jobOutputs[job.ID] {
   482  			job.Outputs = append(job.Outputs, output)
   483  		}
   484  
   485  		sort.Slice(job.Outputs, func(p, q int) bool {
   486  			return job.Outputs[p].Name < job.Outputs[q].Name
   487  		})
   488  
   489  		finalDashboard = append(finalDashboard, job)
   490  	}
   491  
   492  	return finalDashboard
   493  }