github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/exec/task_step.go (about)

     1  package exec
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"path"
     8  	"path/filepath"
     9  	"strings"
    10  
    11  	"code.cloudfoundry.org/lager"
    12  	"code.cloudfoundry.org/lager/lagerctx"
    13  	"github.com/pf-qiu/concourse/v6/atc"
    14  	"github.com/pf-qiu/concourse/v6/atc/db"
    15  	"github.com/pf-qiu/concourse/v6/atc/db/lock"
    16  	"github.com/pf-qiu/concourse/v6/atc/exec/build"
    17  	"github.com/pf-qiu/concourse/v6/atc/runtime"
    18  	"github.com/pf-qiu/concourse/v6/atc/worker"
    19  	"github.com/pf-qiu/concourse/v6/tracing"
    20  	"github.com/pf-qiu/concourse/v6/vars"
    21  	"go.opentelemetry.io/otel/api/trace"
    22  )
    23  
    24  // MissingInputsError is returned when any of the task's required inputs are
    25  // missing.
    26  type MissingInputsError struct {
    27  	Inputs []string
    28  }
    29  
    30  // Error prints a human-friendly message listing the inputs that were missing.
    31  func (err MissingInputsError) Error() string {
    32  	return fmt.Sprintf("missing inputs: %s", strings.Join(err.Inputs, ", "))
    33  }
    34  
    35  type MissingTaskImageSourceError struct {
    36  	SourceName string
    37  }
    38  
    39  func (err MissingTaskImageSourceError) Error() string {
    40  	return fmt.Sprintf(`missing image artifact source: %s
    41  
    42  make sure there's a corresponding 'get' step, or a task that produces it as an output`, err.SourceName)
    43  }
    44  
    45  type TaskImageSourceParametersError struct {
    46  	Err error
    47  }
    48  
    49  func (err TaskImageSourceParametersError) Error() string {
    50  	return fmt.Sprintf("failed to evaluate image resource parameters: %s", err.Err)
    51  }
    52  
    53  //go:generate counterfeiter . TaskDelegateFactory
    54  
    55  type TaskDelegateFactory interface {
    56  	TaskDelegate(state RunState) TaskDelegate
    57  }
    58  
    59  //go:generate counterfeiter . TaskDelegate
    60  
    61  type TaskDelegate interface {
    62  	StartSpan(context.Context, string, tracing.Attrs) (context.Context, trace.Span)
    63  
    64  	FetchImage(context.Context, atc.ImageResource, atc.VersionedResourceTypes, bool) (worker.ImageSpec, error)
    65  
    66  	Stdout() io.Writer
    67  	Stderr() io.Writer
    68  
    69  	SetTaskConfig(config atc.TaskConfig)
    70  
    71  	Initializing(lager.Logger)
    72  	Starting(lager.Logger)
    73  	Finished(lager.Logger, ExitStatus)
    74  	SelectedWorker(lager.Logger, string)
    75  	Errored(lager.Logger, string)
    76  }
    77  
    78  // TaskStep executes a TaskConfig, whose inputs will be fetched from the
    79  // artifact.Repository and outputs will be added to the artifact.Repository.
    80  type TaskStep struct {
    81  	planID            atc.PlanID
    82  	plan              atc.TaskPlan
    83  	defaultLimits     atc.ContainerLimits
    84  	metadata          StepMetadata
    85  	containerMetadata db.ContainerMetadata
    86  	strategy          worker.ContainerPlacementStrategy
    87  	workerClient      worker.Client
    88  	delegateFactory   TaskDelegateFactory
    89  	lockFactory       lock.LockFactory
    90  }
    91  
    92  func NewTaskStep(
    93  	planID atc.PlanID,
    94  	plan atc.TaskPlan,
    95  	defaultLimits atc.ContainerLimits,
    96  	metadata StepMetadata,
    97  	containerMetadata db.ContainerMetadata,
    98  	strategy worker.ContainerPlacementStrategy,
    99  	workerClient worker.Client,
   100  	delegateFactory TaskDelegateFactory,
   101  	lockFactory lock.LockFactory,
   102  ) Step {
   103  	return &TaskStep{
   104  		planID:            planID,
   105  		plan:              plan,
   106  		defaultLimits:     defaultLimits,
   107  		metadata:          metadata,
   108  		containerMetadata: containerMetadata,
   109  		strategy:          strategy,
   110  		workerClient:      workerClient,
   111  		delegateFactory:   delegateFactory,
   112  		lockFactory:       lockFactory,
   113  	}
   114  }
   115  
   116  // Run will first select the worker based on the TaskConfig's platform and the
   117  // TaskStep's tags, and prioritize it by availability of volumes for the TaskConfig's
   118  // inputs. Inputs that did not have volumes available on the worker will be streamed
   119  // in to the container.
   120  //
   121  // If any inputs are not available in the artifact.Repository, MissingInputsError
   122  // is returned.
   123  //
   124  // Once all the inputs are satisfied, the task's script will be executed. If
   125  // the task is canceled via the context, the script will be interrupted.
   126  //
   127  // If the script exits successfully, the outputs specified in the TaskConfig
   128  // are registered with the artifact.Repository. If no outputs are specified, the
   129  // task's entire working directory is registered as an StreamableArtifactSource under the
   130  // name of the task.
   131  func (step *TaskStep) Run(ctx context.Context, state RunState) (bool, error) {
   132  	delegate := step.delegateFactory.TaskDelegate(state)
   133  	ctx, span := delegate.StartSpan(ctx, "task", tracing.Attrs{
   134  		"name": step.plan.Name,
   135  	})
   136  
   137  	ok, err := step.run(ctx, state, delegate)
   138  	tracing.End(span, err)
   139  
   140  	return ok, err
   141  }
   142  
   143  func (step *TaskStep) run(ctx context.Context, state RunState, delegate TaskDelegate) (bool, error) {
   144  	logger := lagerctx.FromContext(ctx)
   145  	logger = logger.Session("task-step", lager.Data{
   146  		"step-name": step.plan.Name,
   147  		"job-id":    step.metadata.JobID,
   148  	})
   149  
   150  	var taskConfigSource TaskConfigSource
   151  	var taskVars []vars.Variables
   152  
   153  	if step.plan.ConfigPath != "" {
   154  		// external task - construct a source which reads it from file, and apply base resource type defaults.
   155  		taskConfigSource = FileConfigSource{ConfigPath: step.plan.ConfigPath, Client: step.workerClient}
   156  
   157  		// for interpolation - use 'vars' from the pipeline, and then fill remaining with cred variables.
   158  		// this 2-phase strategy allows to interpolate 'vars' by cred variables.
   159  		if len(step.plan.Vars) > 0 {
   160  			taskConfigSource = InterpolateTemplateConfigSource{
   161  				ConfigSource:  taskConfigSource,
   162  				Vars:          []vars.Variables{vars.StaticVariables(step.plan.Vars)},
   163  				ExpectAllKeys: false,
   164  			}
   165  		}
   166  		taskVars = []vars.Variables{state}
   167  	} else {
   168  		// embedded task - first we take it
   169  		taskConfigSource = StaticConfigSource{Config: step.plan.Config}
   170  
   171  		// for interpolation - use just cred variables
   172  		taskVars = []vars.Variables{state}
   173  	}
   174  
   175  	// apply resource type defaults
   176  	taskConfigSource = BaseResourceTypeDefaultsApplySource{
   177  		ConfigSource:  taskConfigSource,
   178  		ResourceTypes: step.plan.VersionedResourceTypes,
   179  	}
   180  
   181  	// override params
   182  	taskConfigSource = &OverrideParamsConfigSource{ConfigSource: taskConfigSource, Params: step.plan.Params}
   183  
   184  	// interpolate template vars
   185  	taskConfigSource = InterpolateTemplateConfigSource{
   186  		ConfigSource:  taskConfigSource,
   187  		Vars:          taskVars,
   188  		ExpectAllKeys: true,
   189  	}
   190  
   191  	// validate
   192  	taskConfigSource = ValidatingConfigSource{ConfigSource: taskConfigSource}
   193  
   194  	repository := state.ArtifactRepository()
   195  
   196  	config, err := taskConfigSource.FetchConfig(ctx, logger, repository)
   197  
   198  	delegate.SetTaskConfig(config)
   199  
   200  	for _, warning := range taskConfigSource.Warnings() {
   201  		fmt.Fprintln(delegate.Stderr(), "[WARNING]", warning)
   202  	}
   203  
   204  	if err != nil {
   205  		return false, err
   206  	}
   207  
   208  	if config.Limits == nil {
   209  		config.Limits = &atc.ContainerLimits{}
   210  	}
   211  	if config.Limits.CPU == nil {
   212  		config.Limits.CPU = step.defaultLimits.CPU
   213  	}
   214  	if config.Limits.Memory == nil {
   215  		config.Limits.Memory = step.defaultLimits.Memory
   216  	}
   217  
   218  	delegate.Initializing(logger)
   219  
   220  	imageSpec, err := step.imageSpec(ctx, state, delegate, config)
   221  	if err != nil {
   222  		return false, err
   223  	}
   224  
   225  	containerSpec, err := step.containerSpec(state, imageSpec, config, step.containerMetadata)
   226  	if err != nil {
   227  		return false, err
   228  	}
   229  	tracing.Inject(ctx, &containerSpec)
   230  
   231  	processSpec := runtime.ProcessSpec{
   232  		Path:         config.Run.Path,
   233  		Args:         config.Run.Args,
   234  		Dir:          config.Run.Dir,
   235  		StdoutWriter: delegate.Stdout(),
   236  		StderrWriter: delegate.Stderr(),
   237  	}
   238  
   239  	owner := db.NewBuildStepContainerOwner(step.metadata.BuildID, step.planID, step.metadata.TeamID)
   240  
   241  	result, err := step.workerClient.RunTaskStep(
   242  		ctx,
   243  		logger,
   244  		owner,
   245  		containerSpec,
   246  		step.workerSpec(config),
   247  		step.strategy,
   248  		step.containerMetadata,
   249  		processSpec,
   250  		delegate,
   251  		step.lockFactory,
   252  	)
   253  
   254  	if err != nil {
   255  		if err == context.Canceled || err == context.DeadlineExceeded {
   256  			step.registerOutputs(logger, repository, config, result.VolumeMounts, step.containerMetadata)
   257  		}
   258  		return false, err
   259  	}
   260  
   261  	delegate.Finished(logger, ExitStatus(result.ExitStatus))
   262  
   263  	step.registerOutputs(logger, repository, config, result.VolumeMounts, step.containerMetadata)
   264  
   265  	// Do not initialize caches for one-off builds
   266  	if step.metadata.JobID != 0 {
   267  		err = step.registerCaches(logger, repository, config, result.VolumeMounts, step.containerMetadata)
   268  		if err != nil {
   269  			return false, err
   270  		}
   271  	}
   272  
   273  	return result.ExitStatus == 0, nil
   274  }
   275  
   276  func (step *TaskStep) imageSpec(ctx context.Context, state RunState, delegate TaskDelegate, config atc.TaskConfig) (worker.ImageSpec, error) {
   277  	imageSpec := worker.ImageSpec{
   278  		Privileged: bool(step.plan.Privileged),
   279  	}
   280  
   281  	// Determine the source of the container image
   282  	// a reference to an artifact (get step, task output) ?
   283  	if step.plan.ImageArtifactName != "" {
   284  		art, found := state.ArtifactRepository().ArtifactFor(build.ArtifactName(step.plan.ImageArtifactName))
   285  		if !found {
   286  			return worker.ImageSpec{}, MissingTaskImageSourceError{step.plan.ImageArtifactName}
   287  		}
   288  
   289  		imageSpec.ImageArtifact = art
   290  
   291  		//an image_resource
   292  	} else if config.ImageResource != nil {
   293  		image := *config.ImageResource
   294  		if len(image.Tags) == 0 {
   295  			image.Tags = step.plan.Tags
   296  		}
   297  
   298  		return delegate.FetchImage(
   299  			ctx,
   300  			image,
   301  			step.plan.VersionedResourceTypes,
   302  			step.plan.Privileged,
   303  		)
   304  
   305  		// a rootfs_uri
   306  	} else if config.RootfsURI != "" {
   307  		imageSpec.ImageURL = config.RootfsURI
   308  	}
   309  
   310  	return imageSpec, nil
   311  }
   312  
   313  func (step *TaskStep) containerInputs(repository *build.Repository, config atc.TaskConfig, metadata db.ContainerMetadata) (map[string]runtime.Artifact, error) {
   314  	inputs := map[string]runtime.Artifact{}
   315  
   316  	var missingRequiredInputs []string
   317  
   318  	for _, input := range config.Inputs {
   319  		inputName := input.Name
   320  		if sourceName, ok := step.plan.InputMapping[inputName]; ok {
   321  			inputName = sourceName
   322  		}
   323  
   324  		art, found := repository.ArtifactFor(build.ArtifactName(inputName))
   325  		if !found {
   326  			if !input.Optional {
   327  				missingRequiredInputs = append(missingRequiredInputs, inputName)
   328  			}
   329  			continue
   330  		}
   331  		ti := taskInput{
   332  			config:        input,
   333  			artifact:      art,
   334  			artifactsRoot: metadata.WorkingDirectory,
   335  		}
   336  
   337  		inputs[ti.Path()] = ti.Artifact()
   338  	}
   339  
   340  	if len(missingRequiredInputs) > 0 {
   341  		return nil, MissingInputsError{missingRequiredInputs}
   342  	}
   343  
   344  	for _, cacheConfig := range config.Caches {
   345  		cacheArt := &runtime.CacheArtifact{
   346  			TeamID:   step.metadata.TeamID,
   347  			JobID:    step.metadata.JobID,
   348  			StepName: step.plan.Name,
   349  			Path:     cacheConfig.Path,
   350  		}
   351  		ti := taskCacheInput{
   352  			artifact:      cacheArt,
   353  			artifactsRoot: metadata.WorkingDirectory,
   354  			cachePath:     cacheConfig.Path,
   355  		}
   356  		inputs[ti.Path()] = ti.Artifact()
   357  	}
   358  
   359  	return inputs, nil
   360  }
   361  
   362  func (step *TaskStep) containerSpec(state RunState, imageSpec worker.ImageSpec, config atc.TaskConfig, metadata db.ContainerMetadata) (worker.ContainerSpec, error) {
   363  	var limits worker.ContainerLimits
   364  	if config.Limits != nil {
   365  		limits.CPU = (*uint64)(config.Limits.CPU)
   366  		limits.Memory = (*uint64)(config.Limits.Memory)
   367  	}
   368  
   369  	containerSpec := worker.ContainerSpec{
   370  		TeamID:    step.metadata.TeamID,
   371  		ImageSpec: imageSpec,
   372  		Limits:    limits,
   373  		User:      config.Run.User,
   374  		Dir:       metadata.WorkingDirectory,
   375  		Env:       config.Params.Env(),
   376  		Type:      metadata.Type,
   377  
   378  		Outputs: worker.OutputPaths{},
   379  	}
   380  
   381  	var err error
   382  	containerSpec.ArtifactByPath, err = step.containerInputs(state.ArtifactRepository(), config, metadata)
   383  	if err != nil {
   384  		return worker.ContainerSpec{}, err
   385  	}
   386  
   387  	for _, output := range config.Outputs {
   388  		path := artifactsPath(output, metadata.WorkingDirectory)
   389  		containerSpec.Outputs[output.Name] = path
   390  	}
   391  
   392  	return containerSpec, nil
   393  }
   394  
   395  func (step *TaskStep) workerSpec(config atc.TaskConfig) worker.WorkerSpec {
   396  	return worker.WorkerSpec{
   397  		Platform: config.Platform,
   398  		Tags:     step.plan.Tags,
   399  		TeamID:   step.metadata.TeamID,
   400  	}
   401  }
   402  
   403  func (step *TaskStep) registerOutputs(logger lager.Logger, repository *build.Repository, config atc.TaskConfig, volumeMounts []worker.VolumeMount, metadata db.ContainerMetadata) {
   404  	logger.Debug("registering-outputs", lager.Data{"outputs": config.Outputs})
   405  
   406  	for _, output := range config.Outputs {
   407  		outputName := output.Name
   408  		if destinationName, ok := step.plan.OutputMapping[output.Name]; ok {
   409  			outputName = destinationName
   410  		}
   411  
   412  		outputPath := artifactsPath(output, metadata.WorkingDirectory)
   413  
   414  		for _, mount := range volumeMounts {
   415  			if filepath.Clean(mount.MountPath) == filepath.Clean(outputPath) {
   416  				art := &runtime.TaskArtifact{
   417  					VolumeHandle: mount.Volume.Handle(),
   418  				}
   419  				repository.RegisterArtifact(build.ArtifactName(outputName), art)
   420  			}
   421  		}
   422  	}
   423  }
   424  
   425  func (step *TaskStep) registerCaches(logger lager.Logger, repository *build.Repository, config atc.TaskConfig, volumeMounts []worker.VolumeMount, metadata db.ContainerMetadata) error {
   426  	logger.Debug("initializing-caches", lager.Data{"caches": config.Caches})
   427  
   428  	for _, cacheConfig := range config.Caches {
   429  		for _, volumeMount := range volumeMounts {
   430  			if volumeMount.MountPath == filepath.Join(metadata.WorkingDirectory, cacheConfig.Path) {
   431  				logger.Debug("initializing-cache", lager.Data{"path": volumeMount.MountPath})
   432  
   433  				err := volumeMount.Volume.InitializeTaskCache(
   434  					logger,
   435  					step.metadata.JobID,
   436  					step.plan.Name,
   437  					cacheConfig.Path,
   438  					bool(step.plan.Privileged))
   439  				if err != nil {
   440  					return err
   441  				}
   442  
   443  				continue
   444  			}
   445  		}
   446  	}
   447  	return nil
   448  }
   449  
   450  type taskInput struct {
   451  	config        atc.TaskInputConfig
   452  	artifact      runtime.Artifact
   453  	artifactsRoot string
   454  }
   455  
   456  func (s taskInput) Artifact() runtime.Artifact { return s.artifact }
   457  
   458  func (s taskInput) Path() string {
   459  	subdir := s.config.Path
   460  	if s.config.Path == "" {
   461  		subdir = s.config.Name
   462  	}
   463  
   464  	return filepath.Join(s.artifactsRoot, subdir)
   465  }
   466  
   467  func artifactsPath(outputConfig atc.TaskOutputConfig, artifactsRoot string) string {
   468  	outputSrc := outputConfig.Path
   469  	if len(outputSrc) == 0 {
   470  		outputSrc = outputConfig.Name
   471  	}
   472  
   473  	return path.Join(artifactsRoot, outputSrc) + "/"
   474  }
   475  
   476  type taskCacheInput struct {
   477  	artifact      runtime.Artifact
   478  	artifactsRoot string
   479  	cachePath     string
   480  }
   481  
   482  func (s taskCacheInput) Artifact() runtime.Artifact { return s.artifact }
   483  
   484  func (s taskCacheInput) Path() string {
   485  	return filepath.Join(s.artifactsRoot, s.cachePath)
   486  }