github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/exec/task_step.go (about) 1 package exec 2 3 import ( 4 "context" 5 "fmt" 6 "io" 7 "path" 8 "path/filepath" 9 "strings" 10 11 "code.cloudfoundry.org/lager" 12 "code.cloudfoundry.org/lager/lagerctx" 13 "github.com/pf-qiu/concourse/v6/atc" 14 "github.com/pf-qiu/concourse/v6/atc/db" 15 "github.com/pf-qiu/concourse/v6/atc/db/lock" 16 "github.com/pf-qiu/concourse/v6/atc/exec/build" 17 "github.com/pf-qiu/concourse/v6/atc/runtime" 18 "github.com/pf-qiu/concourse/v6/atc/worker" 19 "github.com/pf-qiu/concourse/v6/tracing" 20 "github.com/pf-qiu/concourse/v6/vars" 21 "go.opentelemetry.io/otel/api/trace" 22 ) 23 24 // MissingInputsError is returned when any of the task's required inputs are 25 // missing. 26 type MissingInputsError struct { 27 Inputs []string 28 } 29 30 // Error prints a human-friendly message listing the inputs that were missing. 31 func (err MissingInputsError) Error() string { 32 return fmt.Sprintf("missing inputs: %s", strings.Join(err.Inputs, ", ")) 33 } 34 35 type MissingTaskImageSourceError struct { 36 SourceName string 37 } 38 39 func (err MissingTaskImageSourceError) Error() string { 40 return fmt.Sprintf(`missing image artifact source: %s 41 42 make sure there's a corresponding 'get' step, or a task that produces it as an output`, err.SourceName) 43 } 44 45 type TaskImageSourceParametersError struct { 46 Err error 47 } 48 49 func (err TaskImageSourceParametersError) Error() string { 50 return fmt.Sprintf("failed to evaluate image resource parameters: %s", err.Err) 51 } 52 53 //go:generate counterfeiter . TaskDelegateFactory 54 55 type TaskDelegateFactory interface { 56 TaskDelegate(state RunState) TaskDelegate 57 } 58 59 //go:generate counterfeiter . TaskDelegate 60 61 type TaskDelegate interface { 62 StartSpan(context.Context, string, tracing.Attrs) (context.Context, trace.Span) 63 64 FetchImage(context.Context, atc.ImageResource, atc.VersionedResourceTypes, bool) (worker.ImageSpec, error) 65 66 Stdout() io.Writer 67 Stderr() io.Writer 68 69 SetTaskConfig(config atc.TaskConfig) 70 71 Initializing(lager.Logger) 72 Starting(lager.Logger) 73 Finished(lager.Logger, ExitStatus) 74 SelectedWorker(lager.Logger, string) 75 Errored(lager.Logger, string) 76 } 77 78 // TaskStep executes a TaskConfig, whose inputs will be fetched from the 79 // artifact.Repository and outputs will be added to the artifact.Repository. 80 type TaskStep struct { 81 planID atc.PlanID 82 plan atc.TaskPlan 83 defaultLimits atc.ContainerLimits 84 metadata StepMetadata 85 containerMetadata db.ContainerMetadata 86 strategy worker.ContainerPlacementStrategy 87 workerClient worker.Client 88 delegateFactory TaskDelegateFactory 89 lockFactory lock.LockFactory 90 } 91 92 func NewTaskStep( 93 planID atc.PlanID, 94 plan atc.TaskPlan, 95 defaultLimits atc.ContainerLimits, 96 metadata StepMetadata, 97 containerMetadata db.ContainerMetadata, 98 strategy worker.ContainerPlacementStrategy, 99 workerClient worker.Client, 100 delegateFactory TaskDelegateFactory, 101 lockFactory lock.LockFactory, 102 ) Step { 103 return &TaskStep{ 104 planID: planID, 105 plan: plan, 106 defaultLimits: defaultLimits, 107 metadata: metadata, 108 containerMetadata: containerMetadata, 109 strategy: strategy, 110 workerClient: workerClient, 111 delegateFactory: delegateFactory, 112 lockFactory: lockFactory, 113 } 114 } 115 116 // Run will first select the worker based on the TaskConfig's platform and the 117 // TaskStep's tags, and prioritize it by availability of volumes for the TaskConfig's 118 // inputs. Inputs that did not have volumes available on the worker will be streamed 119 // in to the container. 120 // 121 // If any inputs are not available in the artifact.Repository, MissingInputsError 122 // is returned. 123 // 124 // Once all the inputs are satisfied, the task's script will be executed. If 125 // the task is canceled via the context, the script will be interrupted. 126 // 127 // If the script exits successfully, the outputs specified in the TaskConfig 128 // are registered with the artifact.Repository. If no outputs are specified, the 129 // task's entire working directory is registered as an StreamableArtifactSource under the 130 // name of the task. 131 func (step *TaskStep) Run(ctx context.Context, state RunState) (bool, error) { 132 delegate := step.delegateFactory.TaskDelegate(state) 133 ctx, span := delegate.StartSpan(ctx, "task", tracing.Attrs{ 134 "name": step.plan.Name, 135 }) 136 137 ok, err := step.run(ctx, state, delegate) 138 tracing.End(span, err) 139 140 return ok, err 141 } 142 143 func (step *TaskStep) run(ctx context.Context, state RunState, delegate TaskDelegate) (bool, error) { 144 logger := lagerctx.FromContext(ctx) 145 logger = logger.Session("task-step", lager.Data{ 146 "step-name": step.plan.Name, 147 "job-id": step.metadata.JobID, 148 }) 149 150 var taskConfigSource TaskConfigSource 151 var taskVars []vars.Variables 152 153 if step.plan.ConfigPath != "" { 154 // external task - construct a source which reads it from file, and apply base resource type defaults. 155 taskConfigSource = FileConfigSource{ConfigPath: step.plan.ConfigPath, Client: step.workerClient} 156 157 // for interpolation - use 'vars' from the pipeline, and then fill remaining with cred variables. 158 // this 2-phase strategy allows to interpolate 'vars' by cred variables. 159 if len(step.plan.Vars) > 0 { 160 taskConfigSource = InterpolateTemplateConfigSource{ 161 ConfigSource: taskConfigSource, 162 Vars: []vars.Variables{vars.StaticVariables(step.plan.Vars)}, 163 ExpectAllKeys: false, 164 } 165 } 166 taskVars = []vars.Variables{state} 167 } else { 168 // embedded task - first we take it 169 taskConfigSource = StaticConfigSource{Config: step.plan.Config} 170 171 // for interpolation - use just cred variables 172 taskVars = []vars.Variables{state} 173 } 174 175 // apply resource type defaults 176 taskConfigSource = BaseResourceTypeDefaultsApplySource{ 177 ConfigSource: taskConfigSource, 178 ResourceTypes: step.plan.VersionedResourceTypes, 179 } 180 181 // override params 182 taskConfigSource = &OverrideParamsConfigSource{ConfigSource: taskConfigSource, Params: step.plan.Params} 183 184 // interpolate template vars 185 taskConfigSource = InterpolateTemplateConfigSource{ 186 ConfigSource: taskConfigSource, 187 Vars: taskVars, 188 ExpectAllKeys: true, 189 } 190 191 // validate 192 taskConfigSource = ValidatingConfigSource{ConfigSource: taskConfigSource} 193 194 repository := state.ArtifactRepository() 195 196 config, err := taskConfigSource.FetchConfig(ctx, logger, repository) 197 198 delegate.SetTaskConfig(config) 199 200 for _, warning := range taskConfigSource.Warnings() { 201 fmt.Fprintln(delegate.Stderr(), "[WARNING]", warning) 202 } 203 204 if err != nil { 205 return false, err 206 } 207 208 if config.Limits == nil { 209 config.Limits = &atc.ContainerLimits{} 210 } 211 if config.Limits.CPU == nil { 212 config.Limits.CPU = step.defaultLimits.CPU 213 } 214 if config.Limits.Memory == nil { 215 config.Limits.Memory = step.defaultLimits.Memory 216 } 217 218 delegate.Initializing(logger) 219 220 imageSpec, err := step.imageSpec(ctx, state, delegate, config) 221 if err != nil { 222 return false, err 223 } 224 225 containerSpec, err := step.containerSpec(state, imageSpec, config, step.containerMetadata) 226 if err != nil { 227 return false, err 228 } 229 tracing.Inject(ctx, &containerSpec) 230 231 processSpec := runtime.ProcessSpec{ 232 Path: config.Run.Path, 233 Args: config.Run.Args, 234 Dir: config.Run.Dir, 235 StdoutWriter: delegate.Stdout(), 236 StderrWriter: delegate.Stderr(), 237 } 238 239 owner := db.NewBuildStepContainerOwner(step.metadata.BuildID, step.planID, step.metadata.TeamID) 240 241 result, err := step.workerClient.RunTaskStep( 242 ctx, 243 logger, 244 owner, 245 containerSpec, 246 step.workerSpec(config), 247 step.strategy, 248 step.containerMetadata, 249 processSpec, 250 delegate, 251 step.lockFactory, 252 ) 253 254 if err != nil { 255 if err == context.Canceled || err == context.DeadlineExceeded { 256 step.registerOutputs(logger, repository, config, result.VolumeMounts, step.containerMetadata) 257 } 258 return false, err 259 } 260 261 delegate.Finished(logger, ExitStatus(result.ExitStatus)) 262 263 step.registerOutputs(logger, repository, config, result.VolumeMounts, step.containerMetadata) 264 265 // Do not initialize caches for one-off builds 266 if step.metadata.JobID != 0 { 267 err = step.registerCaches(logger, repository, config, result.VolumeMounts, step.containerMetadata) 268 if err != nil { 269 return false, err 270 } 271 } 272 273 return result.ExitStatus == 0, nil 274 } 275 276 func (step *TaskStep) imageSpec(ctx context.Context, state RunState, delegate TaskDelegate, config atc.TaskConfig) (worker.ImageSpec, error) { 277 imageSpec := worker.ImageSpec{ 278 Privileged: bool(step.plan.Privileged), 279 } 280 281 // Determine the source of the container image 282 // a reference to an artifact (get step, task output) ? 283 if step.plan.ImageArtifactName != "" { 284 art, found := state.ArtifactRepository().ArtifactFor(build.ArtifactName(step.plan.ImageArtifactName)) 285 if !found { 286 return worker.ImageSpec{}, MissingTaskImageSourceError{step.plan.ImageArtifactName} 287 } 288 289 imageSpec.ImageArtifact = art 290 291 //an image_resource 292 } else if config.ImageResource != nil { 293 image := *config.ImageResource 294 if len(image.Tags) == 0 { 295 image.Tags = step.plan.Tags 296 } 297 298 return delegate.FetchImage( 299 ctx, 300 image, 301 step.plan.VersionedResourceTypes, 302 step.plan.Privileged, 303 ) 304 305 // a rootfs_uri 306 } else if config.RootfsURI != "" { 307 imageSpec.ImageURL = config.RootfsURI 308 } 309 310 return imageSpec, nil 311 } 312 313 func (step *TaskStep) containerInputs(repository *build.Repository, config atc.TaskConfig, metadata db.ContainerMetadata) (map[string]runtime.Artifact, error) { 314 inputs := map[string]runtime.Artifact{} 315 316 var missingRequiredInputs []string 317 318 for _, input := range config.Inputs { 319 inputName := input.Name 320 if sourceName, ok := step.plan.InputMapping[inputName]; ok { 321 inputName = sourceName 322 } 323 324 art, found := repository.ArtifactFor(build.ArtifactName(inputName)) 325 if !found { 326 if !input.Optional { 327 missingRequiredInputs = append(missingRequiredInputs, inputName) 328 } 329 continue 330 } 331 ti := taskInput{ 332 config: input, 333 artifact: art, 334 artifactsRoot: metadata.WorkingDirectory, 335 } 336 337 inputs[ti.Path()] = ti.Artifact() 338 } 339 340 if len(missingRequiredInputs) > 0 { 341 return nil, MissingInputsError{missingRequiredInputs} 342 } 343 344 for _, cacheConfig := range config.Caches { 345 cacheArt := &runtime.CacheArtifact{ 346 TeamID: step.metadata.TeamID, 347 JobID: step.metadata.JobID, 348 StepName: step.plan.Name, 349 Path: cacheConfig.Path, 350 } 351 ti := taskCacheInput{ 352 artifact: cacheArt, 353 artifactsRoot: metadata.WorkingDirectory, 354 cachePath: cacheConfig.Path, 355 } 356 inputs[ti.Path()] = ti.Artifact() 357 } 358 359 return inputs, nil 360 } 361 362 func (step *TaskStep) containerSpec(state RunState, imageSpec worker.ImageSpec, config atc.TaskConfig, metadata db.ContainerMetadata) (worker.ContainerSpec, error) { 363 var limits worker.ContainerLimits 364 if config.Limits != nil { 365 limits.CPU = (*uint64)(config.Limits.CPU) 366 limits.Memory = (*uint64)(config.Limits.Memory) 367 } 368 369 containerSpec := worker.ContainerSpec{ 370 TeamID: step.metadata.TeamID, 371 ImageSpec: imageSpec, 372 Limits: limits, 373 User: config.Run.User, 374 Dir: metadata.WorkingDirectory, 375 Env: config.Params.Env(), 376 Type: metadata.Type, 377 378 Outputs: worker.OutputPaths{}, 379 } 380 381 var err error 382 containerSpec.ArtifactByPath, err = step.containerInputs(state.ArtifactRepository(), config, metadata) 383 if err != nil { 384 return worker.ContainerSpec{}, err 385 } 386 387 for _, output := range config.Outputs { 388 path := artifactsPath(output, metadata.WorkingDirectory) 389 containerSpec.Outputs[output.Name] = path 390 } 391 392 return containerSpec, nil 393 } 394 395 func (step *TaskStep) workerSpec(config atc.TaskConfig) worker.WorkerSpec { 396 return worker.WorkerSpec{ 397 Platform: config.Platform, 398 Tags: step.plan.Tags, 399 TeamID: step.metadata.TeamID, 400 } 401 } 402 403 func (step *TaskStep) registerOutputs(logger lager.Logger, repository *build.Repository, config atc.TaskConfig, volumeMounts []worker.VolumeMount, metadata db.ContainerMetadata) { 404 logger.Debug("registering-outputs", lager.Data{"outputs": config.Outputs}) 405 406 for _, output := range config.Outputs { 407 outputName := output.Name 408 if destinationName, ok := step.plan.OutputMapping[output.Name]; ok { 409 outputName = destinationName 410 } 411 412 outputPath := artifactsPath(output, metadata.WorkingDirectory) 413 414 for _, mount := range volumeMounts { 415 if filepath.Clean(mount.MountPath) == filepath.Clean(outputPath) { 416 art := &runtime.TaskArtifact{ 417 VolumeHandle: mount.Volume.Handle(), 418 } 419 repository.RegisterArtifact(build.ArtifactName(outputName), art) 420 } 421 } 422 } 423 } 424 425 func (step *TaskStep) registerCaches(logger lager.Logger, repository *build.Repository, config atc.TaskConfig, volumeMounts []worker.VolumeMount, metadata db.ContainerMetadata) error { 426 logger.Debug("initializing-caches", lager.Data{"caches": config.Caches}) 427 428 for _, cacheConfig := range config.Caches { 429 for _, volumeMount := range volumeMounts { 430 if volumeMount.MountPath == filepath.Join(metadata.WorkingDirectory, cacheConfig.Path) { 431 logger.Debug("initializing-cache", lager.Data{"path": volumeMount.MountPath}) 432 433 err := volumeMount.Volume.InitializeTaskCache( 434 logger, 435 step.metadata.JobID, 436 step.plan.Name, 437 cacheConfig.Path, 438 bool(step.plan.Privileged)) 439 if err != nil { 440 return err 441 } 442 443 continue 444 } 445 } 446 } 447 return nil 448 } 449 450 type taskInput struct { 451 config atc.TaskInputConfig 452 artifact runtime.Artifact 453 artifactsRoot string 454 } 455 456 func (s taskInput) Artifact() runtime.Artifact { return s.artifact } 457 458 func (s taskInput) Path() string { 459 subdir := s.config.Path 460 if s.config.Path == "" { 461 subdir = s.config.Name 462 } 463 464 return filepath.Join(s.artifactsRoot, subdir) 465 } 466 467 func artifactsPath(outputConfig atc.TaskOutputConfig, artifactsRoot string) string { 468 outputSrc := outputConfig.Path 469 if len(outputSrc) == 0 { 470 outputSrc = outputConfig.Name 471 } 472 473 return path.Join(artifactsRoot, outputSrc) + "/" 474 } 475 476 type taskCacheInput struct { 477 artifact runtime.Artifact 478 artifactsRoot string 479 cachePath string 480 } 481 482 func (s taskCacheInput) Artifact() runtime.Artifact { return s.artifact } 483 484 func (s taskCacheInput) Path() string { 485 return filepath.Join(s.artifactsRoot, s.cachePath) 486 }