github.com/pf-qiu/concourse/v6@v6.7.3-0.20201207032516-1f455d73275f/atc/exec/across_step.go (about)

     1  package exec
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  
     7  	"code.cloudfoundry.org/lager"
     8  	"code.cloudfoundry.org/lager/lagerctx"
     9  	"github.com/pf-qiu/concourse/v6/atc"
    10  	"github.com/pf-qiu/concourse/v6/vars"
    11  )
    12  
    13  type ScopedStep struct {
    14  	Step
    15  	Values []interface{}
    16  }
    17  
    18  // AcrossStep is a step of steps to run in parallel. It behaves the same as InParallelStep
    19  // with the exception that an experimental warning is logged to stderr and that step
    20  // lifecycle build events are emitted (Initializing, Starting, and Finished)
    21  type AcrossStep struct {
    22  	vars     []atc.AcrossVar
    23  	steps    []ScopedStep
    24  	failFast bool
    25  
    26  	delegateFactory BuildStepDelegateFactory
    27  	metadata        StepMetadata
    28  }
    29  
    30  // Across constructs an AcrossStep.
    31  func Across(
    32  	vars []atc.AcrossVar,
    33  	steps []ScopedStep,
    34  	failFast bool,
    35  	delegateFactory BuildStepDelegateFactory,
    36  	metadata StepMetadata,
    37  ) AcrossStep {
    38  	return AcrossStep{
    39  		vars:            vars,
    40  		steps:           steps,
    41  		failFast:        failFast,
    42  		delegateFactory: delegateFactory,
    43  		metadata:        metadata,
    44  	}
    45  }
    46  
    47  // Run calls out to InParallelStep.Run after logging a warning to stderr. It also emits
    48  // step lifecycle build events (Initializing, Starting, and Finished).
    49  func (step AcrossStep) Run(ctx context.Context, state RunState) (bool, error) {
    50  	logger := lagerctx.FromContext(ctx)
    51  	logger = logger.Session("across-step", lager.Data{
    52  		"job-id": step.metadata.JobID,
    53  	})
    54  
    55  	delegate := step.delegateFactory.BuildStepDelegate(state)
    56  
    57  	delegate.Initializing(logger)
    58  
    59  	stderr := delegate.Stderr()
    60  
    61  	fmt.Fprintln(stderr, "\x1b[1;33mWARNING: the across step is experimental and subject to change!\x1b[0m")
    62  	fmt.Fprintln(stderr, "")
    63  	fmt.Fprintln(stderr, "\x1b[33mfollow RFC #29 for updates: https://github.com/concourse/rfcs/pull/29\x1b[0m")
    64  	fmt.Fprintln(stderr, "")
    65  
    66  	for _, v := range step.vars {
    67  		_, found, _ := state.Get(vars.Reference{Source: ".", Path: v.Var})
    68  		if found {
    69  			fmt.Fprintf(stderr, "\x1b[1;33mWARNING: across step shadows local var '%s'\x1b[0m\n", v.Var)
    70  		}
    71  	}
    72  
    73  	delegate.Starting(logger)
    74  
    75  	exec := step.acrossStepExecutor(state, 0, step.steps)
    76  	succeeded, err := exec.run(ctx)
    77  	if err != nil {
    78  		return false, err
    79  	}
    80  
    81  	delegate.Finished(logger, succeeded)
    82  
    83  	return succeeded, nil
    84  }
    85  
    86  func (step AcrossStep) acrossStepExecutor(state RunState, varIndex int, steps []ScopedStep) parallelExecutor {
    87  	if varIndex == len(step.vars)-1 {
    88  		return step.acrossStepLeafExecutor(state, steps)
    89  	}
    90  	stepsPerValue := 1
    91  	for _, v := range step.vars[varIndex+1:] {
    92  		stepsPerValue *= len(v.Values)
    93  	}
    94  	numValues := len(step.vars[varIndex].Values)
    95  	return parallelExecutor{
    96  		stepName: "across",
    97  
    98  		maxInFlight: step.vars[varIndex].MaxInFlight,
    99  		failFast:    step.failFast,
   100  		count:       numValues,
   101  
   102  		runFunc: func(ctx context.Context, i int) (bool, error) {
   103  			startIndex := i * stepsPerValue
   104  			endIndex := (i + 1) * stepsPerValue
   105  			substeps := steps[startIndex:endIndex]
   106  			return step.acrossStepExecutor(state, varIndex+1, substeps).run(ctx)
   107  		},
   108  	}
   109  }
   110  
   111  func (step AcrossStep) acrossStepLeafExecutor(state RunState, steps []ScopedStep) parallelExecutor {
   112  	lastVar := step.vars[len(step.vars)-1]
   113  	return parallelExecutor{
   114  		stepName: "across",
   115  
   116  		maxInFlight: lastVar.MaxInFlight,
   117  		failFast:    step.failFast,
   118  		count:       len(steps),
   119  
   120  		runFunc: func(ctx context.Context, i int) (bool, error) {
   121  			scope := state.NewLocalScope()
   122  			for j, v := range step.vars {
   123  				// Don't redact because the `list` operation of a var_source should return identifiers
   124  				// which should be publicly accessible. For static across steps, the static list is
   125  				// embedded directly in the pipeline
   126  				scope.AddLocalVar(v.Var, steps[i].Values[j], false)
   127  			}
   128  
   129  			return steps[i].Run(ctx, scope)
   130  		},
   131  	}
   132  }