github.com/jenkins-x/jx/v2@v2.1.155/pkg/cmd/step/create/step_create_task.go (about) 1 package create 2 3 import ( 4 "encoding/json" 5 "fmt" 6 "io/ioutil" 7 "os" 8 "path/filepath" 9 "reflect" 10 "strings" 11 "time" 12 13 "k8s.io/apimachinery/pkg/watch" 14 15 "github.com/jenkins-x/jx/v2/pkg/cmd/opts/step" 16 17 "github.com/jenkins-x/jx/v2/pkg/versionstream" 18 "github.com/spf13/viper" 19 20 "github.com/jenkins-x/jx/v2/pkg/cmd/step/git" 21 22 "github.com/ghodss/yaml" 23 jxclient "github.com/jenkins-x/jx-api/pkg/client/clientset/versioned" 24 "github.com/jenkins-x/jx-logging/pkg/log" 25 "github.com/jenkins-x/jx/v2/pkg/cmd/helper" 26 "github.com/jenkins-x/jx/v2/pkg/cmd/opts" 27 syntaxstep "github.com/jenkins-x/jx/v2/pkg/cmd/step/syntax" 28 "github.com/jenkins-x/jx/v2/pkg/cmd/templates" 29 "github.com/jenkins-x/jx/v2/pkg/config" 30 "github.com/jenkins-x/jx/v2/pkg/gits" 31 "github.com/jenkins-x/jx/v2/pkg/jenkinsfile" 32 "github.com/jenkins-x/jx/v2/pkg/jenkinsfile/gitresolver" 33 "github.com/jenkins-x/jx/v2/pkg/kube" 34 "github.com/jenkins-x/jx/v2/pkg/tekton" 35 "github.com/jenkins-x/jx/v2/pkg/tekton/syntax" 36 "github.com/jenkins-x/jx/v2/pkg/util" 37 "github.com/pkg/errors" 38 "github.com/spf13/cobra" 39 pipelineapi "github.com/tektoncd/pipeline/pkg/apis/pipeline/v1alpha1" 40 tektonclient "github.com/tektoncd/pipeline/pkg/client/clientset/versioned" 41 corev1 "k8s.io/api/core/v1" 42 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" 43 kubeclient "k8s.io/client-go/kubernetes" 44 ) 45 46 const ( 47 kanikoSecretMount = "/kaniko-secret/secret.json" // #nosec 48 kanikoSecretName = kube.SecretKaniko 49 kanikoSecretKey = kube.SecretKaniko 50 51 noApplyOptionName = "no-apply" 52 outputOptionName = "output" 53 ) 54 55 var ( 56 createTaskLong = templates.LongDesc(` 57 Creates a Tekton Pipeline Run for a project 58 `) 59 60 createTaskExample = templates.Examples(` 61 # create a Tekton Pipeline Run and render to the console 62 jx step create task 63 64 # create a Tekton Pipeline Task 65 jx step create task -o mytask.yaml 66 67 # view the steps that would be created 68 jx step create task --view 69 70 `) 71 72 lastPipelineRun = time.Now() 73 74 createTaskOutDir string 75 createTaskNoApply bool 76 ) 77 78 // StepCreateTaskOptions contains the command line flags 79 type StepCreateTaskOptions struct { 80 step.StepOptions 81 82 Pack string 83 BuildPackURL string 84 BuildPackRef string 85 PipelineKind string 86 Context string 87 CustomLabels []string 88 CustomEnvs []string 89 NoApply *bool 90 DryRun bool 91 InterpretMode bool 92 DisableConcurrent bool 93 StartStep string 94 EndStep string 95 Trigger string 96 TargetPath string 97 SourceName string 98 CustomImage string 99 DefaultImage string 100 CloneGitURL string 101 Branch string 102 Revision string 103 PullRequestNumber string 104 DeleteTempDir bool 105 ViewSteps bool 106 EffectivePipeline bool 107 NoReleasePrepare bool 108 Duration time.Duration 109 FromRepo bool 110 NoKaniko bool 111 SemanticRelease bool 112 DisableGitClone bool 113 NoOutput bool 114 KanikoImage string 115 KanikoSecretMount string 116 KanikoSecret string 117 KanikoSecretKey string 118 ProjectID string 119 DockerRegistry string 120 DockerRegistryOrg string 121 KanikoFlags string 122 AdditionalEnvVars map[string]string 123 PodTemplates map[string]*corev1.Pod 124 UseBranchAsRevision bool 125 126 GitInfo *gits.GitRepository 127 BuildNumber string 128 labels map[string]string 129 Results tekton.CRDWrapper 130 pipelineParams []pipelineapi.Param 131 version string 132 previewVersionPrefix string 133 VersionResolver *versionstream.VersionResolver 134 CloneDir string 135 EffectiveProjectConfig *config.ProjectConfig 136 } 137 138 // NewCmdStepCreateTask Creates a new Command object 139 func NewCmdStepCreateTask(commonOpts *opts.CommonOptions) *cobra.Command { 140 cmd, _ := NewCmdStepCreateTaskAndOption(commonOpts) 141 return cmd 142 } 143 144 // NewCmdStepCreateTaskAndOption Creates a new Command object and returns the options 145 func NewCmdStepCreateTaskAndOption(commonOpts *opts.CommonOptions) (*cobra.Command, *StepCreateTaskOptions) { 146 options := &StepCreateTaskOptions{ 147 StepOptions: step.StepOptions{ 148 CommonOptions: commonOpts, 149 }, 150 } 151 152 cmd := &cobra.Command{ 153 Use: "task", 154 Short: "Creates a Tekton PipelineRun for the current folder or given build pack", 155 Long: createTaskLong, 156 Example: createTaskExample, 157 Aliases: []string{"bt"}, 158 Run: func(cmd *cobra.Command, args []string) { 159 options.Cmd = cmd 160 options.Args = args 161 err := options.Run() 162 helper.CheckErr(err) 163 }, 164 } 165 166 cmd.Flags().StringVarP(&createTaskOutDir, outputOptionName, "o", "out", "The directory to write the output to as YAML. Defaults to 'out'") 167 cmd.Flags().StringVarP(&options.Branch, "branch", "", "", "The git branch to trigger the build in. Defaults to the current local branch name") 168 cmd.Flags().StringVarP(&options.Revision, "revision", "", "", "The git revision to checkout, can be a branch name or git sha") 169 cmd.Flags().StringVarP(&options.PipelineKind, "kind", "k", "release", "The kind of pipeline to create such as: "+strings.Join(jenkinsfile.PipelineKinds, ", ")) 170 cmd.Flags().StringArrayVarP(&options.CustomLabels, "label", "l", nil, "List of custom labels to be applied to resources that are created") 171 cmd.Flags().StringArrayVarP(&options.CustomEnvs, "env", "e", nil, "List of custom environment variables to be applied to resources that are created") 172 cmd.Flags().StringVarP(&options.CloneGitURL, "clone-git-url", "", "", "Specify the git URL to clone to a temporary directory to get the source code") 173 cmd.Flags().StringVarP(&options.CloneDir, "clone-dir", "", "", "Specify the directory of the directory containing the git clone") 174 cmd.Flags().StringVarP(&options.PullRequestNumber, "pr-number", "", "", "If a Pull Request this is it's number") 175 cmd.Flags().StringVarP(&options.BuildNumber, "build-number", "", "", "The build number") 176 cmd.Flags().BoolVarP(&createTaskNoApply, noApplyOptionName, "", false, "Disables creating the Pipeline resources in the kubernetes cluster and just outputs the generated Task to the console or output file") 177 cmd.Flags().BoolVarP(&options.DryRun, "dry-run", "", false, "Disables creating the Pipeline resources in the kubernetes cluster and just outputs the generated Task to the console or output file, without side effects") 178 cmd.Flags().BoolVarP(&options.InterpretMode, "interpret", "", false, "Enable interpret mode. Rather than spinning up Tekton CRDs to create a Pod just invoke the commands in the current shell directly. Useful for bootstrapping installations of Jenkins X and tekton using a pipeline before you have installed Tekton.") 179 cmd.Flags().StringVarP(&options.StartStep, "start-step", "", "", "When in interpret mode this specifies the step to start at") 180 cmd.Flags().StringVarP(&options.EndStep, "end-step", "", "", "When in interpret mode this specifies the step to end at") 181 cmd.Flags().BoolVarP(&options.ViewSteps, "view", "", false, "Just view the steps that would be created") 182 cmd.Flags().BoolVarP(&options.EffectivePipeline, "effective-pipeline", "", false, "Just view the effective pipeline definition that would be created") 183 cmd.Flags().BoolVarP(&options.SemanticRelease, "semantic-release", "", false, "Enable semantic releases") 184 cmd.Flags().BoolVarP(&options.UseBranchAsRevision, "branch-as-revision", "", false, "Use the provided branch as the revision for release pipelines, not the version tag") 185 186 options.AddCommonFlags(cmd) 187 options.setupViper(cmd) 188 return cmd, options 189 } 190 191 func (o *StepCreateTaskOptions) setupViper(cmd *cobra.Command) { 192 replacer := strings.NewReplacer("-", "_") 193 viper.SetEnvKeyReplacer(replacer) 194 195 _ = viper.BindEnv(noApplyOptionName) 196 _ = viper.BindPFlag(noApplyOptionName, cmd.Flags().Lookup(noApplyOptionName)) 197 198 _ = viper.BindEnv(outputOptionName) 199 _ = viper.BindPFlag(outputOptionName, cmd.Flags().Lookup(outputOptionName)) 200 } 201 202 // AddCommonFlags adds common CLI options 203 func (o *StepCreateTaskOptions) AddCommonFlags(cmd *cobra.Command) { 204 cmd.Flags().StringVarP(&o.Pack, "pack", "p", "", "The build pack name. If none is specified its discovered from the source code") 205 cmd.Flags().StringVarP(&o.BuildPackURL, "url", "u", "", "The URL for the build pack Git repository") 206 cmd.Flags().StringVarP(&o.BuildPackRef, "ref", "r", "", "The Git reference (branch,tag,sha) in the Git repository to use") 207 cmd.Flags().StringVarP(&o.Context, "context", "c", "", "The pipeline context if there are multiple separate pipelines for a given branch") 208 cmd.Flags().StringVarP(&o.ServiceAccount, "service-account", "", tekton.DefaultPipelineSA, "The Kubernetes ServiceAccount to use to run the pipeline") 209 cmd.Flags().StringVarP(&o.TargetPath, "target-path", "", "", "The target path appended to /workspace/${source} to clone the source code") 210 cmd.Flags().StringVarP(&o.SourceName, "source", "", "source", "The name of the source repository") 211 cmd.Flags().StringVarP(&o.CustomImage, "image", "", "", "Specify a custom image to use for the steps which overrides the image in the PodTemplates") 212 cmd.Flags().StringVarP(&o.DefaultImage, "default-image", "", syntax.DefaultContainerImage, "Specify the docker image to use if there is no image specified for a step and there's no Pod Template") 213 cmd.Flags().BoolVarP(&o.DeleteTempDir, "delete-temp-dir", "", true, "Deletes the temporary directory of cloned files if using the 'clone-git-url' option") 214 cmd.Flags().BoolVarP(&o.NoReleasePrepare, "no-release-prepare", "", false, "Disables creating the release version number and tagging git and triggering the release pipeline from the new tag") 215 cmd.Flags().BoolVarP(&o.NoKaniko, "no-kaniko", "", false, "Disables using kaniko directly for building docker images") 216 cmd.Flags().StringVarP(&o.KanikoImage, "kaniko-image", "", syntax.KanikoDockerImage, "The docker image for Kaniko") 217 cmd.Flags().StringVarP(&o.KanikoSecretMount, "kaniko-secret-mount", "", kanikoSecretMount, "The mount point of the Kaniko secret") 218 cmd.Flags().StringVarP(&o.KanikoSecret, "kaniko-secret", "", kanikoSecretName, "The name of the kaniko secret") 219 cmd.Flags().StringVarP(&o.KanikoSecretKey, "kaniko-secret-key", "", kanikoSecretKey, "The key in the Kaniko Secret to mount") 220 cmd.Flags().StringVarP(&o.ProjectID, "project-id", "", "", "The cloud project ID. If not specified we default to the install project") 221 cmd.Flags().StringVarP(&o.DockerRegistry, "docker-registry", "", "", "The Docker Registry host name to use which is added as a prefix to docker images") 222 cmd.Flags().StringVarP(&o.DockerRegistryOrg, "docker-registry-org", "", "", "The Docker registry organisation. If blank the git repository owner is used") 223 cmd.Flags().StringVarP(&o.KanikoFlags, "kaniko-flags", "", "", "Optional flags to pass to kaniko builds; such as to indicate --insecure docker registry being used") 224 cmd.Flags().DurationVarP(&o.Duration, "duration", "", time.Second*30, "Retry duration when trying to create a PipelineRun") 225 } 226 227 // Run implements this command 228 func (o *StepCreateTaskOptions) Run() error { 229 if o.NoApply == nil { 230 b := viper.GetBool(noApplyOptionName) 231 o.NoApply = &b 232 } 233 234 if o.OutDir == "" { 235 s := viper.GetString(outputOptionName) 236 o.OutDir = s 237 } 238 239 var effectiveProjectConfig *config.ProjectConfig 240 var err error 241 242 tektonClient, jxClient, kubeClient, ns, err := o.getClientsAndNamespace() 243 if err != nil { 244 return err 245 } 246 247 if o.CloneDir == "" { 248 o.CloneDir, err = os.Getwd() 249 if err != nil { 250 return err 251 } 252 } 253 254 if o.VersionResolver == nil { 255 o.VersionResolver, err = o.GetVersionResolver() 256 if err != nil { 257 return errors.Wrap(err, "Unable to create version resolver") 258 } 259 } 260 261 pr, err := o.parsePullRefs() 262 if err != nil { 263 return errors.Wrap(err, "Unable to find or parse PULL_REFS from custom environment") 264 } 265 266 exists, err := o.effectiveProjectConfigExists() 267 if err != nil { 268 return err 269 } 270 if !exists { 271 // TODO this branch all things depending on it can be removed once the meta pipeline is working 272 // TODO keeping this to keep existing behavior until then (HF) 273 if o.CloneGitURL != "" && !o.DisableGitClone { 274 o.CloneDir = o.cloneGitRepositoryToTempDir(o.CloneGitURL, o.Branch, o.PullRequestNumber, o.Revision) 275 if o.DeleteTempDir { 276 defer func() { 277 log.Logger().Infof("removing the temp directory %s", o.CloneDir) 278 err := os.RemoveAll(o.CloneDir) 279 if err != nil { 280 log.Logger().Warnf("failed to delete dir %s: %s", o.CloneDir, err.Error()) 281 } 282 }() 283 } 284 // Add the REPO_URL env var 285 o.CustomEnvs = append(o.CustomEnvs, fmt.Sprintf("%s=%s", "REPO_URL", o.CloneGitURL)) 286 err = o.mergePullRefs(pr, o.CloneDir) 287 if err != nil { 288 return errors.Wrapf(err, "Unable to merge PULL_REFS %s in %s", pr, o.CloneDir) 289 } 290 } 291 } 292 293 o.GitInfo, err = o.FindGitInfo(o.CloneDir) 294 if err != nil { 295 return errors.Wrapf(err, "failed to find git information from dir %s", o.CloneDir) 296 } 297 298 if o.Branch == "" { 299 o.Branch, err = o.Git().Branch(o.CloneDir) 300 if err != nil { 301 return errors.Wrapf(err, "failed to find git branch from dir %s", o.CloneDir) 302 } 303 } 304 305 o.PodTemplates, err = kube.LoadPodTemplates(kubeClient, ns) 306 if err != nil { 307 return errors.Wrap(err, "Unable to load pod templates") 308 } 309 310 pipelineName := tekton.PipelineResourceNameFromGitInfo(o.GitInfo, o.Branch, o.Context, tekton.BuildPipeline.String()) 311 312 if o.KanikoFlags == "" { 313 data, err := kube.GetConfigMapData(kubeClient, kube.ConfigMapJenkinsDockerRegistry, ns) 314 if err == nil { 315 o.KanikoFlags = data["kaniko.flags"] 316 } 317 } 318 exists, err = o.effectiveProjectConfigExists() 319 if err != nil { 320 return err 321 } 322 if exists { 323 effectiveProjectConfig, err = o.loadEffectiveProjectConfig() 324 log.Logger().Debug("loaded effective project configuration from file") 325 } else { 326 // TODO: This branch also goes away when the metapipeline is actually in place in pipelinerunner (AB) 327 log.Logger().Debug("Creating effective project configuration") 328 effectiveProjectConfig, err = o.createEffectiveProjectConfigFromOptions(tektonClient, jxClient, kubeClient, ns, pipelineName) 329 if err != nil { 330 return errors.Wrap(err, "failed to create effective project configuration") 331 } 332 } 333 o.EffectiveProjectConfig = effectiveProjectConfig 334 335 err = o.setBuildValues() 336 if err != nil { 337 return err 338 } 339 340 log.Logger().Debug("Setting build version") 341 err = o.setBuildVersion(effectiveProjectConfig) 342 if err != nil { 343 return errors.Wrapf(err, "failed to set the version on release pipelines") 344 } 345 346 log.Logger().Debug("Creating Tekton CRDs") 347 tektonCRDs, err := o.generateTektonCRDs(effectiveProjectConfig, ns, pipelineName) 348 if err != nil { 349 return errors.Wrap(err, "failed to generate Tekton CRDs") 350 } 351 log.Logger().Debugf("Tekton CRDs for %s created", tektonCRDs.PipelineRun().Name) 352 o.Results = *tektonCRDs 353 354 if o.ViewSteps { 355 err = o.viewSteps(tektonCRDs.Tasks()...) 356 if err != nil { 357 return errors.Wrap(err, "unable to view pipeline steps") 358 } 359 return nil 360 } 361 362 if o.InterpretMode { 363 return o.interpretPipeline(ns, effectiveProjectConfig, tektonCRDs) 364 } 365 366 if *o.NoApply || o.DryRun { 367 if !o.NoOutput { 368 log.Logger().Infof("Writing output ") 369 err := tektonCRDs.WriteToDisk(o.OutDir, nil) 370 if err != nil { 371 return errors.Wrapf(err, "Failed to output Tekton CRDs") 372 } 373 } 374 } else { 375 activityKey := tekton.GeneratePipelineActivity(o.BuildNumber, o.Branch, o.GitInfo, o.Context, pr) 376 377 log.Logger().Debugf(" PipelineActivity for %s created successfully", tektonCRDs.Name()) 378 379 if o.DisableConcurrent { 380 o.waitForPreviousPipeline(tektonClient, ns, 10*time.Minute) 381 } 382 log.Logger().Infof("Applying changes ") 383 err := tekton.ApplyPipeline(jxClient, kubeClient, tektonClient, tektonCRDs, ns, activityKey) 384 if err != nil { 385 return errors.Wrapf(err, "failed to apply Tekton CRDs") 386 } 387 tektonCRDs.AddLabels(o.labels) 388 389 log.Logger().Debugf(" for %s", tektonCRDs.PipelineRun().Name) 390 } 391 return nil 392 } 393 394 func (o *StepCreateTaskOptions) waitForPreviousPipeline(tektonClient tektonclient.Interface, ns string, defaultWait time.Duration) { 395 fallbackWait := true 396 labelSelector := fmt.Sprintf("owner=%s,repository=%s,branch=%s", o.GitInfo.Organisation, o.GitInfo.Name, o.Branch) 397 if o.Context != "" { 398 labelSelector += fmt.Sprintf(",context=%s", o.Context) 399 } 400 401 restartWatch: 402 prs, err := tektonClient.TektonV1alpha1().PipelineRuns(ns).List(metav1.ListOptions{ 403 LabelSelector: labelSelector, 404 }) 405 if err != nil { 406 log.Logger().Errorf("Can't list PipelineRuns %s: %s", labelSelector, err) 407 } else { 408 pendingPipelineRuns := make(map[string]bool) 409 for _, pr := range prs.Items { 410 if !(pr.IsDone() || pr.IsCancelled()) { 411 pendingPipelineRuns[pr.Name] = true 412 } 413 } 414 if len(pendingPipelineRuns) > 0 { 415 log.Logger().Infof("Waiting for pending PipelineRuns %v to finish or be deleted", reflect.ValueOf(pendingPipelineRuns).MapKeys()) 416 pipelineWatch, err := tektonClient.TektonV1alpha1().PipelineRuns(ns).Watch(metav1.ListOptions{ 417 LabelSelector: labelSelector, 418 ResourceVersion: prs.ResourceVersion, 419 }) 420 if err != nil { 421 log.Logger().Errorf("Can't watch PipelineRun %s (ResourceVersion %s): %s", labelSelector, prs.ResourceVersion, err) 422 } else { 423 for { 424 update := <-pipelineWatch.ResultChan() 425 if o.Verbose { 426 bytes, err := json.MarshalIndent(update, "", "\t") 427 log.Logger().Debugf("PipelineRun watch update: %s %s", bytes, err) 428 } 429 switch update.Type { 430 case watch.Deleted: 431 pr := update.Object.(*pipelineapi.PipelineRun) 432 if pendingPipelineRuns[pr.Name] { 433 log.Logger().Infof("PipelineRun %s is deleted", pr.Name) 434 delete(pendingPipelineRuns, pr.Name) 435 } 436 case watch.Modified: 437 pr := update.Object.(*pipelineapi.PipelineRun) 438 if pendingPipelineRuns[pr.Name] && (pr.IsDone() || pr.IsCancelled()) { 439 log.Logger().Infof("PipelineRun %s is finished", pr.Name) 440 delete(pendingPipelineRuns, pr.Name) 441 } 442 case watch.Added: 443 pr := update.Object.(*pipelineapi.PipelineRun) 444 if !(pr.IsDone() || pr.IsCancelled()) { 445 log.Logger().Infof("PipelineRun %s is added", pr.Name) 446 pendingPipelineRuns[pr.Name] = true 447 } 448 default: 449 log.Logger().Errorf("Unknown PipelineRun watch update. Restarting watch.") 450 pipelineWatch.Stop() 451 goto restartWatch 452 } 453 if len(pendingPipelineRuns) == 0 { 454 pipelineWatch.Stop() 455 fallbackWait = false 456 break 457 } 458 } 459 } 460 } else { 461 fallbackWait = false 462 } 463 } 464 465 // When failing to wait for a pipeline wait for defaultWait. 466 if fallbackWait { 467 sleepDuration := defaultWait - time.Now().Sub(lastPipelineRun) 468 if sleepDuration > 0 { 469 log.Logger().Errorf("Can't access previous PipelineRun. Waiting %v to ensure it finishes", sleepDuration) 470 time.Sleep(sleepDuration) 471 } 472 lastPipelineRun = time.Now() 473 } 474 } 475 476 func (o *StepCreateTaskOptions) createEffectiveProjectConfigFromOptions(tektonClient tektonclient.Interface, jxClient jxclient.Interface, kubeClient kubeclient.Interface, ns string, pipelineName string) (*config.ProjectConfig, error) { 477 if o.InterpretMode { 478 // lets allow this command to run in an empty cluster 479 o.RemoteCluster = true 480 } 481 settings, err := o.TeamSettings() 482 if err != nil { 483 return nil, err 484 } 485 486 if o.ProjectID == "" { 487 if !o.RemoteCluster { 488 data, err := kube.ReadInstallValues(kubeClient, ns) 489 if err != nil { 490 return nil, errors.Wrapf(err, "failed to read install values from namespace %s", ns) 491 } 492 o.ProjectID = data["projectID"] 493 } 494 if o.ProjectID == "" { 495 o.ProjectID = "todo" 496 } 497 } 498 if o.DefaultImage == "" { 499 o.DefaultImage = syntax.DefaultContainerImage 500 } 501 502 if o.KanikoImage == "" { 503 o.KanikoImage = syntax.KanikoDockerImage 504 } 505 o.KanikoImage, err = o.VersionResolver.ResolveDockerImage(o.KanikoImage) 506 if err != nil { 507 return nil, err 508 } 509 if o.KanikoSecretMount == "" { 510 o.KanikoSecretMount = kanikoSecretMount 511 } 512 513 if o.DockerRegistry == "" && !o.InterpretMode { 514 data, err := kube.GetConfigMapData(kubeClient, kube.ConfigMapJenkinsDockerRegistry, ns) 515 if err != nil { 516 return nil, fmt.Errorf("could not find ConfigMap %s in namespace %s: %s", kube.ConfigMapJenkinsDockerRegistry, ns, err) 517 } 518 o.DockerRegistry = data["docker.registry"] 519 if o.DockerRegistry == "" { 520 return nil, util.MissingOption("docker-registry") 521 } 522 } 523 524 if o.BuildNumber == "" { 525 if *o.NoApply || o.DryRun || o.InterpretMode { 526 o.BuildNumber = "1" 527 } else { 528 log.Logger().Debugf("generating build number...") 529 o.BuildNumber, err = tekton.GenerateNextBuildNumber(tektonClient, jxClient, ns, o.GitInfo, o.Branch, o.Duration, o.Context, false) 530 if err != nil { 531 return nil, err 532 } 533 log.Logger().Debugf("generated build number %s for %s", o.BuildNumber, o.CloneGitURL) 534 } 535 } 536 projectConfig, projectConfigFile, err := o.loadProjectConfig() 537 if err != nil { 538 return nil, errors.Wrapf(err, "failed to load project config in dir %s", o.CloneDir) 539 } 540 if o.BuildPackURL == "" || o.BuildPackRef == "" { 541 if projectConfig.BuildPackGitURL != "" { 542 o.BuildPackURL = projectConfig.BuildPackGitURL 543 } else if o.BuildPackURL == "" { 544 o.BuildPackURL = settings.BuildPackURL 545 } 546 if projectConfig.BuildPackGitURef != "" { 547 o.BuildPackRef = projectConfig.BuildPackGitURef 548 } else if o.BuildPackRef == "" { 549 o.BuildPackRef = settings.BuildPackRef 550 } 551 } 552 if o.BuildPackURL == "" { 553 return nil, util.MissingOption("url") 554 } 555 if o.BuildPackRef == "" { 556 return nil, util.MissingOption("ref") 557 } 558 if o.PipelineKind == "" { 559 return nil, util.MissingOption("kind") 560 } 561 562 if o.Pack == "" { 563 o.Pack = projectConfig.BuildPack 564 } 565 if o.Pack == "" { 566 o.Pack, err = o.DiscoverBuildPack(o.CloneDir, projectConfig, o.Pack) 567 if err != nil { 568 return nil, errors.Wrapf(err, "failed to discover the build pack") 569 } 570 } 571 572 if o.Pack == "" { 573 return nil, util.MissingOption("pack") 574 } 575 576 packsDir, err := gitresolver.InitBuildPack(o.Git(), o.BuildPackURL, o.BuildPackRef) 577 if err != nil { 578 return nil, err 579 } 580 581 resolver, err := gitresolver.CreateResolver(packsDir, o.Git()) 582 if err != nil { 583 return nil, err 584 } 585 586 log.Logger().Debug("creating effective project configuration") 587 effectiveProjectConfig, err := o.createEffectiveProjectConfig(packsDir, projectConfig, projectConfigFile, resolver, ns) 588 return effectiveProjectConfig, err 589 } 590 591 // createEffectiveProjectConfig creates the effective parsed pipeline which is then used to generate the Tekton CRDs. 592 func (o *StepCreateTaskOptions) createEffectiveProjectConfig(packsDir string, projectConfig *config.ProjectConfig, projectConfigFile string, resolver jenkinsfile.ImportFileResolver, ns string) (*config.ProjectConfig, error) { 593 createEffective := &syntaxstep.StepSyntaxEffectiveOptions{ 594 Pack: o.Pack, 595 BuildPackURL: o.BuildPackURL, 596 BuildPackRef: o.BuildPackRef, 597 Context: o.Context, 598 CustomImage: o.CustomImage, 599 DefaultImage: o.DefaultImage, 600 UseKaniko: !o.NoKaniko, 601 KanikoImage: o.KanikoImage, 602 ProjectID: o.ProjectID, 603 DockerRegistry: o.DockerRegistry, 604 DockerRegistryOrg: o.DockerRegistryOrg, 605 SourceName: o.SourceName, 606 CustomEnvs: o.CustomEnvs, 607 GitInfo: o.GitInfo, 608 PodTemplates: o.PodTemplates, 609 VersionResolver: o.VersionResolver, 610 ValidateInCluster: !o.InterpretMode, 611 } 612 commonCopy := *o.CommonOptions 613 createEffective.CommonOptions = &commonCopy 614 615 effectiveProjectConfig, err := createEffective.CreateEffectivePipeline(packsDir, projectConfig, projectConfigFile, resolver) 616 if err != nil { 617 return nil, errors.Wrapf(err, "effective pipeline creation failed") 618 } 619 // lets allow a `jenkins-x.yml` to specify we want to disable release prepare mode which can be useful for 620 // working with custom jenkins pipelines in custom jenkins servers 621 if projectConfig.NoReleasePrepare { 622 o.NoReleasePrepare = true 623 } 624 625 parsed, err := effectiveProjectConfig.GetPipeline(o.PipelineKind) 626 if err != nil { 627 return nil, err 628 } 629 630 if o.EffectivePipeline { 631 log.Logger().Info("Successfully generated effective pipeline:") 632 effective := &jenkinsfile.PipelineLifecycles{ 633 Pipeline: parsed, 634 } 635 effectiveYaml, _ := yaml.Marshal(effective) 636 log.Logger().Infof("%s", effectiveYaml) 637 return nil, nil 638 } 639 return effectiveProjectConfig, nil 640 } 641 642 // GenerateTektonCRDs creates the Pipeline, Task, PipelineResource, PipelineRun, and PipelineStructure CRDs that will be applied to actually kick off the pipeline 643 func (o *StepCreateTaskOptions) generateTektonCRDs(effectiveProjectConfig *config.ProjectConfig, ns string, pipelineName string) (*tekton.CRDWrapper, error) { 644 if effectiveProjectConfig == nil { 645 return nil, errors.New("effective project config cannot be nil") 646 } 647 648 effectivePipeline, err := effectiveProjectConfig.GetPipeline(o.PipelineKind) 649 if err != nil { 650 return nil, errors.Wrapf(err, "unable to extract the requested pipeline") 651 } 652 653 crdParams := syntax.CRDsFromPipelineParams{ 654 PipelineIdentifier: pipelineName, 655 BuildIdentifier: o.BuildNumber, 656 Namespace: ns, 657 PodTemplates: o.PodTemplates, 658 VersionsDir: o.VersionResolver.VersionsDir, 659 TaskParams: o.getDefaultTaskInputs().Params, 660 SourceDir: o.SourceName, 661 Labels: o.labels, 662 DefaultImage: "", 663 InterpretMode: o.InterpretMode, 664 } 665 666 pipeline, tasks, structure, err := effectivePipeline.GenerateCRDs(crdParams) 667 if err != nil { 668 return nil, errors.Wrapf(err, "generation failed for Pipeline") 669 } 670 671 tasks, pipeline = o.enhanceTasksAndPipeline(tasks, pipeline, effectiveProjectConfig.PipelineConfig.Env) 672 resources := []*pipelineapi.PipelineResource{tekton.GenerateSourceRepoResource(pipelineName, o.GitInfo, o.Revision)} 673 674 var timeout *metav1.Duration 675 if effectivePipeline.Options != nil && effectivePipeline.Options.Timeout != nil { 676 timeout, err = effectivePipeline.Options.Timeout.ToDuration() 677 if err != nil { 678 return nil, errors.Wrapf(err, "parsing of pipeline timeout failed") 679 } 680 } 681 prLabels := util.MergeMaps(o.labels, effectivePipeline.GetPodLabels()) 682 run := tekton.CreatePipelineRun(resources, pipeline.Name, pipeline.APIVersion, prLabels, o.ServiceAccount, o.pipelineParams, timeout, effectivePipeline.GetPossibleAffinityPolicy(pipeline.Name), effectivePipeline.GetTolerations()) 683 684 tektonCRDs, err := tekton.NewCRDWrapper(pipeline, tasks, resources, structure, run) 685 if err != nil { 686 return nil, err 687 } 688 689 return tektonCRDs, nil 690 } 691 692 func (o *StepCreateTaskOptions) loadProjectConfig() (*config.ProjectConfig, string, error) { 693 if o.Context != "" { 694 fileName := filepath.Join(o.CloneDir, fmt.Sprintf("jenkins-x-%s.yml", o.Context)) 695 exists, err := util.FileExists(fileName) 696 if err != nil { 697 return nil, fileName, errors.Wrapf(err, "failed to check if file exists %s", fileName) 698 } 699 if exists { 700 config, err := config.LoadProjectConfigFile(fileName) 701 return config, fileName, err 702 } 703 } 704 return config.LoadProjectConfig(o.CloneDir) 705 } 706 707 func (o *StepCreateTaskOptions) effectiveProjectConfigExists() (bool, error) { 708 fileName := o.CloneDir 709 710 if o.Context == "" { 711 fileName = filepath.Join(fileName, "jenkins-x-effective.yml") 712 } else { 713 fileName = filepath.Join(fileName, fmt.Sprintf("jenkins-x-%s-effective.yml", o.Context)) 714 } 715 716 exists, err := util.FileExists(fileName) 717 if err != nil { 718 return false, errors.Wrapf(err, "failed to check existence of %s", fileName) 719 } 720 return exists, nil 721 } 722 723 func (o *StepCreateTaskOptions) loadEffectiveProjectConfig() (*config.ProjectConfig, error) { 724 fileName := o.CloneDir 725 726 if o.Context == "" { 727 fileName = filepath.Join(fileName, "jenkins-x-effective.yml") 728 } else { 729 fileName = filepath.Join(fileName, fmt.Sprintf("jenkins-x-%s-effective.yml", o.Context)) 730 } 731 732 projectConfig, err := config.LoadProjectConfigFile(fileName) 733 return projectConfig, err 734 } 735 736 // getDefaultTaskInputs gets the base, built-in task parameters as an Input. 737 func (o *StepCreateTaskOptions) getDefaultTaskInputs() *pipelineapi.Inputs { 738 inputs := &pipelineapi.Inputs{} 739 taskParams := o.createTaskParams() 740 if len(taskParams) > 0 { 741 inputs.Params = taskParams 742 } 743 return inputs 744 } 745 746 func (o *StepCreateTaskOptions) enhanceTaskWithVolumesEnvAndInputs(task *pipelineapi.Task, env []corev1.EnvVar, inputs pipelineapi.Inputs) { 747 volumes := task.Spec.Volumes 748 for i, step := range task.Spec.Steps { 749 volumes = o.modifyVolumes(&step.Container, volumes) 750 o.modifyEnvVars(&step.Container, env) 751 task.Spec.Steps[i] = step 752 } 753 754 task.Spec.Volumes = volumes 755 if task.Spec.Inputs == nil { 756 task.Spec.Inputs = &inputs 757 } else { 758 task.Spec.Inputs.Params = inputs.Params 759 } 760 } 761 762 // enhanceTasksAndPipeline takes a slice of Tasks and a Pipeline and modifies them to include built-in volumes, environment variables, and parameters 763 func (o *StepCreateTaskOptions) enhanceTasksAndPipeline(tasks []*pipelineapi.Task, pipeline *pipelineapi.Pipeline, env []corev1.EnvVar) ([]*pipelineapi.Task, *pipelineapi.Pipeline) { 764 taskInputs := o.getDefaultTaskInputs() 765 766 for _, t := range tasks { 767 o.enhanceTaskWithVolumesEnvAndInputs(t, env, *taskInputs) 768 } 769 770 taskParams := o.createPipelineTaskParams() 771 772 for i, pt := range pipeline.Spec.Tasks { 773 for _, tp := range taskParams { 774 if !hasPipelineParam(pt.Params, tp.Name) { 775 pt.Params = append(pt.Params, tp) 776 pipeline.Spec.Tasks[i] = pt 777 } 778 } 779 } 780 781 pipeline.Spec.Params = o.createPipelineParams() 782 783 if pipeline.APIVersion == "" { 784 pipeline.APIVersion = syntax.TektonAPIVersion 785 } 786 if pipeline.Kind == "" { 787 pipeline.Kind = "Pipeline" 788 } 789 790 return tasks, pipeline 791 } 792 793 func (o *StepCreateTaskOptions) createTaskParams() []pipelineapi.ParamSpec { 794 taskParams := []pipelineapi.ParamSpec{} 795 for _, param := range o.pipelineParams { 796 name := param.Name 797 description := "" 798 defaultValue := "" 799 switch name { 800 case "version": 801 description = "the version number for this pipeline which is used as a tag on docker images and helm charts" 802 defaultValue = o.version 803 case "build_id": 804 description = "the PipelineRun build number" 805 defaultValue = o.BuildNumber 806 } 807 stringParamDefaultValue := syntax.StringParamValue(defaultValue) 808 taskParams = append(taskParams, pipelineapi.ParamSpec{ 809 Name: name, 810 Description: description, 811 Default: &stringParamDefaultValue, 812 Type: pipelineapi.ParamTypeString, 813 }) 814 } 815 return taskParams 816 } 817 818 func (o *StepCreateTaskOptions) createPipelineParams() []pipelineapi.ParamSpec { 819 answer := []pipelineapi.ParamSpec{} 820 taskParams := o.createTaskParams() 821 for _, tp := range taskParams { 822 answer = append(answer, pipelineapi.ParamSpec{ 823 Name: tp.Name, 824 Description: tp.Description, 825 Default: tp.Default, 826 Type: tp.Type, 827 }) 828 } 829 return answer 830 } 831 832 func (o *StepCreateTaskOptions) createPipelineTaskParams() []pipelineapi.Param { 833 ptp := []pipelineapi.Param{} 834 for _, p := range o.pipelineParams { 835 ptp = append(ptp, pipelineapi.Param{ 836 Name: p.Name, 837 Value: syntax.StringParamValue(fmt.Sprintf("$(params.%s)", p.Name)), 838 }) 839 } 840 return ptp 841 } 842 843 func (o *StepCreateTaskOptions) setBuildValues() error { 844 labels := map[string]string{} 845 if o.GitInfo != nil { 846 labels[tekton.LabelOwner] = o.GitInfo.Organisation 847 labels[tekton.LabelRepo] = o.GitInfo.Name 848 } 849 labels[tekton.LabelBranch] = o.Branch 850 if o.Context != "" { 851 labels[tekton.LabelContext] = o.Context 852 } 853 labels[tekton.LabelBuild] = o.BuildNumber 854 labels[tekton.LabelType] = tekton.BuildPipeline.String() 855 return o.combineLabels(labels) 856 } 857 858 func (o *StepCreateTaskOptions) combineLabels(labels map[string]string) error { 859 // add any custom labels 860 customLabels, err := util.ExtractKeyValuePairs(o.CustomLabels, "=") 861 if err != nil { 862 return err 863 } 864 o.labels = util.MergeMaps(labels, customLabels) 865 return nil 866 } 867 868 func (o *StepCreateTaskOptions) getWorkspaceDir() string { 869 return filepath.Join("/workspace", o.SourceName) 870 } 871 872 func (o *StepCreateTaskOptions) modifyEnvVars(container *corev1.Container, globalEnv []corev1.EnvVar) { 873 envVars := []corev1.EnvVar{} 874 for _, e := range container.Env { 875 name := e.Name 876 if name != "JENKINS_URL" { 877 envVars = append(envVars, e) 878 } 879 } 880 if kube.GetSliceEnvVar(envVars, "DOCKER_REGISTRY") == nil { 881 envVars = append(envVars, corev1.EnvVar{ 882 Name: "DOCKER_REGISTRY", 883 Value: o.DockerRegistry, 884 }) 885 } 886 if kube.GetSliceEnvVar(envVars, "DOCKER_REGISTRY_ORG") == nil && o.DockerRegistryOrg != "" { 887 envVars = append(envVars, corev1.EnvVar{ 888 Name: "DOCKER_REGISTRY_ORG", 889 Value: o.DockerRegistryOrg, 890 }) 891 } 892 if kube.GetSliceEnvVar(envVars, "KANIKO_FLAGS") == nil && o.KanikoFlags != "" { 893 envVars = append(envVars, corev1.EnvVar{ 894 Name: "KANIKO_FLAGS", 895 Value: o.KanikoFlags, 896 }) 897 } 898 if kube.GetSliceEnvVar(envVars, "BUILD_NUMBER") == nil { 899 envVars = append(envVars, corev1.EnvVar{ 900 Name: "BUILD_NUMBER", 901 Value: o.BuildNumber, 902 }) 903 } 904 if o.PipelineKind != "" && kube.GetSliceEnvVar(envVars, "PIPELINE_KIND") == nil { 905 envVars = append(envVars, corev1.EnvVar{ 906 Name: "PIPELINE_KIND", 907 Value: o.PipelineKind, 908 }) 909 } 910 if o.Context != "" && kube.GetSliceEnvVar(envVars, "PIPELINE_CONTEXT") == nil { 911 envVars = append(envVars, corev1.EnvVar{ 912 Name: "PIPELINE_CONTEXT", 913 Value: o.Context, 914 }) 915 } 916 gitUserName := util.DefaultGitUserName 917 gitUserEmail := util.DefaultGitUserEmail 918 919 settings, err := o.TeamSettings() 920 // If there's an error getting the team settings, just ignore it and keep using the defaults. 921 if err == nil { 922 if settings.PipelineUsername != "" { 923 gitUserName = settings.PipelineUsername 924 } 925 if settings.PipelineUserEmail != "" { 926 gitUserEmail = settings.PipelineUserEmail 927 } 928 } 929 930 if kube.GetSliceEnvVar(envVars, "GIT_AUTHOR_NAME") == nil { 931 envVars = append(envVars, corev1.EnvVar{ 932 Name: "GIT_AUTHOR_NAME", 933 Value: gitUserName, 934 }) 935 } 936 if kube.GetSliceEnvVar(envVars, "GIT_AUTHOR_EMAIL") == nil { 937 envVars = append(envVars, corev1.EnvVar{ 938 Name: "GIT_AUTHOR_EMAIL", 939 Value: gitUserEmail, 940 }) 941 } 942 if kube.GetSliceEnvVar(envVars, "GIT_COMMITTER_NAME") == nil { 943 envVars = append(envVars, corev1.EnvVar{ 944 Name: "GIT_COMMITTER_NAME", 945 Value: gitUserName, 946 }) 947 } 948 if kube.GetSliceEnvVar(envVars, "GIT_COMMITTER_EMAIL") == nil { 949 envVars = append(envVars, corev1.EnvVar{ 950 Name: "GIT_COMMITTER_EMAIL", 951 Value: gitUserEmail, 952 }) 953 } 954 955 gitInfo := o.GitInfo 956 branch := o.Branch 957 if gitInfo != nil { 958 u := gitInfo.CloneURL 959 if u != "" && kube.GetSliceEnvVar(envVars, "SOURCE_URL") == nil { 960 envVars = append(envVars, corev1.EnvVar{ 961 Name: "SOURCE_URL", 962 Value: u, 963 }) 964 } 965 repo := gitInfo.Name 966 owner := gitInfo.Organisation 967 if owner != "" && kube.GetSliceEnvVar(envVars, "REPO_OWNER") == nil { 968 envVars = append(envVars, corev1.EnvVar{ 969 Name: "REPO_OWNER", 970 Value: owner, 971 }) 972 } 973 if repo != "" && kube.GetSliceEnvVar(envVars, "REPO_NAME") == nil { 974 envVars = append(envVars, corev1.EnvVar{ 975 Name: "REPO_NAME", 976 Value: repo, 977 }) 978 } 979 if owner != "" && repo != "" && branch != "" { 980 jobName := fmt.Sprintf("%s/%s/%s", owner, repo, branch) 981 if kube.GetSliceEnvVar(envVars, "JOB_NAME") == nil { 982 envVars = append(envVars, corev1.EnvVar{ 983 Name: "JOB_NAME", 984 Value: jobName, 985 }) 986 } 987 } 988 989 // lets keep the APP_NAME environment variable we need for previews 990 if repo != "" && kube.GetSliceEnvVar(envVars, "APP_NAME") == nil { 991 envVars = append(envVars, corev1.EnvVar{ 992 Name: "APP_NAME", 993 Value: repo, 994 }) 995 } 996 } 997 if branch != "" { 998 if kube.GetSliceEnvVar(envVars, util.EnvVarBranchName) == nil { 999 envVars = append(envVars, corev1.EnvVar{ 1000 Name: util.EnvVarBranchName, 1001 Value: branch, 1002 }) 1003 } 1004 } 1005 if o.InterpretMode { 1006 if kube.GetSliceEnvVar(envVars, "JX_INTERPRET_PIPELINE") == nil { 1007 envVars = append(envVars, corev1.EnvVar{ 1008 Name: "JX_INTERPRET_PIPELINE", 1009 Value: "true", 1010 }) 1011 } 1012 } else { 1013 if kube.GetSliceEnvVar(envVars, "JX_BATCH_MODE") == nil { 1014 envVars = append(envVars, corev1.EnvVar{ 1015 Name: "JX_BATCH_MODE", 1016 Value: "true", 1017 }) 1018 } 1019 } 1020 1021 for _, param := range o.pipelineParams { 1022 name := strings.ToUpper(param.Name) 1023 if kube.GetSliceEnvVar(envVars, name) == nil { 1024 envVars = append(envVars, corev1.EnvVar{ 1025 Name: name, 1026 Value: "$(inputs.params." + param.Name + ")", 1027 }) 1028 } 1029 } 1030 1031 for _, e := range globalEnv { 1032 if kube.GetSliceEnvVar(envVars, e.Name) == nil && e.ValueFrom != nil { 1033 envVars = append(envVars, e) 1034 } 1035 } 1036 1037 for i := range envVars { 1038 if envVars[i].Name == "XDG_CONFIG_HOME" { 1039 envVars[i].Value = "/workspace/xdg_config" 1040 } 1041 } 1042 1043 if isKanikoExecutorStep(container) && !o.NoKaniko && kube.GetSliceEnvVar(envVars, "NO_GOOGLE_APPLICATION_CREDENTIALS") == nil { 1044 if kube.GetSliceEnvVar(envVars, "GOOGLE_APPLICATION_CREDENTIALS") == nil { 1045 envVars = append(envVars, corev1.EnvVar{ 1046 Name: "GOOGLE_APPLICATION_CREDENTIALS", 1047 Value: o.KanikoSecretMount, 1048 }) 1049 } 1050 } 1051 if kube.GetSliceEnvVar(envVars, "PREVIEW_VERSION") == nil && kube.GetSliceEnvVar(envVars, "VERSION") != nil { 1052 envVars = append(envVars, corev1.EnvVar{ 1053 Name: "PREVIEW_VERSION", 1054 Value: "$(inputs.params.version)", 1055 }) 1056 } 1057 for k, v := range o.AdditionalEnvVars { 1058 if kube.GetSliceEnvVar(envVars, k) == nil { 1059 envVars = append(envVars, corev1.EnvVar{ 1060 Name: k, 1061 Value: v, 1062 }) 1063 } 1064 } 1065 container.Env = envVars 1066 } 1067 1068 func (o *StepCreateTaskOptions) modifyVolumes(container *corev1.Container, volumes []corev1.Volume) []corev1.Volume { 1069 answer := volumes 1070 1071 if isKanikoExecutorStep(container) && !o.NoKaniko { 1072 kubeClient, ns, err := o.KubeClientAndDevNamespace() 1073 if err != nil { 1074 log.Logger().Warnf("failed to find kaniko secret: %s", err) 1075 } else { 1076 if o.KanikoSecret == "" { 1077 o.KanikoSecret = kanikoSecretName 1078 } 1079 if o.KanikoSecretKey == "" { 1080 o.KanikoSecretKey = kanikoSecretKey 1081 } 1082 secretName := o.KanikoSecret 1083 key := o.KanikoSecretKey 1084 secret, err := kubeClient.CoreV1().Secrets(ns).Get(secretName, metav1.GetOptions{}) 1085 if err != nil { 1086 log.Logger().Warnf("failed to find secret %s in namespace %s: %s", secretName, ns, err) 1087 } else if secret != nil && secret.Data != nil && secret.Data[key] != nil { 1088 // lets mount the kaniko secret 1089 volumeName := "kaniko-secret" 1090 _, fileName := filepath.Split(o.KanikoSecretMount) 1091 1092 volume := corev1.Volume{ 1093 Name: volumeName, 1094 VolumeSource: corev1.VolumeSource{ 1095 Secret: &corev1.SecretVolumeSource{ 1096 SecretName: secretName, 1097 Items: []corev1.KeyToPath{ 1098 { 1099 Key: key, 1100 Path: fileName, 1101 }, 1102 }, 1103 }, 1104 }, 1105 } 1106 if !kube.ContainsVolume(answer, volume) { 1107 answer = append(answer, volume) 1108 } 1109 1110 mountDir, _ := filepath.Split(o.KanikoSecretMount) 1111 mountDir = strings.TrimSuffix(mountDir, "/") 1112 volumeMount := corev1.VolumeMount{ 1113 Name: volumeName, 1114 MountPath: mountDir, 1115 ReadOnly: true, 1116 } 1117 if !kube.ContainsVolumeMount(container.VolumeMounts, volumeMount) { 1118 container.VolumeMounts = append(container.VolumeMounts, volumeMount) 1119 } 1120 } 1121 } 1122 } 1123 1124 podInfoName := "podinfo" 1125 volume := corev1.Volume{ 1126 Name: podInfoName, 1127 VolumeSource: corev1.VolumeSource{ 1128 DownwardAPI: &corev1.DownwardAPIVolumeSource{ 1129 Items: []corev1.DownwardAPIVolumeFile{ 1130 { 1131 Path: "labels", 1132 FieldRef: &corev1.ObjectFieldSelector{ 1133 FieldPath: "metadata.labels", 1134 }, 1135 }, 1136 }, 1137 }, 1138 }, 1139 } 1140 if !kube.ContainsVolume(volumes, volume) { 1141 answer = append(answer, volume) 1142 } 1143 volumeMount := corev1.VolumeMount{ 1144 Name: podInfoName, 1145 MountPath: "/etc/podinfo", 1146 ReadOnly: true, 1147 } 1148 if !kube.ContainsVolumeMount(container.VolumeMounts, volumeMount) { 1149 container.VolumeMounts = append(container.VolumeMounts, volumeMount) 1150 } 1151 return answer 1152 } 1153 1154 func (o *StepCreateTaskOptions) cloneGitRepositoryToTempDir(gitURL string, branch string, pullRequestNumber string, revision string) string { 1155 var tmpDir string 1156 err := o.Retry(3, time.Second*2, func() error { 1157 var err error 1158 tmpDir, err = ioutil.TempDir("", "git") 1159 if err != nil { 1160 return err 1161 } 1162 log.Logger().Infof("shallow cloning repository %s to temp dir %s", gitURL, tmpDir) 1163 err = o.Git().Init(tmpDir) 1164 if err != nil { 1165 return errors.Wrapf(err, "failed to init a new git repository in directory %s", tmpDir) 1166 } 1167 log.Logger().Debugf("ran git init in %s", tmpDir) 1168 err = o.Git().AddRemote(tmpDir, "origin", gitURL) 1169 if err != nil { 1170 return errors.Wrapf(err, "failed to add remote origin with url %s in directory %s", gitURL, tmpDir) 1171 } 1172 log.Logger().Debugf("ran git add remote origin %s in %s", gitURL, tmpDir) 1173 commitish := make([]string, 0) 1174 if pullRequestNumber != "" { 1175 pr := fmt.Sprintf("pull/%s/head:%s", pullRequestNumber, branch) 1176 log.Logger().Debugf("will fetch %s for %s in dir %s", pr, gitURL, tmpDir) 1177 commitish = append(commitish, pr) 1178 } 1179 if revision != "" { 1180 log.Logger().Debugf("will fetch %s for %s in dir %s", revision, gitURL, tmpDir) 1181 commitish = append(commitish, revision) 1182 } else { 1183 commitish = append(commitish, "master") 1184 } 1185 err = o.Git().FetchBranchShallow(tmpDir, "origin", commitish...) 1186 if err != nil { 1187 return errors.Wrapf(err, "failed to fetch %s from %s in directory %s", commitish, gitURL, tmpDir) 1188 } 1189 if revision != "" { 1190 err = o.Git().Checkout(tmpDir, revision) 1191 if err != nil { 1192 return errors.Wrapf(err, "failed to checkout revision %s", revision) 1193 } 1194 } else { 1195 err = o.Git().Checkout(tmpDir, "master") 1196 if err != nil { 1197 return errors.Wrapf(err, "failed to checkout revision master") 1198 } 1199 } 1200 return nil 1201 }) 1202 1203 // if we have failed to clone three times it's likely things wont recover so lets kill the process and let 1204 // kubernetes reschedule a new pod, however if it's because the revision didn't exist, then it's more likely it's 1205 // because that object is already deleted by a force-push 1206 if err != nil { 1207 if gits.IsUnadvertisedObjectError(err) { 1208 log.Logger().Warnf("Commit most likely overwritten by force-push, so ignorning underlying error %v", err) 1209 } else { 1210 log.Logger().Fatalf("failed to clone three times it's likely things wont recover so lets kill the process; %v", err) 1211 panic(err) 1212 } 1213 } 1214 1215 return tmpDir 1216 } 1217 1218 // parsePullRefs creates a Prow PullRefs struct from the PULL_REFS environment variable, if it id set. 1219 func (o *StepCreateTaskOptions) parsePullRefs() (*tekton.PullRefs, error) { 1220 var pr *tekton.PullRefs 1221 var err error 1222 1223 for _, envVar := range o.CustomEnvs { 1224 parts := strings.Split(envVar, "=") 1225 if parts[0] == "PULL_REFS" { 1226 pr, err = tekton.ParsePullRefs(parts[1]) 1227 if err != nil { 1228 return pr, err 1229 } 1230 } 1231 } 1232 1233 return pr, nil 1234 } 1235 1236 // mergePullRefs merges the pull refs specified into the git repository specified via CloneDir. 1237 func (o *StepCreateTaskOptions) mergePullRefs(pr *tekton.PullRefs, cloneDir string) error { 1238 if pr == nil { 1239 return nil 1240 } 1241 var shas []string 1242 for _, sha := range pr.ToMerge { 1243 shas = append(shas, sha) 1244 } 1245 1246 mergeOpts := git.StepGitMergeOptions{ 1247 StepOptions: step.StepOptions{ 1248 CommonOptions: o.CommonOptions, 1249 }, 1250 Dir: cloneDir, 1251 BaseSHA: pr.BaseSha, 1252 SHAs: shas, 1253 BaseBranch: pr.BaseBranch, 1254 } 1255 mergeOpts.Verbose = true 1256 err := mergeOpts.Run() 1257 if err != nil { 1258 return errors.Wrapf(err, "failed to merge git shas %s with base sha %s", shas, pr.BaseSha) 1259 } 1260 return nil 1261 } 1262 1263 func (o *StepCreateTaskOptions) viewSteps(tasks ...*pipelineapi.Task) error { 1264 table := o.CreateTable() 1265 showTaskName := len(tasks) > 1 1266 if showTaskName { 1267 table.AddRow("TASK", "NAME", "COMMAND", "IMAGE") 1268 } else { 1269 table.AddRow("NAME", "COMMAND", "IMAGE") 1270 } 1271 for _, task := range tasks { 1272 for _, step := range task.Spec.Steps { 1273 command := append([]string{}, step.Command...) 1274 command = append(command, step.Args...) 1275 commands := strings.Join(command, " ") 1276 if showTaskName { 1277 table.AddRow(task.Name, step.Name, commands, step.Image) 1278 } else { 1279 table.AddRow(step.Name, commands, step.Image) 1280 } 1281 } 1282 } 1283 table.Render() 1284 return nil 1285 } 1286 1287 func getVersionFromFile(dir string) (string, error) { 1288 var version string 1289 versionFile := filepath.Join(dir, "VERSION") 1290 exist, err := util.FileExists(versionFile) 1291 if err != nil { 1292 return "", err 1293 } 1294 if exist { 1295 data, err := ioutil.ReadFile(versionFile) 1296 if err != nil { 1297 return "", errors.Wrapf(err, "failed to read file %s", versionFile) 1298 } 1299 text := strings.TrimSpace(string(data)) 1300 if text == "" { 1301 log.Logger().Warnf("versions file %s is empty!", versionFile) 1302 } else { 1303 version = text 1304 if version != "" { 1305 return version, nil 1306 } 1307 } 1308 } 1309 return "", errors.New("failed to read file " + versionFile) 1310 } 1311 1312 func (o *StepCreateTaskOptions) setBuildVersion(projectConfig *config.ProjectConfig) error { 1313 if o.DockerRegistryOrg == "" { 1314 o.DockerRegistryOrg = o.GetDockerRegistryOrg(projectConfig, o.GitInfo) 1315 } 1316 if o.NoReleasePrepare || o.ViewSteps || o.EffectivePipeline || projectConfig.NoReleasePrepare { 1317 return nil 1318 } 1319 pipelineConfig := projectConfig.PipelineConfig 1320 version := "" 1321 1322 if o.DryRun { 1323 version, err := getVersionFromFile(o.CloneDir) 1324 if err != nil { 1325 log.Logger().Warn("No version file or incorrect content; using 0.0.1 as version") 1326 version = "0.0.1" 1327 } 1328 o.version = version 1329 o.setRevisionForReleasePipeline(version) 1330 o.pipelineParams = append(o.pipelineParams, pipelineapi.Param{ 1331 Name: "version", 1332 Value: syntax.StringParamValue(o.version), 1333 }) 1334 log.Logger().Infof("Version used: '%s'", util.ColorInfo(version)) 1335 1336 return nil 1337 } else if o.PipelineKind == jenkinsfile.PipelineKindRelease { 1338 release := pipelineConfig.Pipelines.Release 1339 if release == nil { 1340 return fmt.Errorf("no Release pipeline available") 1341 } 1342 sv := release.SetVersion 1343 if sv == nil { 1344 command := "jx step next-version --use-git-tag-only --tag" 1345 if o.SemanticRelease { 1346 command = "jx step next-version --semantic-release --tag" 1347 } 1348 // lets create a default set version pipeline 1349 sv = &jenkinsfile.PipelineLifecycle{ 1350 Steps: []*syntax.Step{ 1351 { 1352 Command: command, 1353 Name: "next-version", 1354 Comment: "tags git with the next version", 1355 }, 1356 }, 1357 } 1358 } 1359 steps := sv.Steps 1360 err := o.invokeSteps(steps) 1361 if err != nil { 1362 return err 1363 } 1364 version, err = getVersionFromFile(o.CloneDir) 1365 if err != nil { 1366 return err 1367 } 1368 o.setRevisionForReleasePipeline(version) 1369 } else { 1370 // lets use the branch name if we can find it for the version number 1371 branch := o.Branch 1372 if branch == "" { 1373 branch = o.Revision 1374 } 1375 buildNumber := o.BuildNumber 1376 o.previewVersionPrefix = "0.0.0-SNAPSHOT-" + branch + "-" 1377 version = o.previewVersionPrefix + buildNumber 1378 } 1379 if version != "" { 1380 if !hasParam(o.pipelineParams, "version") { 1381 o.pipelineParams = append(o.pipelineParams, pipelineapi.Param{ 1382 Name: "version", 1383 Value: syntax.StringParamValue(version), 1384 }) 1385 } 1386 } 1387 o.version = version 1388 if o.BuildNumber != "" { 1389 if !hasParam(o.pipelineParams, "build_id") { 1390 o.pipelineParams = append(o.pipelineParams, pipelineapi.Param{ 1391 Name: "build_id", 1392 Value: syntax.StringParamValue(o.BuildNumber), 1393 }) 1394 } 1395 } 1396 return nil 1397 } 1398 1399 func (o *StepCreateTaskOptions) setRevisionForReleasePipeline(version string) { 1400 if o.UseBranchAsRevision { 1401 o.Revision = o.Branch 1402 } else { 1403 o.Revision = "v" + version 1404 } 1405 } 1406 1407 func hasParam(params []pipelineapi.Param, name string) bool { 1408 for _, param := range params { 1409 if param.Name == name { 1410 return true 1411 } 1412 } 1413 return false 1414 } 1415 1416 func hasPipelineParam(params []pipelineapi.Param, name string) bool { 1417 for _, param := range params { 1418 if param.Name == name { 1419 return true 1420 } 1421 } 1422 return false 1423 } 1424 1425 func (o *StepCreateTaskOptions) runStepCommand(step *syntax.Step) error { 1426 c := step.GetFullCommand() 1427 if c == "" { 1428 return nil 1429 } 1430 log.Logger().Infof("running command: %s", util.ColorInfo(c)) 1431 1432 commandText := strings.Replace(step.GetFullCommand(), "\\$", "$", -1) 1433 1434 cmd := util.Command{ 1435 Name: util.GetSh(), 1436 Args: []string{"-c", commandText}, 1437 Out: o.Out, 1438 Err: o.Err, 1439 Dir: o.CloneDir, 1440 } 1441 result, err := cmd.RunWithoutRetry() 1442 if err != nil { 1443 return err 1444 } 1445 log.Logger().Infof("%s", result) 1446 return nil 1447 } 1448 1449 func (o *StepCreateTaskOptions) invokeSteps(steps []*syntax.Step) error { 1450 for _, s := range steps { 1451 if s == nil { 1452 continue 1453 } 1454 if len(s.Steps) > 0 { 1455 err := o.invokeSteps(s.Steps) 1456 if err != nil { 1457 return err 1458 } 1459 } 1460 when := strings.TrimSpace(s.When) 1461 if when == "!prow" || s.GetCommand() == "" { 1462 continue 1463 } 1464 err := o.runStepCommand(s) 1465 if err != nil { 1466 return err 1467 } 1468 } 1469 return nil 1470 } 1471 1472 func (o *StepCreateTaskOptions) getClientsAndNamespace() (tektonclient.Interface, jxclient.Interface, kubeclient.Interface, string, error) { 1473 tektonClient, _, err := o.TektonClient() 1474 if err != nil { 1475 return nil, nil, nil, "", errors.Wrap(err, "unable to create Tekton client") 1476 } 1477 1478 jxClient, _, err := o.JXClient() 1479 if err != nil { 1480 return nil, nil, nil, "", errors.Wrap(err, "unable to create JX client") 1481 } 1482 1483 kubeClient, ns, err := o.KubeClientAndDevNamespace() 1484 if err != nil { 1485 return nil, nil, nil, "", errors.Wrap(err, "unable to create Kube client") 1486 } 1487 1488 return tektonClient, jxClient, kubeClient, ns, nil 1489 } 1490 1491 func (o *StepCreateTaskOptions) interpretPipeline(ns string, projectConfig *config.ProjectConfig, crds *tekton.CRDWrapper) error { 1492 steps := []corev1.Container{} 1493 for _, task := range crds.Tasks() { 1494 for _, s := range task.Spec.Steps { 1495 steps = append(steps, s.Container) 1496 } 1497 } 1498 1499 if o.StartStep != "" { 1500 found := false 1501 for i, step := range steps { 1502 if step.Name == o.StartStep { 1503 found = true 1504 steps = steps[i:] 1505 break 1506 } 1507 } 1508 if !found { 1509 names := []string{} 1510 for _, step := range steps { 1511 names = append(names, step.Name) 1512 } 1513 return util.InvalidOption("start-step", o.StartStep, names) 1514 } 1515 } 1516 1517 if o.EndStep != "" { 1518 found := false 1519 for i, step := range steps { 1520 if step.Name == o.EndStep { 1521 found = true 1522 steps = steps[:i+1] 1523 break 1524 } 1525 } 1526 if !found { 1527 names := []string{} 1528 for _, step := range steps { 1529 names = append(names, step.Name) 1530 } 1531 return util.InvalidOption("end-step", o.EndStep, names) 1532 } 1533 } 1534 1535 for _, step := range steps { 1536 s := step 1537 err := o.interpretStep(ns, &s) 1538 if err != nil { 1539 return err 1540 } 1541 } 1542 return nil 1543 } 1544 1545 func (o *StepCreateTaskOptions) interpretStep(ns string, step *corev1.Container) error { 1546 command := step.Command 1547 if len(command) == 0 { 1548 return nil 1549 } 1550 1551 // ignore some unnecessary commands 1552 // TODO is there a nicer way to disable the git-merge step? 1553 if step.Name == "git-merge" || step.Name == "setup-builder-home" { 1554 return nil 1555 } 1556 commandAndArgs := append(step.Command, step.Args...) 1557 commandLine := strings.Join(commandAndArgs, " ") 1558 dir := step.WorkingDir 1559 if dir != "" { 1560 workspaceDir := o.getWorkspaceDir() 1561 if strings.HasPrefix(dir, workspaceDir) { 1562 curDir := o.CloneDir 1563 if curDir == "" { 1564 var err error 1565 curDir, err = os.Getwd() 1566 if err != nil { 1567 return err 1568 } 1569 } 1570 relPath, err := filepath.Rel(workspaceDir, dir) 1571 if err != nil { 1572 return err 1573 } 1574 dir = filepath.Join(curDir, relPath) 1575 } 1576 } 1577 1578 envMap := createEnvMapForInterpretExecution(step.Env) 1579 1580 suffix := "" 1581 if o.Verbose { 1582 suffix = fmt.Sprintf(" with env: %s", util.ColorInfo(fmt.Sprintf("%#v", envMap))) 1583 } 1584 path, err := filepath.Abs(dir) 1585 if err != nil { 1586 path = dir 1587 } 1588 log.Logger().Infof("\nSTEP: %s command: %s in dir: %s%s\n\n", util.ColorInfo(step.Name), util.ColorInfo(commandLine), util.ColorInfo(path), suffix) 1589 1590 if !o.DryRun { 1591 cmd := util.Command{ 1592 Name: commandAndArgs[0], 1593 Args: commandAndArgs[1:], 1594 Dir: dir, 1595 Out: os.Stdout, 1596 Err: os.Stdout, 1597 In: os.Stdin, 1598 Env: envMap, 1599 } 1600 _, err := cmd.RunWithoutRetry() 1601 if err != nil { 1602 return err 1603 } 1604 } else { 1605 log.Logger().Infof("%s", envMap) 1606 } 1607 1608 return nil 1609 } 1610 1611 func createEnvMapForInterpretExecution(envVars []corev1.EnvVar) map[string]string { 1612 m := map[string]string{} 1613 for _, envVar := range envVars { 1614 m[envVar.Name] = envVar.Value 1615 } 1616 1617 if _, exists := m["JX_LOG_LEVEL"]; !exists { 1618 m["JX_LOG_LEVEL"] = log.GetLevel() 1619 } 1620 1621 return m 1622 } 1623 1624 // isKanikoExecutorStep looks at a container and determines whether its command or args starts with /kaniko/executor. 1625 func isKanikoExecutorStep(container *corev1.Container) bool { 1626 return strings.HasPrefix(strings.Join(container.Command, " "), "/kaniko/executor") || 1627 (len(container.Args) > 0 && strings.HasPrefix(strings.Join(container.Args, " "), "/kaniko/executor")) 1628 }